diff --git a/.gitignore b/.gitignore index 5274fea0..ce60b65c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,9 @@ build/ # Ignore .cache directory generated by clangd .cache/ + +# Ignore any Python Virtual Environments +.venv/ + +# Ignore any pycache files +__pycache__/ diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt new file mode 100644 index 00000000..afac9114 --- /dev/null +++ b/python/CMakeLists.txt @@ -0,0 +1,67 @@ +cmake_minimum_required(VERSION 3.21) +project(hipfile-python LANGUAGES C) + +# ---- Python & Cython ----------------------------------------------------- + +find_package(Python REQUIRED COMPONENTS Interpreter Development.Module) +find_program(CYTHON_EXECUTABLE cython REQUIRED) + +# ---- hipFile library & headers -------------------------------------------- + +# hipfile.h location (default: sibling include/ directory) +set(HIPFILE_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../include" + CACHE PATH "Path to directory containing hipfile.h") + +# libhipfile.so location +find_library(HIPFILE_LIBRARY hipfile + HINTS + "${CMAKE_CURRENT_SOURCE_DIR}/../build/src/amd_detail" + "/opt/rocm/lib" +) +if(NOT HIPFILE_LIBRARY) + message(FATAL_ERROR + "Could not find libhipfile. Set -DHIPFILE_LIBRARY= or " + "install hipfile to /opt/rocm.") +endif() + +# HIP runtime headers (needed because hipfile.h includes hip/hip_runtime_api.h) +find_path(HIP_INCLUDE_DIR hip/hip_runtime_api.h + HINTS + "/opt/rocm/include" + "/opt/rocm/hip/include" +) +if(NOT HIP_INCLUDE_DIR) + message(FATAL_ERROR + "Could not find hip/hip_runtime_api.h. Set -DHIP_INCLUDE_DIR= " + "or install HIP development headers.") +endif() + +# ---- Cythonize ------------------------------------------------------------ + +set(_PYX_SRC "${CMAKE_CURRENT_SOURCE_DIR}/hipfile/_hipfile.pyx") +set(_C_OUT "${CMAKE_CURRENT_BINARY_DIR}/_hipfile.c") + +add_custom_command( + OUTPUT "${_C_OUT}" + COMMAND "${CYTHON_EXECUTABLE}" "${_PYX_SRC}" -o "${_C_OUT}" -3 + DEPENDS + "${_PYX_SRC}" + "${CMAKE_CURRENT_SOURCE_DIR}/hipfile/_chipfile.pxd" + COMMENT "Cythonizing _hipfile.pyx" +) + +# ---- Build extension module ----------------------------------------------- +# lint_cmake: -readability/wonkycase +Python_add_library(_hipfile MODULE "${_C_OUT}" WITH_SOABI) +# lint_cmake: +readability/wonkycase +target_include_directories(_hipfile PRIVATE + "${HIPFILE_INCLUDE_DIR}" + "${HIP_INCLUDE_DIR}" +) + +target_link_libraries(_hipfile PRIVATE "${HIPFILE_LIBRARY}") + +# _hipfile bindings only support AMD for now +target_compile_definitions(_hipfile PRIVATE __HIP_PLATFORM_AMD__) + +install(TARGETS _hipfile DESTINATION hipfile) diff --git a/python/README.md b/python/README.md new file mode 100644 index 00000000..1769f457 --- /dev/null +++ b/python/README.md @@ -0,0 +1,25 @@ +# hipFile Python Bindings + +> [!CAUTION] +> These bindings in particular are *experimental* and the API will change. + +## Building & Installing + +1. Setup a Python virtual environment. +```bash +$ python3 -m venv .venv +``` +2. Activate the Python virtual environment. +```bash +$ source .venv/bin/activate +``` +3. Build the C hipFile library. See [INSTALL.md](../INSTALL.md). +4. Build & Install the hipFile package. +```bash +(.venv) $ pip install -e python -Ccmake.define.HIPFILE_INCLUDE_DIR=../include -Ccmake.define.HIP_INCLUDE_DIR=/opt/rocm/include +``` + +This will install an editable version of hipFile in your virtual environment. +It is editable in the sense that any changes you make to the hipFile Python +source code will be immediately available for any tests/scripts that use the hipFile library. +Any changes to the Cython source code will require a rebuild step. \ No newline at end of file diff --git a/python/hipfile/__init__.py b/python/hipfile/__init__.py new file mode 100644 index 00000000..e68d4dff --- /dev/null +++ b/python/hipfile/__init__.py @@ -0,0 +1,27 @@ +# pylint: disable=C0114 + +from hipfile._hipfile import ( # pylint: disable=E0401,E0611 + # Constants + VERSION_MAJOR as _VERSION_MAJOR, + VERSION_MINOR as _VERSION_MINOR, + VERSION_PATCH as _VERSION_PATCH, +) +from hipfile.buffer import Buffer +from hipfile.driver import Driver +from hipfile.enums import FileHandleType, OpError +from hipfile.error import HipFileException +from hipfile.file import FileHandle +from hipfile.properties import driver_get_properties, get_version + +__all__ = [ + "__version__", + "Driver", + "FileHandle", + "Buffer", + "HipFileException", + "FileHandleType", + "OpError", + "driver_get_properties", + "get_version", +] +__version__ = f"{_VERSION_MAJOR}.{_VERSION_MINOR}.{_VERSION_PATCH}" diff --git a/python/hipfile/_chipfile.pxd b/python/hipfile/_chipfile.pxd new file mode 100644 index 00000000..cb841e72 --- /dev/null +++ b/python/hipfile/_chipfile.pxd @@ -0,0 +1,190 @@ +# cython: language_level=3 +""" +C declarations for hipFile API (extern from hipfile.h). + +This .pxd file declares the subset of the hipFile C API that is +wrapped by the low-level Cython bindings. +""" + +from libc.stdint cimport int64_t, uint64_t +from posix.types cimport off_t + + +# --------------------------------------------------------------------------- +# HIP runtime stub — only hipError_t is needed +# --------------------------------------------------------------------------- + +cdef extern from "hip/hip_runtime_api.h": + ctypedef enum hipError_t: + hipSuccess = 0 + + hipError_t hipPeekAtLastError() + + +# --------------------------------------------------------------------------- +# hipFile public API +# --------------------------------------------------------------------------- + +cdef extern from "hipfile.h": + + # -- Version constants -------------------------------------------------- + + int HIPFILE_VERSION_MAJOR + int HIPFILE_VERSION_MINOR + int HIPFILE_VERSION_PATCH + int HIPFILE_BASE_ERR + + # -- Platform-independent types ----------------------------------------- + + ctypedef off_t hoff_t + + # -- Error handling ----------------------------------------------------- + + ctypedef enum hipFileOpError_t: + hipFileSuccess + hipFileDriverNotInitialized + hipFileDriverInvalidProps + hipFileDriverUnsupportedLimit + hipFileDriverVersionMismatch + hipFileDriverVersionReadError + hipFileDriverClosing + hipFilePlatformNotSupported + hipFileIONotSupported + hipFileDeviceNotSupported + hipFileDriverError + hipFileHipDriverError + hipFileHipPointerInvalid + hipFileHipMemoryTypeInvalid + hipFileHipPointerRangeError + hipFileHipContextMismatch + hipFileInvalidMappingSize + hipFileInvalidMappingRange + hipFileInvalidFileType + hipFileInvalidFileOpenFlag + hipFileDIONotSet + # 5021 intentionally unused + hipFileInvalidValue + hipFileMemoryAlreadyRegistered + hipFileMemoryNotRegistered + hipFilePermissionDenied + hipFileDriverAlreadyOpen + hipFileHandleNotRegistered + hipFileHandleAlreadyRegistered + hipFileDeviceNotFound + hipFileInternalError + hipFileGetNewFDFailed + # 5032 intentionally unused + hipFileDriverSetupError + hipFileIODisabled + hipFileBatchSubmitFailed + hipFileGPUMemoryPinningFailed + hipFileBatchFull + hipFileAsyncNotSupported + hipFileIOMaxError + + ctypedef struct hipFileError_t: + hipFileOpError_t err + hipError_t hip_drv_err + + # -- Opaque handles ----------------------------------------------------- + + ctypedef void *hipFileHandle_t + + # -- File handle types -------------------------------------------------- + + ctypedef enum hipFileFileHandleType_t: + hipFileHandleTypeOpaqueFD + hipFileHandleTypeOpaqueWin32 + hipFileHandleTypeUserspaceFS + + # -- Userspace FS ops (opaque — only needed as pointer type) ------------ + + ctypedef struct hipFileFSOps_t: + pass + + # -- File descriptor ---------------------------------------------------- + # The anonymous union is accessed via Cython C-name strings. + + ctypedef struct hipFileDescr_t: + hipFileFileHandleType_t type + int fd "handle.fd" + void *hFile "handle.hFile" + const hipFileFSOps_t *fs_ops + + # -- Driver status / control / feature flag enums ----------------------- + + ctypedef enum hipFileDriverStatusFlags_t: + hipFileLustreSupported + hipFileWekaFSSupported + hipFileNFSSupported + hipFileGPFSSupported + hipFileNVMeSupported + hipFileNVMeoFSupported + hipFileSCSISupported + hipFileScaleFluxCSDSupported + hipFileNVMeshSupported + hipFileBeeGFSSupported + # 10 reserved for YRCloudFile + hipFileNVMeP2PSupported + hipFileScatefsSupported + + ctypedef enum hipFileDriverControlFlags_t: + hipFileUsePollMode + hipFileAllowCompatMode + + ctypedef enum hipFileFeatureFlags_t: + hipFileDynRoutingSupported + hipFileBatchIOSupported + hipFileStreamsSupported + hipFileParallelIOSupported + + # -- Driver properties -------------------------------------------------- + # Nested anonymous struct ``nvfs`` is flattened with C-name strings. + + ctypedef struct hipFileDriverProps_t: + unsigned int nvfs_major_version "nvfs.major_version" + unsigned int nvfs_minor_version "nvfs.minor_version" + uint64_t nvfs_poll_thresh_size "nvfs.poll_thresh_size" + uint64_t nvfs_max_direct_io_size "nvfs.max_direct_io_size" + unsigned int nvfs_driver_status_flags "nvfs.driver_status_flags" + unsigned int nvfs_driver_control_flags "nvfs.driver_control_flags" + unsigned int feature_flags + uint64_t max_device_cache_size + uint64_t per_buffer_cache_size + uint64_t max_device_pinned_mem_size + unsigned int max_batch_io_count + unsigned int max_batch_io_timeout_msecs + + # -- Function declarations ---------------------------------------------- + + # Error + const char *hipFileGetOpErrorString(hipFileOpError_t status) + + # File handles + hipFileError_t hipFileHandleRegister(hipFileHandle_t *fh, + hipFileDescr_t *descr) + void hipFileHandleDeregister(hipFileHandle_t fh) + + # Buffer registration + hipFileError_t hipFileBufRegister(const void *buffer_base, + size_t length, int flags) + hipFileError_t hipFileBufDeregister(const void *buffer_base) + + # Synchronous I/O + ssize_t hipFileRead(hipFileHandle_t fh, void *buffer_base, size_t size, + hoff_t file_offset, hoff_t buffer_offset) + ssize_t hipFileWrite(hipFileHandle_t fh, const void *buffer_base, + size_t size, hoff_t file_offset, + hoff_t buffer_offset) + + # Driver lifecycle + hipFileError_t hipFileDriverOpen() + hipFileError_t hipFileDriverClose() + int64_t hipFileUseCount() + + # Driver properties + hipFileError_t hipFileDriverGetProperties(hipFileDriverProps_t *props) + + # Version + hipFileError_t hipFileGetVersion(unsigned *major, unsigned *minor, + unsigned *patch) diff --git a/python/hipfile/_hipfile.pyx b/python/hipfile/_hipfile.pyx new file mode 100644 index 00000000..e74aeb3e --- /dev/null +++ b/python/hipfile/_hipfile.pyx @@ -0,0 +1,284 @@ +# cython: language_level=3 +""" +Low-level Cython wrappers for the hipFile C API. + +Every function mirrors the C API as closely as possible. +Functions that return ``hipFileError_t`` in C return a +``(hipFileOpError_t, hipError_t)`` 2-tuple here. +""" + +from libc.errno cimport errno +from libc.string cimport memset +from libc.stdint cimport uintptr_t + +cimport hipfile._chipfile as _c + + +# --------------------------------------------------------------------------- +# Module-level constants +# --------------------------------------------------------------------------- + +VERSION_MAJOR = _c.HIPFILE_VERSION_MAJOR +VERSION_MINOR = _c.HIPFILE_VERSION_MINOR +VERSION_PATCH = _c.HIPFILE_VERSION_PATCH +BASE_ERR = _c.HIPFILE_BASE_ERR + +# --------------------------------------------------------------------------- +# Enum re-exports (C → Python) +# +# ctypedef enum values from _chipfile.pxd are C-level only after cimport. +# These assignments create Python-visible module attributes whose values +# are resolved from the C enum at compile time. +# --------------------------------------------------------------------------- + +# hipFileOpError_t +hipFileSuccess = _c.hipFileSuccess +hipFileDriverNotInitialized = _c.hipFileDriverNotInitialized +hipFileDriverInvalidProps = _c.hipFileDriverInvalidProps +hipFileDriverUnsupportedLimit = _c.hipFileDriverUnsupportedLimit +hipFileDriverVersionMismatch = _c.hipFileDriverVersionMismatch +hipFileDriverVersionReadError = _c.hipFileDriverVersionReadError +hipFileDriverClosing = _c.hipFileDriverClosing +hipFilePlatformNotSupported = _c.hipFilePlatformNotSupported +hipFileIONotSupported = _c.hipFileIONotSupported +hipFileDeviceNotSupported = _c.hipFileDeviceNotSupported +hipFileDriverError = _c.hipFileDriverError +hipFileHipDriverError = _c.hipFileHipDriverError +hipFileHipPointerInvalid = _c.hipFileHipPointerInvalid +hipFileHipMemoryTypeInvalid = _c.hipFileHipMemoryTypeInvalid +hipFileHipPointerRangeError = _c.hipFileHipPointerRangeError +hipFileHipContextMismatch = _c.hipFileHipContextMismatch +hipFileInvalidMappingSize = _c.hipFileInvalidMappingSize +hipFileInvalidMappingRange = _c.hipFileInvalidMappingRange +hipFileInvalidFileType = _c.hipFileInvalidFileType +hipFileInvalidFileOpenFlag = _c.hipFileInvalidFileOpenFlag +hipFileDIONotSet = _c.hipFileDIONotSet +hipFileInvalidValue = _c.hipFileInvalidValue +hipFileMemoryAlreadyRegistered = _c.hipFileMemoryAlreadyRegistered +hipFileMemoryNotRegistered = _c.hipFileMemoryNotRegistered +hipFilePermissionDenied = _c.hipFilePermissionDenied +hipFileDriverAlreadyOpen = _c.hipFileDriverAlreadyOpen +hipFileHandleNotRegistered = _c.hipFileHandleNotRegistered +hipFileHandleAlreadyRegistered = _c.hipFileHandleAlreadyRegistered +hipFileDeviceNotFound = _c.hipFileDeviceNotFound +hipFileInternalError = _c.hipFileInternalError +hipFileGetNewFDFailed = _c.hipFileGetNewFDFailed +hipFileDriverSetupError = _c.hipFileDriverSetupError +hipFileIODisabled = _c.hipFileIODisabled +hipFileBatchSubmitFailed = _c.hipFileBatchSubmitFailed +hipFileGPUMemoryPinningFailed = _c.hipFileGPUMemoryPinningFailed +hipFileBatchFull = _c.hipFileBatchFull +hipFileAsyncNotSupported = _c.hipFileAsyncNotSupported +hipFileIOMaxError = _c.hipFileIOMaxError + +# hipFileFileHandleType_t +hipFileHandleTypeOpaqueFD = _c.hipFileHandleTypeOpaqueFD +hipFileHandleTypeOpaqueWin32 = _c.hipFileHandleTypeOpaqueWin32 +hipFileHandleTypeUserspaceFS = _c.hipFileHandleTypeUserspaceFS + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + +cdef inline tuple _err(_c.hipFileError_t e): + return (e.err, e.hip_drv_err) + + +# --------------------------------------------------------------------------- +# Error-handling helpers (replacements for C macros) +# --------------------------------------------------------------------------- + +def is_hipfile_err(int err_code): + """Equivalent of the ``IS_HIPFILE_ERR`` C macro.""" + return abs(err_code) > _c.HIPFILE_BASE_ERR + + +def hipfile_errstr(int err_code): + """Equivalent of the ``HIPFILE_ERRSTR`` C macro.""" + cdef const char *s = _c.hipFileGetOpErrorString(<_c.hipFileOpError_t>abs(err_code)) + if s == NULL: + return "" + return s.decode("utf-8") + + +def is_hip_drv_err(tuple err): + """Equivalent of the ``IS_HIP_DRV_ERR`` C macro. + + Takes an error tuple as returned by the wrapper functions. + """ + return err[0] == _c.hipFileHipDriverError + + +def hip_drv_err(tuple err): + """Equivalent of the ``HIP_DRV_ERR`` C macro. + + Takes an error tuple and returns the ``hipError_t`` component. + """ + return err[1] + + +def hipFileGetOpErrorString(int status): + """Wrapper for ``hipFileGetOpErrorString``.""" + cdef const char *s = _c.hipFileGetOpErrorString(<_c.hipFileOpError_t>status) + if s == NULL: + return "" + return s.decode("utf-8") + + +# --------------------------------------------------------------------------- +# Driver lifecycle +# --------------------------------------------------------------------------- + +def hipFileDriverOpen(): + """Wrapper for ``hipFileDriverOpen``.""" + return _err(_c.hipFileDriverOpen()) + + +def hipFileDriverClose(): + """Wrapper for ``hipFileDriverClose``.""" + return _err(_c.hipFileDriverClose()) + + +def hipFileUseCount(): + """Wrapper for ``hipFileUseCount``.""" + return _c.hipFileUseCount() + + +# --------------------------------------------------------------------------- +# Version +# --------------------------------------------------------------------------- + +def hipFileGetVersion(): + """Wrapper for ``hipFileGetVersion``. + + Returns ``((major, minor, patch), error_tuple)``. + """ + cdef unsigned major = 0, minor = 0, patch = 0 + cdef _c.hipFileError_t e = _c.hipFileGetVersion(&major, &minor, &patch) + return ((major, minor, patch), _err(e)) + + +# --------------------------------------------------------------------------- +# File handles +# --------------------------------------------------------------------------- + +def hipFileHandleRegister(uintptr_t handle_value, int handle_type): + """Wrapper for ``hipFileHandleRegister``. + + Parameters + ---------- + handle_value : int + POSIX file descriptor or Win32 HANDLE, depending on *handle_type*. + handle_type : int + Value from ``hipFileFileHandleType_t``. + + Returns ``(handle_int, error_tuple)``. The handle is an opaque + integer that must be passed back to other hipFile calls. + """ + cdef _c.hipFileHandle_t fh = NULL + cdef _c.hipFileDescr_t descr + memset(&descr, 0, sizeof(descr)) + descr.type = <_c.hipFileFileHandleType_t>handle_type + if handle_type == _c.hipFileHandleTypeOpaqueWin32: + descr.hFile = handle_value + else: + descr.fd = handle_value + cdef _c.hipFileError_t e = _c.hipFileHandleRegister(&fh, &descr) + return (fh, _err(e)) + + +def hipFileHandleDeregister(uintptr_t handle): + """Wrapper for ``hipFileHandleDeregister``.""" + _c.hipFileHandleDeregister(<_c.hipFileHandle_t>handle) + + +# --------------------------------------------------------------------------- +# Buffer registration +# --------------------------------------------------------------------------- + +def hipFileBufRegister(uintptr_t buffer_base, size_t length, int flags=0): + """Wrapper for ``hipFileBufRegister``.""" + return _err(_c.hipFileBufRegister(buffer_base, length, flags)) + + +def hipFileBufDeregister(uintptr_t buffer_base): + """Wrapper for ``hipFileBufDeregister``.""" + return _err(_c.hipFileBufDeregister(buffer_base)) + + +# --------------------------------------------------------------------------- +# Synchronous I/O +# --------------------------------------------------------------------------- + +def hipFileRead(uintptr_t handle, uintptr_t buffer_base, size_t size, + _c.hoff_t file_offset, _c.hoff_t buffer_offset): + """Wrapper for ``hipFileRead``. + + Returns ``(result, extra)``: + * ``result >= 0`` — number of bytes read, ``extra = 0`` + * ``result == -1`` — system error, ``extra = errno`` + * ``result < -1`` — negated ``hipFileOpError_t``; if + ``-hipFileHipDriverError``, ``extra = hipError_t`` from + ``hipPeekAtLastError()``, otherwise ``extra = 0`` + """ + cdef ssize_t ret = _c.hipFileRead(<_c.hipFileHandle_t>handle, + buffer_base, size, + file_offset, buffer_offset) + cdef int extra = 0 + if ret == -1: + extra = errno + elif ret == -_c.hipFileHipDriverError: + extra = _c.hipPeekAtLastError() + return (ret, extra) + + +def hipFileWrite(uintptr_t handle, uintptr_t buffer_base, size_t size, + _c.hoff_t file_offset, _c.hoff_t buffer_offset): + """Wrapper for ``hipFileWrite``. + + Returns ``(result, extra)``: + * ``result >= 0`` — number of bytes written, ``extra = 0`` + * ``result == -1`` — system error, ``extra = errno`` + * ``result < -1`` — negated ``hipFileOpError_t``; if + ``-hipFileHipDriverError``, ``extra = hipError_t`` from + ``hipPeekAtLastError()``, otherwise ``extra = 0`` + """ + cdef ssize_t ret = _c.hipFileWrite(<_c.hipFileHandle_t>handle, + buffer_base, size, + file_offset, buffer_offset) + cdef int extra = 0 + if ret == -1: + extra = errno + elif ret == -_c.hipFileHipDriverError: + extra = _c.hipPeekAtLastError() + return (ret, extra) + + +# --------------------------------------------------------------------------- +# Driver properties +# --------------------------------------------------------------------------- + +def hipFileDriverGetProperties(): + """Wrapper for ``hipFileDriverGetProperties``. + + Returns ``(props_dict, error_tuple)``. + """ + cdef _c.hipFileDriverProps_t props + memset(&props, 0, sizeof(props)) + cdef _c.hipFileError_t e = _c.hipFileDriverGetProperties(&props) + d = { + "nvfs_major_version": props.nvfs_major_version, + "nvfs_minor_version": props.nvfs_minor_version, + "nvfs_poll_thresh_size": props.nvfs_poll_thresh_size, + "nvfs_max_direct_io_size": props.nvfs_max_direct_io_size, + "nvfs_driver_status_flags": props.nvfs_driver_status_flags, + "nvfs_driver_control_flags": props.nvfs_driver_control_flags, + "feature_flags": props.feature_flags, + "max_device_cache_size": props.max_device_cache_size, + "per_buffer_cache_size": props.per_buffer_cache_size, + "max_device_pinned_mem_size": props.max_device_pinned_mem_size, + "max_batch_io_count": props.max_batch_io_count, + "max_batch_io_timeout_msecs": props.max_batch_io_timeout_msecs, + } + return (d, _err(e)) diff --git a/python/hipfile/buffer.py b/python/hipfile/buffer.py new file mode 100644 index 00000000..e5f9b60a --- /dev/null +++ b/python/hipfile/buffer.py @@ -0,0 +1,59 @@ +# pylint: disable=C0114,C0115,C0116 +from __future__ import annotations +from typing import TYPE_CHECKING +from sys import stderr + +from hipfile._hipfile import ( # pylint: disable=E0401,E0611 + hipFileBufDeregister, + hipFileBufRegister, +) +from hipfile.error import HipFileException + +if TYPE_CHECKING: + from ctypes import c_void_p + + +class Buffer: + + @classmethod + def from_ctypes_void_p(cls, ctypes_void_p: c_void_p, length, flags): + return cls(ctypes_void_p.value, length, flags) + + def __init__(self, buffer_ptr, length, flags) -> None: + self._buffer_ptr = buffer_ptr + self._flags = flags + self._length = length + self._registered = False + + def __del__(self): + # We did not create the underlying buffer. Don't try to free it. + try: + self.deregister() + except Exception: # pylint: disable=W0718 # Suppress exceptions in a dtor + print( + "Failed to deregister hipFile.Buffer at destruction time.", file=stderr + ) + + def __enter__(self): + self.register() + return self + + def __exit__(self, exc_type, exc, tb): + self.deregister() + + @property + def ptr(self): + return self._buffer_ptr + + def deregister(self): + if self._registered: + err = hipFileBufDeregister(self._buffer_ptr) + if err[0] != 0: + raise HipFileException(err[0], err[1]) + self._registered = False + + def register(self): + err = hipFileBufRegister(self._buffer_ptr, self._length, self._flags) + if err[0] != 0: + raise HipFileException(err[0], err[1]) + self._registered = True diff --git a/python/hipfile/driver.py b/python/hipfile/driver.py new file mode 100644 index 00000000..16242c58 --- /dev/null +++ b/python/hipfile/driver.py @@ -0,0 +1,31 @@ +# pylint: disable=C0114,C0115,C0116 +from hipfile._hipfile import ( # pylint: disable=E0401,E0611 + hipFileDriverOpen, + hipFileDriverClose, + hipFileUseCount, +) +from hipfile.error import HipFileException + + +class Driver: + + @staticmethod + def use_count(): + return hipFileUseCount() + + def __enter__(self): + self.open() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + def close(self): + err = hipFileDriverClose() + if err[0] != 0: + raise HipFileException(err[0], err[1]) + + def open(self): + err = hipFileDriverOpen() + if err[0] != 0: + raise HipFileException(err[0], err[1]) diff --git a/python/hipfile/enums.py b/python/hipfile/enums.py new file mode 100644 index 00000000..61e7a08c --- /dev/null +++ b/python/hipfile/enums.py @@ -0,0 +1,107 @@ +# pylint: disable=C0114,C0115,C0116 +from enum import IntEnum + +from hipfile._hipfile import ( # pylint: disable=E0401,E0611 + # hipFileOpError_t values (resolved from C at build time) + hipFileSuccess, + hipFileDriverNotInitialized, + hipFileDriverInvalidProps, + hipFileDriverUnsupportedLimit, + hipFileDriverVersionMismatch, + hipFileDriverVersionReadError, + hipFileDriverClosing, + hipFilePlatformNotSupported, + hipFileIONotSupported, + hipFileDeviceNotSupported, + hipFileDriverError, + hipFileHipDriverError, + hipFileHipPointerInvalid, + hipFileHipMemoryTypeInvalid, + hipFileHipPointerRangeError, + hipFileHipContextMismatch, + hipFileInvalidMappingSize, + hipFileInvalidMappingRange, + hipFileInvalidFileType, + hipFileInvalidFileOpenFlag, + hipFileDIONotSet, + hipFileInvalidValue, + hipFileMemoryAlreadyRegistered, + hipFileMemoryNotRegistered, + hipFilePermissionDenied, + hipFileDriverAlreadyOpen, + hipFileHandleNotRegistered, + hipFileHandleAlreadyRegistered, + hipFileDeviceNotFound, + hipFileInternalError, + hipFileGetNewFDFailed, + hipFileDriverSetupError, + hipFileIODisabled, + hipFileBatchSubmitFailed, + hipFileGPUMemoryPinningFailed, + hipFileBatchFull, + hipFileAsyncNotSupported, + hipFileIOMaxError, + # hipFileFileHandleType_t values (resolved from C at build time) + hipFileHandleTypeOpaqueFD, + hipFileHandleTypeOpaqueWin32, + hipFileHandleTypeUserspaceFS, +) + + +class OpError(IntEnum): + """Python enum mirroring hipFileOpError_t. + + Values are sourced from the C enum via the Cython layer, not + redefined. Rebuilding the extension picks up any value changes + in hipfile.h automatically. + """ + + SUCCESS = hipFileSuccess + DRIVER_NOT_INITIALIZED = hipFileDriverNotInitialized + DRIVER_INVALID_PROPS = hipFileDriverInvalidProps + DRIVER_UNSUPPORTED_LIMIT = hipFileDriverUnsupportedLimit + DRIVER_VERSION_MISMATCH = hipFileDriverVersionMismatch + DRIVER_VERSION_READ_ERROR = hipFileDriverVersionReadError + DRIVER_CLOSING = hipFileDriverClosing + PLATFORM_NOT_SUPPORTED = hipFilePlatformNotSupported + IO_NOT_SUPPORTED = hipFileIONotSupported + DEVICE_NOT_SUPPORTED = hipFileDeviceNotSupported + DRIVER_ERROR = hipFileDriverError + HIP_DRIVER_ERROR = hipFileHipDriverError + HIP_POINTER_INVALID = hipFileHipPointerInvalid + HIP_MEMORY_TYPE_INVALID = hipFileHipMemoryTypeInvalid + HIP_POINTER_RANGE_ERROR = hipFileHipPointerRangeError + HIP_CONTEXT_MISMATCH = hipFileHipContextMismatch + INVALID_MAPPING_SIZE = hipFileInvalidMappingSize + INVALID_MAPPING_RANGE = hipFileInvalidMappingRange + INVALID_FILE_TYPE = hipFileInvalidFileType + INVALID_FILE_OPEN_FLAG = hipFileInvalidFileOpenFlag + DIO_NOT_SET = hipFileDIONotSet + INVALID_VALUE = hipFileInvalidValue + MEMORY_ALREADY_REGISTERED = hipFileMemoryAlreadyRegistered + MEMORY_NOT_REGISTERED = hipFileMemoryNotRegistered + PERMISSION_DENIED = hipFilePermissionDenied + DRIVER_ALREADY_OPEN = hipFileDriverAlreadyOpen + HANDLE_NOT_REGISTERED = hipFileHandleNotRegistered + HANDLE_ALREADY_REGISTERED = hipFileHandleAlreadyRegistered + DEVICE_NOT_FOUND = hipFileDeviceNotFound + INTERNAL_ERROR = hipFileInternalError + GET_NEW_FD_FAILED = hipFileGetNewFDFailed + DRIVER_SETUP_ERROR = hipFileDriverSetupError + IO_DISABLED = hipFileIODisabled + BATCH_SUBMIT_FAILED = hipFileBatchSubmitFailed + GPU_MEMORY_PINNING_FAILED = hipFileGPUMemoryPinningFailed + BATCH_FULL = hipFileBatchFull + ASYNC_NOT_SUPPORTED = hipFileAsyncNotSupported + IO_MAX_ERROR = hipFileIOMaxError + + +class FileHandleType(IntEnum): + """Python enum mirroring hipFileFileHandleType_t. + + Values are sourced from the C enum via the Cython layer. + """ + + OPAQUE_FD = hipFileHandleTypeOpaqueFD + OPAQUE_WIN32 = hipFileHandleTypeOpaqueWin32 + USERSPACE_FS = hipFileHandleTypeUserspaceFS diff --git a/python/hipfile/error.py b/python/hipfile/error.py new file mode 100644 index 00000000..f20882aa --- /dev/null +++ b/python/hipfile/error.py @@ -0,0 +1,23 @@ +# pylint: disable=C0114,C0115,C0116 +from hipfile._hipfile import hipFileGetOpErrorString # pylint: disable=E0401,E0611 +from hipfile.enums import OpError + + +class HipFileException(Exception): + def __init__(self, hipfile_err, hip_err): + self._hipfile_err = hipfile_err + self._hip_err = hip_err + + @property + def hipfile_err(self): + return self._hipfile_err + + @property + def hip_err(self): + return self._hip_err + + def __str__(self): + err_msg = f"{self._hipfile_err} - {hipFileGetOpErrorString(self._hipfile_err)}" + if self._hipfile_err == OpError.HIP_DRIVER_ERROR: + err_msg += f" {self._hip_err}" + return err_msg diff --git a/python/hipfile/file.py b/python/hipfile/file.py new file mode 100644 index 00000000..9dfe327d --- /dev/null +++ b/python/hipfile/file.py @@ -0,0 +1,120 @@ +# pylint: disable=C0114,C0115,C0116 +import os +import stat + +from hipfile._hipfile import ( # pylint: disable=E0401,E0611 + hipFileHandleRegister, + hipFileHandleDeregister, + hipFileRead, + hipFileWrite, +) +from hipfile.enums import FileHandleType +from hipfile.error import HipFileException + + +class FileHandle: + DEFAULT_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH + + def __init__( + self, path, flags, mode=DEFAULT_MODE, handle_type=FileHandleType.OPAQUE_FD + ): + self._fd = None + self._flags = flags + self._handle = None + self._handle_type = None + self._mode = mode + self._path = path + + self.handle_type = handle_type + + def __del__(self): + self.close() + + def __enter__(self): + self.open() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + @property + def flags(self): + return self._flags + + @property + def handle(self): + return self._handle + + @property + def handle_type(self): + return self._handle_type + + @handle_type.setter + def handle_type(self, _handle_type): + if self._handle is not None: + raise RuntimeError("Cannot modify handle_type while FileHandle is open") + if _handle_type not in FileHandleType: + raise ValueError(f"'{_handle_type}' is not a member of enum FileHandleType") + if _handle_type == FileHandleType.OPAQUE_WIN32: + raise NotImplementedError( + "FileHandle does not currently support Win32 Handles" + ) + self._handle_type = _handle_type + + @property + def mode(self): + return self._mode + + @property + def path(self): + return self._path + + def open(self): + if self._handle is not None: + raise RuntimeError("The FileHandle is already open.") + self._fd = os.open(self._path, self._flags, self._mode) + handle, err = hipFileHandleRegister(self._fd, self._handle_type) + if err[0] != 0: + os.close(self._fd) + raise HipFileException(err[0], err[1]) + self._handle = handle + + def close(self): + if self._handle is not None: + hipFileHandleDeregister(self._handle) + self._handle = None + if self._fd is not None: + os.close(self._fd) + self._fd = None + + def read(self, buffer, size, file_offset, buffer_offset): + if self._handle is None: + raise RuntimeError("The FileHandle is not open.") + bytes_read, extra_err = hipFileRead( + self._handle, buffer.ptr, size, file_offset, buffer_offset + ) + if bytes_read == -1: + # extra_err is errno + raise OSError(extra_err, os.strerror(extra_err)) + if bytes_read < -1: + # hipFile Error + # If -bytes_read == OpError.HIP_DRIVER_ERROR, extra_err is hipError_t. + # Otherwise, extra_err is 0. + raise HipFileException(-bytes_read, extra_err) + return bytes_read + + def write(self, buffer, size, file_offset, buffer_offset): + if self._handle is None: + raise RuntimeError("The FileHandle is not open.") + bytes_written, extra_err = hipFileWrite( + self._handle, buffer.ptr, size, file_offset, buffer_offset + ) + if bytes_written == -1: + # extra_err is errno + raise OSError(extra_err, os.strerror(extra_err)) + if bytes_written < -1: + # hipFile Error + # If -bytes_written == OpError.HIP_DRIVER_ERROR, extra_err is hipError_t. + # Otherwise, extra_err is 0. + raise HipFileException(-bytes_written, extra_err) + return bytes_written diff --git a/python/hipfile/hipMalloc.py b/python/hipfile/hipMalloc.py new file mode 100644 index 00000000..803be6b2 --- /dev/null +++ b/python/hipfile/hipMalloc.py @@ -0,0 +1,41 @@ +# pylint: disable=all +""" +This is a hack to have some semblance of GPU memory management +without introducing a dependency at this early stage of +development. Do not rely upon anything in this module. +""" + +import ctypes +import sys + +# Load the HIP runtime library +if sys.platform.startswith("linux"): + _hip_lib_name = "libamdhip64.so" +elif sys.platform == "win32": + _hip_lib_name = "amdhip64.dll" +else: + raise OSError("Unsupported platform for HIP runtime") + +_hip = ctypes.CDLL(_hip_lib_name) + +# hipError_t hipMalloc(void** ptr, size_t size); +_hip.hipMalloc.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.c_size_t] +_hip.hipMalloc.restype = ctypes.c_int + +# hipError_t hipFree(void* ptr); +_hip.hipFree.argtypes = [ctypes.c_void_p] +_hip.hipFree.restype = ctypes.c_int + + +def hipMalloc(size_bytes: int) -> ctypes.c_void_p: + d_ptr = ctypes.c_void_p() + status = _hip.hipMalloc(ctypes.byref(d_ptr), ctypes.c_size_t(size_bytes)) + if status != 0: + raise RuntimeError(f"hipMalloc failed ({status})") + return d_ptr + + +def hipFree(ptr: ctypes.c_void_p) -> None: + status = _hip.hipFree(ptr) + if status != 0: + raise RuntimeError(f"hipFree failed ({status})") diff --git a/python/hipfile/properties.py b/python/hipfile/properties.py new file mode 100644 index 00000000..a6ae68e2 --- /dev/null +++ b/python/hipfile/properties.py @@ -0,0 +1,20 @@ +# pylint: disable=C0114,C0116 +from hipfile._hipfile import ( # pylint: disable=E0401,E0611 + hipFileDriverGetProperties, + hipFileGetVersion, +) +from hipfile.error import HipFileException + + +def driver_get_properties(): + _props, err = hipFileDriverGetProperties() + if err[0] != 0: + raise HipFileException(err[0], err[1]) + return _props + + +def get_version(): + version_tuple, err = hipFileGetVersion() + if err[0] != 0: + raise HipFileException(err[0], err[1]) + return version_tuple diff --git a/python/main.py b/python/main.py new file mode 100644 index 00000000..a80ca6fb --- /dev/null +++ b/python/main.py @@ -0,0 +1,70 @@ +""" +A quick & rough script for testing the Cython bindings to the +hipFile C library. Reads a given file and copies it to an +output file, and then prints the hashes of the files. +""" + +import hashlib +import os +import pathlib + +from hipfile.hipMalloc import hipFree, hipMalloc + +from hipfile import ( + Driver, + FileHandle, + Buffer, + FileHandleType, + get_version, +) + +hipfile_version = get_version() + +input_path = pathlib.Path("/mnt/ais/ext4/random_2MiB.bin") +output_path = pathlib.Path("/mnt/ais/ext4/output.bin") + +print(f"hipFile Version: {hipfile_version}") +print(f"Driver Use Count Before: {Driver.use_count()}") + +# Max to a 2GiB - 4KiB Buffer +# Note: Max IO in a single transaction is 2GiB - 4KiB as set by the Linux Kernel +# Larger IOs will be quietly truncated. +size = min(input_path.stat().st_size, 2 * 1024 * 1024 * 1024 - 4 * 1024) +buffer = hipMalloc(size) +buffer_ptr = buffer.value # pylint: disable=C0103 # False Positive +print(f"Buffer located at: {buffer_ptr} | {hex(buffer_ptr)}") + +with Driver() as hipfile_driver: + print(f"Driver Use Count After: {hipfile_driver.use_count()}") + with Buffer.from_ctypes_void_p(buffer, size, 0) as registered_buffer: + with FileHandle( + input_path, + os.O_RDWR | os.O_DIRECT | os.O_CREAT, + handle_type=FileHandleType.OPAQUE_FD, + ) as fh_input: + with FileHandle( + output_path, os.O_RDWR | os.O_DIRECT | os.O_CREAT | os.O_TRUNC + ) as fh_output: + print(f"Transferring {size} bytes...") + bytes_read = fh_input.read(registered_buffer, size, 0, 0) + print(f"Bytes Read: {bytes_read}") + bytes_written = fh_output.write(registered_buffer, size, 0, 0) + print(f"Bytes Written: {bytes_written}") + +hipFree(buffer) + +with open(input_path, "br") as file_in: + hash_in = hashlib.sha256() + chunk = file_in.read(1 * 1024 * 1024) # 1 MiB + while len(chunk) != 0: + hash_in.update(chunk) + chunk = file_in.read(1 * 1024 * 1024) # 1 MiB + print(f"Input File Hash: {hash_in.hexdigest()}") + +with open(output_path, "br") as file_out: + hash_out = hashlib.sha256() + chunk = file_out.read(1 * 1024 * 1024) # 1 MiB + while len(chunk) != 0: + hash_out.update(chunk) + chunk = file_out.read(1 * 1024 * 1024) # 1 MiB + print(f"Output File Hash: {hash_out.hexdigest()}") diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 00000000..9a334dd8 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,14 @@ +[build-system] +requires = ["scikit-build-core>=0.10", "cython>=3.0"] +build-backend = "scikit_build_core.build" + +[project] +name = "hipfile" +version = "0.2.0" +description = "Low-level Python bindings for hipFile (direct-to-GPU IO)" +requires-python = ">=3.10" +license = {text = "MIT"} + +[tool.scikit-build] +cmake.build-type = "Release" +wheel.packages = ["hipfile"]