From d76d51dad66ef321050ce14bcddd6aea07d8abcc Mon Sep 17 00:00:00 2001 From: Carlos Guzman Date: Fri, 13 Feb 2026 15:36:01 -0600 Subject: [PATCH 01/16] Using conan as package manager. Conan is used as package manager. CmakeLists were cleaned up. Separated files into /include & /src. --- .gitignore | 2 + CONAN-INTEGRATION.md | 313 ++++++++++ arraymorph/.clangd | 22 + arraymorph/CMakeLists.txt | 75 ++- arraymorph/CMakeUserPresets.json | 9 + arraymorph/conanfile.py | 34 ++ .../arraymorph/core}/constants.h | 33 +- .../{src => include/arraymorph/core}/logger.h | 7 +- .../include/arraymorph/core/operators.h | 153 +++++ arraymorph/include/arraymorph/core/utils.h | 56 ++ .../include/arraymorph/s3vl/chunk_obj.h | 58 ++ .../arraymorph/s3vl/dataset_callbacks.h | 42 ++ .../include/arraymorph/s3vl/dataset_obj.h | 59 ++ .../include/arraymorph/s3vl/file_callbacks.h | 20 + .../include/arraymorph/s3vl/group_callbacks.h | 27 + .../include/arraymorph/s3vl/initialize.h | 72 +++ .../arraymorph/s3vl/vol_connector.h} | 6 +- arraymorph/justfile | 41 ++ arraymorph/src/CMakeLists.txt | 104 ++-- arraymorph/src/{ => core}/constants.cc | 4 +- arraymorph/src/{ => core}/operators.cc | 6 +- arraymorph/src/core/utils.cc | 145 +++++ arraymorph/src/operators.h | 127 ---- arraymorph/src/s3vl/chunk_obj.cc | 51 ++ arraymorph/src/s3vl/dataset_callbacks.cc | 209 +++++++ arraymorph/src/s3vl/dataset_obj.cc | 541 ++++++++++++++++++ arraymorph/src/s3vl/file_callbacks.cc | 140 +++++ arraymorph/src/s3vl/group_callbacks.cc | 115 ++++ arraymorph/src/s3vl/vol_connector.cc | 164 ++++++ arraymorph/src/s3vl_chunk_obj.cc | 48 -- arraymorph/src/s3vl_chunk_obj.h | 60 -- arraymorph/src/s3vl_dataset_callbacks.cc | 206 ------- arraymorph/src/s3vl_dataset_callbacks.h | 26 - arraymorph/src/s3vl_dataset_obj.cc | 493 ---------------- arraymorph/src/s3vl_dataset_obj.h | 54 -- arraymorph/src/s3vl_file_callbacks.cc | 86 --- arraymorph/src/s3vl_file_callbacks.h | 18 - arraymorph/src/s3vl_group_callbacks.cc | 99 ---- arraymorph/src/s3vl_group_callbacks.h | 19 - arraymorph/src/s3vl_initialize.h | 66 --- arraymorph/src/s3vl_vol_connector.cc | 149 ----- arraymorph/src/utils.cc | 124 ---- arraymorph/src/utils.h | 41 -- env-example.txt | 13 + 44 files changed, 2417 insertions(+), 1720 deletions(-) create mode 100644 CONAN-INTEGRATION.md create mode 100644 arraymorph/.clangd create mode 100644 arraymorph/CMakeUserPresets.json create mode 100644 arraymorph/conanfile.py rename arraymorph/{src => include/arraymorph/core}/constants.h (74%) rename arraymorph/{src => include/arraymorph/core}/logger.h (70%) create mode 100644 arraymorph/include/arraymorph/core/operators.h create mode 100644 arraymorph/include/arraymorph/core/utils.h create mode 100644 arraymorph/include/arraymorph/s3vl/chunk_obj.h create mode 100644 arraymorph/include/arraymorph/s3vl/dataset_callbacks.h create mode 100644 arraymorph/include/arraymorph/s3vl/dataset_obj.h create mode 100644 arraymorph/include/arraymorph/s3vl/file_callbacks.h create mode 100644 arraymorph/include/arraymorph/s3vl/group_callbacks.h create mode 100644 arraymorph/include/arraymorph/s3vl/initialize.h rename arraymorph/{src/s3vl_vol_connector.h => include/arraymorph/s3vl/vol_connector.h} (91%) create mode 100644 arraymorph/justfile rename arraymorph/src/{ => core}/constants.cc (51%) rename arraymorph/src/{ => core}/operators.cc (99%) create mode 100644 arraymorph/src/core/utils.cc delete mode 100644 arraymorph/src/operators.h create mode 100644 arraymorph/src/s3vl/chunk_obj.cc create mode 100644 arraymorph/src/s3vl/dataset_callbacks.cc create mode 100644 arraymorph/src/s3vl/dataset_obj.cc create mode 100644 arraymorph/src/s3vl/file_callbacks.cc create mode 100644 arraymorph/src/s3vl/group_callbacks.cc create mode 100644 arraymorph/src/s3vl/vol_connector.cc delete mode 100644 arraymorph/src/s3vl_chunk_obj.cc delete mode 100644 arraymorph/src/s3vl_chunk_obj.h delete mode 100644 arraymorph/src/s3vl_dataset_callbacks.cc delete mode 100644 arraymorph/src/s3vl_dataset_callbacks.h delete mode 100644 arraymorph/src/s3vl_dataset_obj.cc delete mode 100644 arraymorph/src/s3vl_dataset_obj.h delete mode 100644 arraymorph/src/s3vl_file_callbacks.cc delete mode 100644 arraymorph/src/s3vl_file_callbacks.h delete mode 100644 arraymorph/src/s3vl_group_callbacks.cc delete mode 100644 arraymorph/src/s3vl_group_callbacks.h delete mode 100644 arraymorph/src/s3vl_initialize.h delete mode 100644 arraymorph/src/s3vl_vol_connector.cc delete mode 100644 arraymorph/src/utils.cc delete mode 100644 arraymorph/src/utils.h create mode 100644 env-example.txt diff --git a/.gitignore b/.gitignore index 1ca38a0..c648f1b 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,5 @@ Thumbs.db .vscode # build arraymorph/build + +.venv diff --git a/CONAN-INTEGRATION.md b/CONAN-INTEGRATION.md new file mode 100644 index 0000000..9829db3 --- /dev/null +++ b/CONAN-INTEGRATION.md @@ -0,0 +1,313 @@ +# ArrayMorph VOL Plugin — Local Garage Setup Changes + +This document describes all changes made to get the ArrayMorph HDF5 VOL plugin +working with a local [Garage](https://garagehq.deuxfleurs.fr/) S3-compatible +storage instance running in Docker on macOS (ARM64). + +## Directory Tree + +Changes made to restructure the code base. CMakeList files were cleaned up as well. + +``` +. +├── CMakeLists.txt +├── CMakeUserPresets.json // Needed so clangd (LSP) can see packages from conan +├── conanfile.py // Needed for conan to pull packages +├── deactivate.sh +├── include // Moved all header files +│   └── arraymorph +│   ├── core +│   │   ├── constants.h +│   │   ├── logger.h +│   │   ├── operators.h +│   │   └── utils.h +│   └── s3vl +│   ├── chunk_obj.h +│   ├── dataset_callbacks.h +│   ├── dataset_obj.h +│   ├── file_callbacks.h +│   ├── group_callbacks.h +│   ├── initialize.h +│   └── vol_connector.h +├── justfile // Not necessary, but helps with reptetitive cli commands (need to install just). +├── meta.yaml +├── README.md +├── scripts +│   └── extract_perspective.py +└── src // All source files + ├── CMakeLists.txt + ├── core + │   ├── constants.cc + │   ├── operators.cc + │   └── utils.cc + └── s3vl + ├── chunk_obj.cc + ├── dataset_callbacks.cc + ├── dataset_obj.cc + ├── file_callbacks.cc + ├── group_callbacks.cc + └── vol_connector.cc +``` + +## Environment + +- **OS:** macOS (Apple Silicon / ARM64) +- **Garage:** v2.2.0 in Docker, S3 API on `http://localhost:3900`, admin API on `localhost:3903` +- **Python:** h5py with `HDF5_VOL_CONNECTOR=arraymorph` +- **Build:** Conan + CMake + Ninja +- **Variables**: env-example.txt + +## Required Changes + +These changes are necessary for the plugin to work with a local Garage instance. + +### 1. S3 Client Configuration (`src/s3vl/file_callbacks.cpp`) + +The S3 client was hardcoded for AWS. Five changes were needed to target a local +Garage instance: + +#### a. HTTP Scheme (`file_callbacks.cpp) + +```cpp +// Before +s3ClientConfig->scheme = Scheme::HTTPS; + +// After +s3ClientConfig->scheme = Scheme::HTTP; +``` + +**Why:** Garage runs on `http://localhost:3900`. HTTPS causes connection failure. + +#### b. Endpoint Override (`file_callbacks.cpp) + +```cpp +// Added +const char *endpoint = getenv("AWS_ENDPOINT_URL_S3"); +if (endpoint) { + s3ClientConfig->endpointOverride = endpoint; +} +``` + +**Why:** Without this, the AWS SDK sends requests to AWS's default S3 endpoint +instead of the local Garage instance. The endpoint is read from the +`AWS_ENDPOINT_URL_S3` environment variable. + +#### c. Region Override (`file_callbacks.cpp) + +```cpp +// Added +const char *region = getenv("AWS_REGION"); +if (region) { + s3ClientConfig->region = region; +} +``` + +**Why:** AWS SigV4 includes the region in the request signature. The SDK +defaults to `us-east-1`, but Garage is configured with region `garage`. A +mismatch produces `AccessDenied: Forbidden: Invalid signature`. + +#### d. Payload Signing Policy (`file_callbacks.cpp) + +```cpp +// Before +Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never + +// After +Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always +``` + +**Why:** Garage requires signed payloads. `Never` skips payload signing, causing +signature validation failures. + +#### e. Path-Style Addressing (`file_callbacks.cpp) + +```cpp +// Before (last argument = true = virtual-hosted style) +client = std::make_unique(cred, ..., true); + +// After (last argument = false = path-style) +client = std::make_unique(cred, ..., false); +``` + +**Why:** Virtual-hosted style puts the bucket name in the hostname +(`bucket.endpoint`). Garage (and most S3-compatible stores) require path-style +(`endpoint/bucket`). With virtual-hosted style, the SDK was interpreting the HDF5 +filename as the bucket name. + +### 2. AWS SDK Lifecycle (`include/arraymorph/s3vl/initialize.hpp`) + +#### a. Global SDK Options (`initialize.hpp:10`) + +```cpp +// Added at file scope +static Aws::SDKOptions g_sdk_options; +``` + +The `SDKOptions` was previously a local variable in `s3VL_initialize_init`. It +must persist so `Aws::ShutdownAPI` can be called during close. + +#### b. Added `operators.hpp` Include (`initialize.hpp:4`) + +```cpp +#include "arraymorph/core/operators.hpp" +``` + +Needed for the `global_cloud_client` symbol used in the close function. + +#### c. Proper SDK Shutdown (`initialize.hpp:59-65`) + +```cpp +// Before +inline herr_t S3VLINITIALIZE::s3VL_initialize_close() { + Logger::log("------ Close VOL"); + // Aws::ShutdownAPI(opt); + return ARRAYMORPH_SUCCESS; +} + +// After +inline herr_t S3VLINITIALIZE::s3VL_initialize_close() { + Logger::log("------ Close VOL"); + global_cloud_client = std::monostate{}; + Aws::ShutdownAPI(g_sdk_options); + return ARRAYMORPH_SUCCESS; +} +``` + +**Why:** The S3Client must be destroyed *before* `ShutdownAPI` is called. +Otherwise the SDK's internal state is torn down while the client still holds +references to it. Resetting `global_cloud_client` to `monostate` triggers the +`unique_ptr` destructor. + +#### d. Terminate Handler (macOS Workaround) (`initialize.hpp:29`) + +```cpp +// Added before Aws::InitAPI +std::set_terminate([]() { _exit(0); }); +``` + +**Why:** The AWS CRT (Common Runtime) library registers `atexit` handlers that +conflict with Python's interpreter shutdown on macOS. After all S3 operations +complete successfully, the process crashes with: + +``` +std::__1::system_error: mutex lock failed: Invalid argument +``` + +This is a `pthread_mutex_lock` returning `EINVAL` during static destruction — +the mutex has already been destroyed by the time the CRT's cleanup code runs. +The terminate handler catches this and exits cleanly. This is a known class of +issue with the AWS C++ SDK when loaded as a shared library inside another +runtime (Python/h5py). + +## Tuning Changes + +These values were reduced from production/cloud defaults to work within macOS +thread and resource limits. They are not strictly required for correctness but +prevent `thread constructor failed: Resource temporarily unavailable` crashes. + +### `include/arraymorph/core/constants.hpp` + +| Variable | Line | Original | Current | Recommended | Notes | +|---|---|---|---|---|---| +| `POOLEXECUTOR` | `constants.hpp:11` | `#define` | `// #define` | Re-enable | Gives the SDK a dedicated thread pool. Only crashed at `poolSize=8192`. Safe at 32. | +| `s3Connections` | `constants.hpp:13` | `256` | `8` | `32` | Max concurrent HTTP connections. Not threads. 256 is fine for AWS but unnecessary locally. | +| `poolSize` | `constants.hpp:16` | `8192` | `16` | `32` | Size of `PooledThreadExecutor` thread pool. 8192 exceeded macOS thread limits. | +| `THREAD_NUM` | `constants.hpp:19` | `256` | `8` | `16` | Batch size for `std::async` chunk upload/download. 256 can hit macOS per-process thread limits on large datasets. | + +### Recommended Constants for Local Development + +```cpp +#define POOLEXECUTOR + +const int s3Connections = 32; +const int poolSize = 32; +const int THREAD_NUM = 16; +``` + +### Recommended Constants for Production (Linux, Cloud VMs) + +```cpp +#define POOLEXECUTOR + +const int s3Connections = 256; +const int poolSize = 256; +const int THREAD_NUM = 64; +``` + +## `.env` Configuration + +The following environment variables must be set (loaded via `justfile` with +`set dotenv-load`): + +```env +STORAGE_PLATFORM=S3 +BUCKET_NAME=playgrounds +AWS_ENDPOINT_URL_S3=garage-s3-endpoint +AWS_S3_ADDRESSING_STYLE="path" +AWS_ACCESS_KEY_ID=garage-access-key +AWS_SECRET_ACCESS_KEY=secret-access-key +AWS_REGION=garage +AWS_USE_PATH_STYLE=true +AWS_USE_TLS=true +AWS_SIGNED_PAYLOADS=true +HDF5_VOL_CONNECTOR=arraymorph +HDF5_PLUGIN_PATH=path-to-libarraymorph +``` + +## Change Necessity Assessment + +Not every change made was strictly necessary. + +### Absolutely Necessary (plugin fails without these) + +These are the S3 client config changes in `file_callbacks.cpp`: + +- **Endpoint override** from `AWS_ENDPOINT_URL_S3` (`file_callbacks.cpp) — without it, requests go to AWS +- **Region override** from `AWS_REGION` (`file_callbacks.cpp) — without it, SigV4 signature is wrong +- **Path-style addressing** (`true` → `false`) (`file_callbacks.cpp) — without it, bucket name is misrouted +- **`PayloadSigningPolicy::Always`** (`file_callbacks.cpp) — without it, Garage rejects unsigned payloads +- **`Scheme::HTTP`** (`file_callbacks.cpp) — without it, can't connect to` +- **Garage dead node removal** — without it, writes fail with quorum errors + +### Necessary Workaround (operations work, but process crashes on exit) + +- **`std::set_terminate([]() { _exit(0); })`** (`initialize.hpp:29`) — masks the AWS CRT mutex crash + rather than fixing the root cause. It swallows *all* uncaught exceptions, not + just the CRT one. A more targeted fix would require understanding the CRT's + internal teardown order. This should be revisited if the plugin is used outside + of Python/macOS. + +### Good Practice but Not Strictly Required + +These changes improve correctness/hygiene but the plugin works without them: + +- **`global_cloud_client = std::monostate{}`** in close (`initialize.hpp:62`) — clean teardown order, + but the terminate handler already catches the crash +- **`Aws::ShutdownAPI(g_sdk_options)`** (`initialize.hpp:63`) — proper SDK cleanup, but didn't fix the + crash by itself +- **`static Aws::SDKOptions g_sdk_options`** (`initialize.hpp:10`) — only needed because of ShutdownAPI +- **`#include "arraymorph/core/operators.hpp"`** (`initialize.hpp:4`) — only + needed for the monostate reset + +### Not Necessary, Can Be Reverted + +These were made during investigation and are not required for correctness: + +- **`// #define POOLEXECUTOR`** (`constants.hpp:11`) — disabling this was investigative. The crash was + from `poolSize=8192`, not from the executor itself. Safe to re-enable now that + `poolSize` is 16. +- **`THREAD_NUM` 256 → 8** (`constants.hpp:19`) — investigative. The test dataset has 1 chunk so + this value didn't matter. The original thread crash was from `poolSize`, not + `THREAD_NUM`. Can be set back to a moderate value (16-32). +- **`s3Connections` 256 → 8** (`constants.hpp:13`) — this is HTTP connections, not threads. 256 was + fine. Changed during investigation of the thread crash. + +## Sequence of Errors Encountered + +| # | Error | Root Cause | Fix | Location | +|---|-------|------------|-----|----------| +| 1 | `thread constructor failed: Resource temporarily unavailable` | `poolSize=8192` exceeding macOS thread limit | Reduced `poolSize` | `constants.hpp:16` | +| 2 | `NoSuchBucket: Bucket not found: test.h5` | No endpoint override + virtual-hosted addressing | Added `endpointOverride`, set path-style | `file_callbacks.cpp:28-31,40` | +| 3 | `AccessDenied: Forbidden: Invalid signature` | Wrong region + unsigned payloads + HTTPS scheme | Set region from env, `PayloadSigningPolicy::Always`, `Scheme::HTTP` | `file_callbacks.cpp:17,32-35,39` | +| 4 | `ServiceUnavailable: Could not reach quorum` | Dead node in Garage cluster layout | Removed dead node via admin API | Garage admin API | +| 5 | `mutex lock failed: Invalid argument` | AWS CRT static destruction conflict with Python on macOS | `std::set_terminate` workaround + proper SDK shutdown | `initialize.hpp:29,59-65` | diff --git a/arraymorph/.clangd b/arraymorph/.clangd new file mode 100644 index 0000000..537aa72 --- /dev/null +++ b/arraymorph/.clangd @@ -0,0 +1,22 @@ +# .clangd file +CompileFlags: + CompilationDatabase: "build/Release" + +# Diagnostics are configured in .clang-tidy +Diagnostics: + UnusedIncludes: Strict + MissingIncludes: Strict + +# Editor features +InlayHints: + Enabled: true + ParameterNames: true + DeducedTypes: true + BlockEnd: true + +Hover: + ShowAKA: true + +# Index settings for better code navigation +Index: + Background: Build diff --git a/arraymorph/CMakeLists.txt b/arraymorph/CMakeLists.txt index ea7f6a0..5759c01 100644 --- a/arraymorph/CMakeLists.txt +++ b/arraymorph/CMakeLists.txt @@ -1,19 +1,68 @@ -cmake_minimum_required (VERSION 3.5) +cmake_minimum_required(VERSION 3.20) + +project(ArrayMorph + VERSION 0.1.0 + LANGUAGES C CXX +) + set(CMAKE_CXX_STANDARD 20) -set(CMAKE_BUILD_TYPE Release) -set(CMAKE_CXX_FLAGS "-O3") +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) # b_staticpic=true +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +# Build type and optimization +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +if(CMAKE_BUILD_TYPE STREQUAL "Release") + set(CMAKE_CXX_FLAGS_RELEASE "-O3") +endif() + +# Include directory +include_directories(${CMAKE_SOURCE_DIR}/include) + +# --- Find External Dependencies --- + +# AWS SDK +find_package(AWSSDK REQUIRED COMPONENTS core s3) + +# Azure SDK +find_package(AzureSDK CONFIG REQUIRED) + +# cURL and OpenSSL +find_package(CURL REQUIRED) +find_package(OpenSSL REQUIRED) + +if(NOT DEFINED ENV{H5PY_HDF5_DIR}) + message(FATAL_ERROR "H5PY_HDF5_DIR environment variable not set") +endif() + +# --- HDF5 with h5py workaround --- +set(H5PY_LIB_DIR "$ENV{H5PY_HDF5_DIR}") -project (s3_vol DESCRIPTION "S3 VOL connector") +# Find HDF5 for headers (via Conan or system) +find_package(HDF5 REQUIRED COMPONENTS C) -# Find HDF5 -find_package (HDF5 1.14.2 REQUIRED COMPONENTS C) -link_directories (${HDF5_LIBRARY_DIRS}) -include_directories (${HDF5_INCLUDE_DIR}) -set (LINK_LIBS ${LINK_LIBS} ${HDF5_C_${LIB_TYPE}_LIBRARY}) +# Create custom HDF5 dependency: Conan headers + h5py binary +add_library(hdf5_custom INTERFACE) +target_include_directories(hdf5_custom INTERFACE ${HDF5_INCLUDE_DIRS}) +target_link_directories(hdf5_custom INTERFACE ${H5PY_LIB_DIR}) +target_link_libraries(hdf5_custom INTERFACE hdf5.310) -# It's really easy to pick up the wrong HDF5 library if you set the path -# wrong. Turn this on for added confirmation that you got it right. -message (DEPRECATION "Include: ${HDF5_INCLUDE_DIR}") +# Print confirmation +message(STATUS "HDF5 found: ${HDF5_VERSION}") -add_subdirectory (src) +# Collect all dependencies into a list +set(ALL_DEPS + AWS::aws-sdk-cpp-core + AWS::aws-sdk-cpp-s3 + Azure::azure-storage-blobs + hdf5_custom + OpenSSL::SSL + OpenSSL::Crypto + CURL::libcurl +) +# Enter src directory +add_subdirectory(src) diff --git a/arraymorph/CMakeUserPresets.json b/arraymorph/CMakeUserPresets.json new file mode 100644 index 0000000..71aeace --- /dev/null +++ b/arraymorph/CMakeUserPresets.json @@ -0,0 +1,9 @@ +{ + "version": 4, + "vendor": { + "conan": {} + }, + "include": [ + "build/Release/generators/CMakePresets.json" + ] +} \ No newline at end of file diff --git a/arraymorph/conanfile.py b/arraymorph/conanfile.py new file mode 100644 index 0000000..0f6c4d4 --- /dev/null +++ b/arraymorph/conanfile.py @@ -0,0 +1,34 @@ +from conan import ConanFile +from conan.tools.cmake import cmake_layout, CMakeToolchain, CMakeDeps +from conan.tools.gnu import PkgConfigDeps +from conan.tools.layout import basic_layout + + +class ArrayMorphRecipe(ConanFile): + name = "ArrayMorph" + version = "0.2.0" + settings = "os", "compiler", "build_type", "arch" + + def requirements(self): + self.requires("aws-sdk-cpp/1.11.692") + self.requires("azure-sdk-for-cpp/1.16.1") + # self.requires("azure-storage-cpp/7.5.0") + self.requires("hdf5/1.14.6") + self.requires("libcurl/8.17.0") + self.requires("openssl/3.6.1") + + def configure(self): + self.options["*"].shared = False + self.options["aws-sdk-cpp"].s3 = True + self.options["aws-sdk-cpp"].text_to_speech = False + + def layout(self): + cmake_layout(self) + + def generate(self): + tc = CMakeToolchain(self, generator="Ninja") + tc.generate() + self.options["azure-sdk-for-cpp"].with_storage_blobs = True + self.options["azure-sdk-for-cpp"].with_storage_datalake = False + pc = CMakeDeps(self) + pc.generate() diff --git a/arraymorph/src/constants.h b/arraymorph/include/arraymorph/core/constants.h similarity index 74% rename from arraymorph/src/constants.h rename to arraymorph/include/arraymorph/core/constants.h index fa8ffd8..0dcbe93 100644 --- a/arraymorph/src/constants.h +++ b/arraymorph/include/arraymorph/core/constants.h @@ -1,8 +1,8 @@ #ifndef __CONSTANTS__ #define __CONSTANTS__ +#include #include -#include // #define LOG_ENABLE // #define PROFILE_ENABLE @@ -18,41 +18,18 @@ const int connectTimeoutMs = 30000; const int poolSize = 8192; const int retries = 3; -const int THREAD_NUM=256; +const int THREAD_NUM = 256; extern std::string BUCKET_NAME; typedef struct Result { - std::vector data; + std::vector data; } Result; -enum QPlan { - NONE=-1, - GET=0 -}; +enum QPlan { NONE = -1, GET = 0 }; -enum SPlan -{ - S3=0, - GOOGLE, - AZURE_BLOB -}; +enum SPlan { S3 = 0, GOOGLE, AZURE_BLOB }; extern SPlan SP; #endif - - - - - - - - - - - - - - - diff --git a/arraymorph/src/logger.h b/arraymorph/include/arraymorph/core/logger.h similarity index 70% rename from arraymorph/src/logger.h rename to arraymorph/include/arraymorph/core/logger.h index 5952939..b517c8c 100644 --- a/arraymorph/src/logger.h +++ b/arraymorph/include/arraymorph/core/logger.h @@ -1,17 +1,16 @@ +#include "arraymorph/core/constants.h" #include #include -#include "constants.h" #ifndef __LOGGER__ #define __LOGGER__ class Logger { public: - template - static void log(Args&&... args) { + template static void log(Args &&...args) { #ifdef LOG_ENABLE std::ostringstream oss; ((oss << std::forward(args) << ' '), ...); std::cout << oss.str() << std::endl; #endif - } + } }; #endif diff --git a/arraymorph/include/arraymorph/core/operators.h b/arraymorph/include/arraymorph/core/operators.h new file mode 100644 index 0000000..466c458 --- /dev/null +++ b/arraymorph/include/arraymorph/core/operators.h @@ -0,0 +1,153 @@ +#ifndef OPERATORS +#define OPERATORS +#include "arraymorph/core/constants.h" +#include "arraymorph/core/logger.h" +#include +#include +#include +#include // for Scheme +#include +#include +#include // for CSVInput +#include // for CSVOutput +#include +#include +#include // for Expressio... +#include // for FileHeade... +#include +#include +#include // for InputSeri... +#include // for OutputSer... +#include +#include // for RecordsEvent +#include // for SelectObj... +#include +#include // for StatsEvent +#include +#include // for Azure blob +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +using namespace Aws; +using namespace Aws::Auth; +using namespace Aws::Http; +using namespace Aws::Client; +using namespace Aws::S3; +using namespace Aws::S3::Model; +using namespace Azure::Storage::Blobs; + +using CloudClient = + std::variant, + std::unique_ptr>; + +extern CloudClient global_cloud_client; + +class OperationTracker { +public: + static OperationTracker &getInstance(); + + void add(); + int get() const; + void reset(); + +private: + std::atomic finish{0}; + + OperationTracker() = default; + OperationTracker(const OperationTracker &) = delete; + OperationTracker &operator=(const OperationTracker &) = delete; +}; + +class AsyncWriteInput : public AsyncCallerContext { +public: + AsyncWriteInput(const char *buf) : buf(buf) {} + const char *buf; +}; + +class AsyncReadInput : public AsyncCallerContext { +public: + AsyncReadInput(const void *buf, + const std::vector> &mapping, + const int lambda = 0, const std::string bucket_name = "", + const std::string uri = "") + : buf(buf), mapping(mapping), lambda(lambda), bucket_name(bucket_name), + uri(uri) {} + const void *buf; + const std::vector> mapping; + const int lambda; + // for re-issuing GET if lambda fails + const std::string bucket_name; + const std::string uri; +}; + +class Operators { +public: + // S3 + static Result S3Get(const S3Client *client, const std::string &bucket_name, + const Aws::String &object_name); + static herr_t + S3GetAsync(const S3Client *client, const std::string &bucket_name, + const Aws::String &object_name, + const std::shared_ptr input); + static herr_t + S3GetByteRangeAsync(const S3Client *client, const std::string &bucket_name, + const Aws::String &object_name, uint64_t beg, + uint64_t end, + const std::shared_ptr input); + static herr_t S3Put(const S3Client *client, const std::string &bucket_name, + const std::string &object_name, Result &re); + static herr_t S3PutBuf(const S3Client *client, const std::string &bucket_name, + const std::string &object_name, + std::shared_ptr buf, hsize_t length); + static herr_t S3PutAsync(const S3Client *client, + const std::string &bucket_name, + const Aws::String &object_name, Result &re); + static herr_t S3Delete(const S3Client *client, const std::string &bucket_name, + const Aws::String &object_name); + static void GetAsyncCallback( + const Aws::S3::S3Client *s3Client, + const Aws::S3::Model::GetObjectRequest &request, + Aws::S3::Model::GetObjectOutcome outcome, + const std::shared_ptr &context); + + // Azure + static Result AzureGet(const BlobContainerClient *client, + const std::string &blob_name); + static herr_t AzurePut(const BlobContainerClient *client, + const std::string &blob_name, + std::shared_ptr buf, size_t length); + static herr_t + AzureGetAndProcess(const BlobContainerClient *client, + const std::string &blob_name, + const std::shared_ptr context); + static herr_t + AzureGetRange(const BlobContainerClient *client, const std::string &blob_name, + uint64_t beg, uint64_t end, + const std::shared_ptr context); +}; + +inline herr_t Operators::S3GetByteRangeAsync( + const S3Client *client, const std::string &bucket_name, + const Aws::String &object_name, uint64_t beg, uint64_t end, + const std::shared_ptr input) { + Logger::log("------ S3getRangeAsync ", object_name); + GetObjectRequest request; + request.SetBucket(bucket_name); + request.SetKey(object_name); + std::stringstream ss; + ss << "bytes=" << beg << '-' << end; + // std::cout << ss.str() << std::endl; + request.SetRange(ss.str().c_str()); + client->GetObjectAsync(request, GetAsyncCallback, input); + return ARRAYMORPH_SUCCESS; +} + +#endif diff --git a/arraymorph/include/arraymorph/core/utils.h b/arraymorph/include/arraymorph/core/utils.h new file mode 100644 index 0000000..3a14c20 --- /dev/null +++ b/arraymorph/include/arraymorph/core/utils.h @@ -0,0 +1,56 @@ +#ifndef UTILS +#define UTILS +#include "arraymorph/core/constants.h" +#include +#include +#include +#include +#include +#include + +class Segment { +public: + Segment(hsize_t start_offset, hsize_t end_offset, hsize_t required_data_size, + const std::list>::iterator &mapping_start, + const std::list>::iterator &mapping_end, + hsize_t mapping_size); + Segment(hsize_t start_offset, hsize_t end_offset, + const std::list>::iterator &mapping_start, + const std::list>::iterator &mapping_end, + hsize_t mapping_size); + Segment(const std::list>::iterator &mapping_start, + const std::list>::iterator &mapping_end, + hsize_t mapping_size); + std::string to_string(); + hsize_t start_offset; // inclusive + hsize_t end_offset; // inclusive + hsize_t required_data_size; + hsize_t mapping_size; + std::list>::iterator mapping_start; // inclusive + std::list>::iterator mapping_end; // exclusive +}; + +std::vector reduceAdd(std::vector &a, + std::vector &b); + +// calculate each row's start position in serial order +std::vector calSerialOffsets(std::vector> ranges, + std::vector shape); + +// mapping offsets between two hyperslab +std::list> +mapHyperslab(std::vector &local, std::vector &global, + std::vector &out, hsize_t &input_rsize, + hsize_t &out_rsize, hsize_t &data_size); + +// create Json query for Lambda function +std::string createQuery(hsize_t data_size, int ndims, + const std::vector &shape, + std::vector> &ranges); + +// generate list of segments for chunk-level query + +std::vector> +generateSegments(std::list> &mapping, hsize_t chunk_size); + +#endif diff --git a/arraymorph/include/arraymorph/s3vl/chunk_obj.h b/arraymorph/include/arraymorph/s3vl/chunk_obj.h new file mode 100644 index 0000000..57378eb --- /dev/null +++ b/arraymorph/include/arraymorph/s3vl/chunk_obj.h @@ -0,0 +1,58 @@ +#ifndef S3VL_CHUNK_OBJ +#define S3VL_CHUNK_OBJ +#include "arraymorph/core/constants.h" +#include "arraymorph/core/logger.h" +#include "arraymorph/core/operators.h" +#include "arraymorph/core/utils.h" +#include +#include +#include +#include +#include +#include +#include + +typedef struct CPlan { + int chunk_id; + QPlan qp; + size_t num_requests; + std::vector> segments; + std::string lambda_query = ""; + + CPlan(int id, QPlan q, size_t reqs, std::vector> &&s) + : chunk_id(id), qp(q), num_requests(reqs), segments(std::move(s)) {} + + CPlan(const CPlan &) = delete; + CPlan &operator=(const CPlan &) = delete; + + CPlan(CPlan &&) = default; + CPlan &operator=(CPlan &&) = default; + +} CPlan; + +class S3VLChunkObj { +public: + S3VLChunkObj(const std::string &uri, hid_t dtype, + std::vector> &ranges, + const std::vector &shape, + std::vector &return_offsets); + ~S3VLChunkObj() {}; + + std::string to_string(); + bool checkFullWrite(); + + const std::string uri; + hid_t dtype; + std::vector> ranges; + const std::vector shape; + std::vector reduc_per_dim; + int ndims; + + int data_size; + hsize_t size; + hsize_t required_size; + std::vector global_offsets; + std::vector local_offsets; +}; + +#endif diff --git a/arraymorph/include/arraymorph/s3vl/dataset_callbacks.h b/arraymorph/include/arraymorph/s3vl/dataset_callbacks.h new file mode 100644 index 0000000..0ed71f9 --- /dev/null +++ b/arraymorph/include/arraymorph/s3vl/dataset_callbacks.h @@ -0,0 +1,42 @@ +#ifndef S3VL_DATASET_CALLBACKS + +#include "arraymorph/core/operators.h" +#include "arraymorph/s3vl/dataset_obj.h" +#include "arraymorph/s3vl/file_callbacks.h" +#include +#include + +class S3VLDatasetCallbacks { +public: + static void *S3VL_dataset_create(void *obj, + const H5VL_loc_params_t *loc_params, + const char *name, hid_t lcpl_id, + hid_t type_id, hid_t space_id, hid_t dcpl_id, + hid_t dapl_id, hid_t dxpl_id, void **req); + static void *S3VL_dataset_open(void *obj, const H5VL_loc_params_t *loc_params, + const char *name, hid_t dapl_id, hid_t dxpl_id, + void **req); + static void *S3VL_obj_open(void *obj, const H5VL_loc_params_t *loc_params, + H5I_type_t *opened_type, hid_t dxpl_id, + void **req); + static herr_t S3VL_obj_get(void *obj, const H5VL_loc_params_t *loc_params, + H5VL_object_get_args_t *args, hid_t dxpl_id, + void **req); + static void *S3VL_wrap_object(void *obj, H5I_type_t obj_type, void *wrap_ctx); + static void *S3VL_get_object(const void *obj); + static herr_t S3VL_dataset_read(size_t count, void **dset, hid_t *mem_type_id, + hid_t *mem_space_id, hid_t *file_space_id, + hid_t plist_id, void **buf, void **req); + static herr_t S3VL_dataset_write(size_t count, void **dset, + hid_t *mem_type_id, hid_t *mem_space_id, + hid_t *file_space_id, hid_t plist_id, + const void **buf, void **req); + static herr_t S3VL_dataset_get(void *dset, H5VL_dataset_get_args_t *args, + hid_t dxpl_id, void **req); + static herr_t S3VL_dataset_close(void *dset, hid_t dxpl_id, void **req); + static herr_t S3VL_dataset_specific(void *obj, + H5VL_dataset_specific_args_t *args, + hid_t dxpl_id, void **req); +}; +#define S3VL_DATASET_CALLBACKS +#endif diff --git a/arraymorph/include/arraymorph/s3vl/dataset_obj.h b/arraymorph/include/arraymorph/s3vl/dataset_obj.h new file mode 100644 index 0000000..8ed0d25 --- /dev/null +++ b/arraymorph/include/arraymorph/s3vl/dataset_obj.h @@ -0,0 +1,59 @@ +#ifndef S3VL_DATASET_OBJ +#define S3VL_DATASET_OBJ +#include "arraymorph/core/constants.h" +#include "arraymorph/core/operators.h" +#include "arraymorph/s3vl/chunk_obj.h" +#include +#include +#include +#include +#include +#include +#include + +class S3VLDatasetObj { +public: + S3VLDatasetObj(const std::string &name, const std::string &uri, hid_t dtype, + int ndims, std::vector &shape, + std::vector &chunk_shape, int chunk_num, + const std::string &bucket_name, const CloudClient &client); + ~S3VLDatasetObj() {}; + + static S3VLDatasetObj *getDatasetObj(const CloudClient &client, + const std::string &bucket_name, + const std::string &uri); + static S3VLDatasetObj *getDatasetObj(const CloudClient &client, + const std::string &bucket_name, + std::vector &buffer); + char *toBuffer(int *length); + std::vector> + generateChunks(std::vector> ranges); + std::string to_string(); + std::vector getChunkOffsets(int chunk_idx); + std::vector> getChunkRanges(int chunk_idx); + std::vector> selectionFromSpace(hid_t space_id); + // QPlan getQueryPlan(FileFormat format, vector> ranges); + + void upload(); + herr_t write(hid_t mem_space_id, hid_t file_space_id, const void *buf); + herr_t read(hid_t mem_space_id, hid_t file_space_id, void *buf); + + const std::string name; + const std::string uri; + hid_t dtype; + const int ndims; + const std::vector shape; + const std::vector chunk_shape; + const int chunk_num; + const std::string bucket_name; + + hsize_t data_size; + std::vector num_per_dim; + std::vector reduc_per_dim; + hsize_t element_per_chunk; + bool is_modified{false}; + + const CloudClient &client; +}; + +#endif diff --git a/arraymorph/include/arraymorph/s3vl/file_callbacks.h b/arraymorph/include/arraymorph/s3vl/file_callbacks.h new file mode 100644 index 0000000..70bcbb6 --- /dev/null +++ b/arraymorph/include/arraymorph/s3vl/file_callbacks.h @@ -0,0 +1,20 @@ +#ifndef S3VL_FILE_CALLBACKS +#include +#include + +typedef struct S3VLFileObj { + std::string name; +} S3VLFileObj; + +class S3VLFileCallbacks { +public: + static void *S3VL_file_create(const char *name, unsigned flags, hid_t fcpl_id, + hid_t fapl_id, hid_t dxpl_id, void **req); + static void *S3VL_file_open(const char *name, unsigned flags, hid_t fapl_id, + hid_t dxpl_id, void **req); + static herr_t S3VL_file_get(void *file, H5VL_file_get_args_t *args, + hid_t dxpl_id, void **req); + static herr_t S3VL_file_close(void *file, hid_t dxpl_id, void **req); +}; +#define S3VL_FILE_CALLBACKS +#endif diff --git a/arraymorph/include/arraymorph/s3vl/group_callbacks.h b/arraymorph/include/arraymorph/s3vl/group_callbacks.h new file mode 100644 index 0000000..4e04554 --- /dev/null +++ b/arraymorph/include/arraymorph/s3vl/group_callbacks.h @@ -0,0 +1,27 @@ +#ifndef S3VL_GROUP_CALLBACKS +#include +#include +#include + +typedef struct S3VLGroupObj { + std::vector keys; +} S3VLGroupObj; + +class S3VLGroupCallbacks { +public: + static void *S3VLgroup_open(void *obj, const H5VL_loc_params_t *loc_params, + const char *name, hid_t gapl_id, hid_t dxpl_id, + void **req); + static herr_t S3VLgroup_close(void *grp, hid_t dxpl_id, void **req); + static herr_t S3VLgroup_get(void *obj, H5VL_group_get_args_t *args, + hid_t dxpl_id, void **req); + static herr_t S3VLlink_get(void *obj, const H5VL_loc_params_t *loc_params, + H5VL_link_get_args_t *args, hid_t dxpl_id, + void **req); + static herr_t S3VLlink_specific(void *obj, + const H5VL_loc_params_t *loc_params, + H5VL_link_specific_args_t *args, + hid_t dxpl_id, void **req); +}; +#define S3VL_GROUP_CALLBACKS +#endif diff --git a/arraymorph/include/arraymorph/s3vl/initialize.h b/arraymorph/include/arraymorph/s3vl/initialize.h new file mode 100644 index 0000000..cc11cb7 --- /dev/null +++ b/arraymorph/include/arraymorph/s3vl/initialize.h @@ -0,0 +1,72 @@ +#ifndef S3VL_INITIALIZE +#define S3VL_INITIALIZE + +#include "arraymorph/core/constants.h" +#include "arraymorph/core/logger.h" +#include "arraymorph/core/operators.h" +#include "arraymorph/s3vl/vol_connector.h" +#include +#include +#include +#include +#include +#include + +static Aws::SDKOptions g_sdk_options; + +class S3VLINITIALIZE { +public: + static herr_t s3VL_initialize_init(hid_t vipl_id); + static herr_t s3VL_initialize_close(); +}; +std::optional getEnv(const char *var) { + const char *val = std::getenv(var); + return val ? std::optional(val) : std::nullopt; +} + +inline herr_t S3VLINITIALIZE::s3VL_initialize_init(hid_t vipl_id) { + // Aws::SDKOptions options; // Changed to use global sdk options for proper + // shutdown + g_sdk_options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Off; + std::set_terminate([]() { + _exit(0); + }); // Shutdown conflicts with Python's interpretor shutdown on MacOS. + // Terminate handler catches this and exits cleanly. + Aws::InitAPI(g_sdk_options); + Logger::log("------ Init VOL"); + std::optional platform = getEnv("STORAGE_PLATFORM"); + if (platform.has_value()) { + std::string platform_str = platform.value(); + if (platform_str == "S3") + Logger::log("------ Using S3"); + else if (platform_str == "Azure") { + Logger::log("------ Using Azure"); + SP = SPlan::AZURE_BLOB; + } else { + Logger::log("------ Unsupported platform"); + return ARRAYMORPH_FAIL; + } + } else { + Logger::log("------ Using default platform S3"); + } + std::optional bucket_name = getEnv("BUCKET_NAME"); + if (bucket_name.has_value()) { + BUCKET_NAME = bucket_name.value(); + Logger::log("------ Using bucket", BUCKET_NAME); + } else { + Logger::log("------ Bucekt not set"); + return ARRAYMORPH_FAIL; + } + return S3_VOL_CONNECTOR_VALUE; +} + +inline herr_t S3VLINITIALIZE::s3VL_initialize_close() { + Logger::log("------ Close VOL"); + // Proper SDK shutdown. + global_cloud_client = + std::monostate{}; // Reset the client's state to trigger destructor + Aws::ShutdownAPI(g_sdk_options); + return ARRAYMORPH_SUCCESS; +} + +#endif diff --git a/arraymorph/src/s3vl_vol_connector.h b/arraymorph/include/arraymorph/s3vl/vol_connector.h similarity index 91% rename from arraymorph/src/s3vl_vol_connector.h rename to arraymorph/include/arraymorph/s3vl/vol_connector.h index a3f5335..112f109 100644 --- a/arraymorph/src/s3vl_vol_connector.h +++ b/arraymorph/include/arraymorph/s3vl/vol_connector.h @@ -18,10 +18,8 @@ #ifndef s3_vol_connector_H #define s3_vol_connector_H - /* The value must be between 256 and 65535 (inclusive) */ -#define S3_VOL_CONNECTOR_VALUE ((H5VL_class_value_t)12202) -#define S3_VOL_CONNECTOR_NAME "arraymorph" +#define S3_VOL_CONNECTOR_VALUE ((H5VL_class_value_t)12202) +#define S3_VOL_CONNECTOR_NAME "arraymorph" #endif /* _template_vol_connector_H */ - diff --git a/arraymorph/justfile b/arraymorph/justfile new file mode 100644 index 0000000..f65169c --- /dev/null +++ b/arraymorph/justfile @@ -0,0 +1,41 @@ +# https://just.systems +# Variables + +BUILD_DIR := "build" +COMPILER_STD := "gnu20" + +# Set this to the path where H5Py installs HDF5 binary. On macOS its .dylib, Linux .so, Windows .dll + +H5PY_HDF5_DIR := "../.venv/lib/python3.14/site-packages/h5py/.dylibs" + +# Settings + +set dotenv-load := true +set export := true + +# Default recipe: list available commands +default: + @just --list + +# 1. Install dependencies via Conan + +# We put all .pc and .ini files in a hidden .conan_deps folder +deps: + conan install . --build=missing -s compiler.cppstd={{ COMPILER_STD }} + +# 2. Setup the CMake build environment +setup: + export H5PY_HDF5_DIR={{ H5PY_HDF5_DIR }}; \ + cmake --preset conan-release + +build: + cmake --build --preset conan-release + +# 5. Full build from scratch +full-build: deps setup build + +# 6. Clean all build and dependency artifacts +clean: + rm -rf {{ BUILD_DIR }} + +full-clean-build: clean full-build diff --git a/arraymorph/src/CMakeLists.txt b/arraymorph/src/CMakeLists.txt index 5fe900c..af80f5a 100644 --- a/arraymorph/src/CMakeLists.txt +++ b/arraymorph/src/CMakeLists.txt @@ -1,50 +1,54 @@ -# Set the template VOL connector name -set (TVC_NAME arraymorph) -set(CMAKE_POSITION_INDEPENDENT_CODE ON) -set(CMAKE_CXX_STANDARD 20) -#Load required services/packages: This basic example uses S3. -find_package(AWSSDK REQUIRED COMPONENTS s3) -find_package(CURL REQUIRED) -find_package(azure-storage-blobs-cpp CONFIG REQUIRED) -# find_package(OpenMP REQUIRED) -find_package(OpenSSL REQUIRED) - -# Build the template VOL -add_library (${TVC_NAME} SHARED s3vl_vol_connector.cc) -add_library (CONSTANTS constants.cc) -add_library (OPERATORS operators.cc) -add_library (VL_CHUNK_OBJ s3vl_chunk_obj.cc) -add_library (VL_FILE s3vl_file_callbacks.cc) -add_library (VL_DSET s3vl_dataset_callbacks.cc) -add_library (VL_DSET_OBJ s3vl_dataset_obj.cc) -add_library (VL_GROUP s3vl_group_callbacks.cc) -add_library (UTILS utils.cc) - -target_link_libraries(OPERATORS PRIVATE Azure::azure-storage-blobs) -target_link_libraries(OPERATORS PUBLIC ${OPENSSL_LIBRARIES}) -message(STATUS "AWS SDK LIBRARIES: ${AWSSDK_LINK_LIBRARIES}") -message(STATUS "AWS SDK INCLUDE DIRS: ${AWSSDK_INCLUDE_DIRS}") -target_link_libraries(OPERATORS PUBLIC ${AWSSDK_LINK_LIBRARIES}) -# target_include_directories(OPERATORS -# PUBLIC -# ${AWSSDK_INCLUDE_DIRS} -# ) -target_link_libraries(OPERATORS PUBLIC CONSTANTS) -target_link_libraries(VL_CHUNK_OBJ OPERATORS) -target_link_libraries(VL_CHUNK_OBJ UTILS) -target_link_libraries(VL_DSET_OBJ VL_CHUNK_OBJ) -target_link_libraries(VL_FILE OPERATORS) -# target_link_libraries(storage_planner VL_DSET_OBJ) -# target_link_libraries (storage_planner ${HDF5_C_LIBRARIES}) -# target_link_libraries(VL_DSET_OBJ OpenMP::OpenMP_CXX) -target_link_libraries(VL_DSET VL_FILE) -target_link_libraries(VL_DSET VL_DSET_OBJ) -target_link_libraries(VL_GROUP VL_DSET) -target_link_libraries(${TVC_NAME} VL_DSET) -target_link_libraries(${TVC_NAME} VL_FILE) -target_link_libraries(${TVC_NAME} VL_GROUP) -target_link_libraries(${TVC_NAME} ${HDF5_C_LIBRARIES}) - -# set_target_properties (${TVC_NAME} PROPERTIES VERSION ${PROJECT_VERSION}) -set_target_properties (${TVC_NAME} PROPERTIES SOVERSION 1) -set_target_properties (${TVC_NAME} PROPERTIES PUBLIC_HEADER "${TVC_NAME}.h") +# 1. Include directories +# In CMake, you typically set these per-target rather than globally +set(PROJECT_INCLUDE_DIRS + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../include +) + +# 2. Internal Static Libraries +add_library(utils STATIC core/utils.cc) +target_include_directories(utils PUBLIC ${PROJECT_INCLUDE_DIRS}) +target_link_libraries(utils PUBLIC ${HDF5_LIBRARIES}) + +add_library(constants STATIC core/constants.cc) +target_include_directories(constants PUBLIC ${PROJECT_INCLUDE_DIRS}) + +# all_deps would be your list of dependencies (AWS, Azure, SSL, etc.) +add_library(operators STATIC core/operators.cc) +target_include_directories(operators PUBLIC ${PROJECT_INCLUDE_DIRS}) +target_link_libraries(operators PUBLIC constants ${ALL_DEPS}) + +add_library(chunk_obj STATIC s3vl/chunk_obj.cc) +target_include_directories(chunk_obj PUBLIC ${PROJECT_INCLUDE_DIRS}) +target_link_libraries(chunk_obj PUBLIC operators utils ${ALL_DEPS}) + +add_library(dataset_obj STATIC s3vl/dataset_obj.cc) +target_include_directories(dataset_obj PUBLIC ${PROJECT_INCLUDE_DIRS}) +target_link_libraries(dataset_obj PUBLIC chunk_obj ${ALL_DEPS}) + +add_library(file_callbacks STATIC s3vl/file_callbacks.cc) +target_include_directories(file_callbacks PUBLIC ${PROJECT_INCLUDE_DIRS}) +target_link_libraries(file_callbacks PUBLIC operators ${ALL_DEPS}) + +add_library(dataset_callbacks STATIC s3vl/dataset_callbacks.cc) +target_include_directories(dataset_callbacks PUBLIC ${PROJECT_INCLUDE_DIRS}) +target_link_libraries(dataset_callbacks PUBLIC file_callbacks dataset_obj ${ALL_DEPS}) + +add_library(group_callbacks STATIC s3vl/group_callbacks.cc) +target_include_directories(group_callbacks PUBLIC ${PROJECT_INCLUDE_DIRS}) +target_link_libraries(group_callbacks PUBLIC dataset_callbacks ${ALL_DEPS}) + +# 3. The Final VOL Connector (shared library) +add_library(array_morph SHARED s3vl/vol_connector.cc) +set_target_properties(array_morph PROPERTIES PREFIX "lib_") +target_include_directories(array_morph PUBLIC ${PROJECT_INCLUDE_DIRS}) +target_link_libraries(array_morph PUBLIC + dataset_callbacks + file_callbacks + group_callbacks + ${ALL_DEPS} +) + +install(TARGETS array_morph + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} +) diff --git a/arraymorph/src/constants.cc b/arraymorph/src/core/constants.cc similarity index 51% rename from arraymorph/src/constants.cc rename to arraymorph/src/core/constants.cc index 8310670..235ee54 100644 --- a/arraymorph/src/constants.cc +++ b/arraymorph/src/core/constants.cc @@ -1,5 +1,5 @@ -#include "constants.h" +#include "arraymorph/core/constants.h" #include SPlan SP = SPlan::S3; QPlan SINGLE_PLAN = QPlan::NONE; -std::string BUCKET_NAME = ""; \ No newline at end of file +std::string BUCKET_NAME = ""; diff --git a/arraymorph/src/operators.cc b/arraymorph/src/core/operators.cc similarity index 99% rename from arraymorph/src/operators.cc rename to arraymorph/src/core/operators.cc index 04022ef..9fef41e 100644 --- a/arraymorph/src/operators.cc +++ b/arraymorph/src/core/operators.cc @@ -1,5 +1,5 @@ -#include "operators.h" -#include "logger.h" +#include "arraymorph/core/operators.h" +#include "arraymorph/core/logger.h" #include #include #include @@ -300,4 +300,4 @@ herr_t Operators::AzureGetRange(const BlobContainerClient *client, const std::st #endif delete[] buf; return ARRAYMORPH_SUCCESS; -} \ No newline at end of file +} diff --git a/arraymorph/src/core/utils.cc b/arraymorph/src/core/utils.cc new file mode 100644 index 0000000..23a938a --- /dev/null +++ b/arraymorph/src/core/utils.cc @@ -0,0 +1,145 @@ +#include "arraymorph/core/logger.h" +#include "arraymorph/core/utils.h" +#include +#include + +Segment::Segment(hsize_t start_offset, hsize_t end_offset, + hsize_t required_data_size, + const std::list>::iterator &mapping_start, + const std::list>::iterator &mapping_end, + hsize_t mapping_size) + : start_offset(start_offset), end_offset(end_offset), + required_data_size(required_data_size), mapping_start(mapping_start), + mapping_end(mapping_end), mapping_size(mapping_size) { + assert(mapping_start != mapping_end); +} + +Segment::Segment(hsize_t start_offset, hsize_t end_offset, + const std::list>::iterator &mapping_start, + const std::list>::iterator &mapping_end, + hsize_t mapping_size) + : start_offset(start_offset), end_offset(end_offset), + mapping_start(mapping_start), mapping_end(mapping_end), + mapping_size(mapping_size) { + assert(mapping_start != mapping_end); + required_data_size = 0; + for (auto it = mapping_start; it != mapping_end; ++it) + required_data_size += (*it)[2]; +} + +Segment::Segment(const std::list>::iterator &mapping_start, + const std::list>::iterator &mapping_end, + hsize_t mapping_size) + : mapping_start(mapping_start), mapping_end(mapping_end), + mapping_size(mapping_size) { + assert(mapping_start != mapping_end); + start_offset = (*mapping_start)[0]; + auto last_mapping = std::prev(mapping_end); + end_offset = (*last_mapping)[0] + (*last_mapping)[2] - 1; + required_data_size = 0; + for (auto it = mapping_start; it != mapping_end; ++it) + required_data_size += (*it)[2]; +} + +std::string Segment::to_string() { + std::stringstream ss; + ss << "start_offset: " << start_offset << " " << "end_offset: " << end_offset + << std::endl; + for (auto it = mapping_start; it != mapping_end; ++it) + ss << (*it)[0] << " " << (*it)[1] << " " << (*it)[2] << std::endl; + return ss.str(); +} + +std::vector reduceAdd(std::vector &a, + std::vector &b) { + std::vector re; + re.reserve(a.size() * b.size()); + for (auto &i : a) + for (auto &j : b) + re.push_back(i + j); + return re; +} +// calculate each row's start position in serial order +std::vector calSerialOffsets(std::vector> ranges, + std::vector shape) { + assert(ranges.size() == shape.size()); + int dims = ranges.size(); + std::vector offsets_per_dim(dims); + std::vector> offsets(dims - 1); + + offsets_per_dim[dims - 1] = 1; + for (int i = dims - 2; i >= 0; i--) + offsets_per_dim[i] = offsets_per_dim[i + 1] * shape[i + 1]; + + for (int i = 0; i < dims - 1; i++) + for (int j = ranges[i][0]; j <= ranges[i][1]; j++) + offsets[i].push_back(j * offsets_per_dim[i]); + std::vector re = {0}; + for (int i = 0; i < dims - 1; i++) + re = reduceAdd(re, offsets[i]); + for (auto &o : re) + o += ranges[dims - 1][0]; + + return re; +} + +// mapping offsets between two hyperslab +std::list> +mapHyperslab(std::vector &local, std::vector &global, + std::vector &out, hsize_t &input_rsize, + hsize_t &out_rsize, hsize_t &data_size) { + std::list> mapping; + + for (int i = 0; i < local.size(); i++) { + hsize_t idx = global[i] / out_rsize; + hsize_t start_offset = out[idx] + global[i] % out_rsize; + hsize_t remaining = out[idx] + out_rsize - start_offset; + if (remaining >= input_rsize) { + mapping.push_back({local[i], start_offset, input_rsize}); + } else { + mapping.push_back({local[i], start_offset, remaining}); + hsize_t cur = remaining; + int n = (input_rsize - remaining) / out_rsize; + for (int j = 1; j <= n; j++) { + mapping.push_back({local[i] + cur, out[idx + j], out_rsize}); + cur += out_rsize; + } + if ((input_rsize - remaining) % out_rsize > 0) { + mapping.push_back({local[i] + cur, out[idx + n + 1], + (input_rsize - remaining) % out_rsize}); + } + } + } + for (auto &m : mapping) + for (auto &o : m) + o *= data_size; + return mapping; +} + +std::vector> +generateSegments(std::list> &mapping, hsize_t chunk_size) { + std::vector> segments; + size_t n = mapping.size(); + if (n == 0) + return segments; + segments.emplace_back(std::make_unique( + 0, chunk_size - 1, mapping.begin(), mapping.end(), mapping.size())); +#ifdef LOG_ENABLE + for (auto const &s : segments) + Logger::log(s->to_string()); +#endif + return segments; +} + +std::string createQuery(hsize_t data_size, int ndims, + const std::vector &shape, + std::vector> &ranges) { + std::stringstream ss; + ss << "-" << data_size << "-" << ndims << "-"; + for (auto &s : shape) + ss << s << "-"; + for (auto &r : ranges) + ss << r[0] << "-" << r[1] << "-"; + std::string re = ss.str(); + return re.substr(0, re.size() - 1); +} diff --git a/arraymorph/src/operators.h b/arraymorph/src/operators.h deleted file mode 100644 index 662f1ba..0000000 --- a/arraymorph/src/operators.h +++ /dev/null @@ -1,127 +0,0 @@ -#ifndef OPERATORS -#define OPERATORS -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include // for CSVInput -#include // for CSVOutput -#include // for Expressio... -#include // for FileHeade... -#include // for InputSeri... -#include // for OutputSer... -#include // for RecordsEvent -#include // for SelectObj... -#include // for StatsEvent -#include // for Scheme -#include -#include // for Azure blob -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "constants.h" -#include "logger.h" -#include -using namespace Aws; -using namespace Aws::Auth; -using namespace Aws::Http; -using namespace Aws::Client; -using namespace Aws::S3; -using namespace Aws::S3::Model; -using namespace Azure::Storage::Blobs; - - -using CloudClient = std::variant< - std::monostate, - std::unique_ptr, - std::unique_ptr - >; - -extern CloudClient global_cloud_client; - -class OperationTracker { -public: - static OperationTracker& getInstance(); - - void add(); - int get() const; - void reset(); - -private: - std::atomic finish{0}; - - OperationTracker() = default; - OperationTracker(const OperationTracker&) = delete; - OperationTracker& operator=(const OperationTracker&) = delete; -}; - -class AsyncWriteInput:public AsyncCallerContext { -public: - AsyncWriteInput(const char* buf): buf(buf){} - const char* buf; -}; - -class AsyncReadInput: public AsyncCallerContext { -public: - AsyncReadInput(const void* buf, const std::vector> &mapping, const int lambda=0, const std::string bucket_name="", const std::string uri="") - :buf(buf), mapping(mapping), lambda(lambda), bucket_name(bucket_name), uri(uri){} - const void* buf; - const std::vector> mapping; - const int lambda; - // for re-issuing GET if lambda fails - const std::string bucket_name; - const std::string uri; -}; - -class Operators { -public: -// S3 - static Result S3Get(const S3Client *client, const std::string& bucket_name, const Aws::String &object_name); - static herr_t S3GetAsync(const S3Client *client, const std::string& bucket_name, const Aws::String &object_name, const std::shared_ptr input); - static herr_t S3GetByteRangeAsync(const S3Client *client, const std::string& bucket_name, const Aws::String &object_name, - uint64_t beg, uint64_t end, const std::shared_ptr input); - static herr_t S3Put(const S3Client *client, const std::string& bucket_name, const std::string& object_name, Result &re); - static herr_t S3PutBuf(const S3Client *client, const std::string& bucket_name, const std::string& object_name, std::shared_ptr buf, hsize_t length); - static herr_t S3PutAsync(const S3Client *client, const std::string& bucket_name, const Aws::String &object_name, Result &re); - static herr_t S3Delete(const S3Client *client, const std::string& bucket_name, const Aws::String &object_name); - static void GetAsyncCallback(const Aws::S3::S3Client* s3Client, const Aws::S3::Model::GetObjectRequest& request, Aws::S3::Model::GetObjectOutcome outcome, const std::shared_ptr& context); - -//Azure - static Result AzureGet(const BlobContainerClient *client, const std::string& blob_name); - static herr_t AzurePut(const BlobContainerClient *client, const std::string& blob_name, std::shared_ptr buf, size_t length); - static herr_t AzureGetAndProcess(const BlobContainerClient *client, const std::string& blob_name, const std::shared_ptr context); - static herr_t AzureGetRange(const BlobContainerClient *client, const std::string& blob_name, uint64_t beg, uint64_t end, const std::shared_ptr context); -}; - -inline herr_t Operators::S3GetByteRangeAsync(const S3Client *client, const std::string& bucket_name, const Aws::String &object_name, uint64_t beg, uint64_t end, - const std::shared_ptr input) -{ - Logger::log("------ S3getRangeAsync ", object_name); - GetObjectRequest request; - request.SetBucket(bucket_name); - request.SetKey(object_name); - std::stringstream ss; - ss << "bytes=" << beg << '-' << end; - // std::cout << ss.str() << std::endl; - request.SetRange(ss.str().c_str()); - client->GetObjectAsync(request, GetAsyncCallback, input); - return ARRAYMORPH_SUCCESS; -} - - - -#endif diff --git a/arraymorph/src/s3vl/chunk_obj.cc b/arraymorph/src/s3vl/chunk_obj.cc new file mode 100644 index 0000000..7cb190b --- /dev/null +++ b/arraymorph/src/s3vl/chunk_obj.cc @@ -0,0 +1,51 @@ +#include "arraymorph/s3vl/chunk_obj.h" +#include +#include +#include +S3VLChunkObj::S3VLChunkObj(const std::string &name, hid_t dtype, + std::vector> &ranges, + const std::vector &shape, + std::vector &return_offsets) + : uri(name), dtype(dtype), ranges(ranges), shape(shape), + global_offsets(return_offsets) { + this->data_size = H5Tget_size(dtype); + // this->data_size = 4; + this->ndims = shape.size(); + assert(this->ndims = ranges.size()); + + reduc_per_dim.resize(this->ndims); + // reduc_per_dim[0] = 1; + // for (int i = 1; i < this->ndims; i++) + // reduc_per_dim[i] = reduc_per_dim[i - 1] * shape[i - 1]; + reduc_per_dim[this->ndims - 1] = 1; + for (int i = this->ndims - 2; i >= 0; i--) + reduc_per_dim[i] = reduc_per_dim[i + 1] * shape[i + 1]; + this->local_offsets = calSerialOffsets(ranges, shape); + // this->size = reduc_per_dim[ndims - 1] * shape[ndims - 1] * data_size; + this->size = reduc_per_dim[0] * shape[0] * data_size; + hsize_t sr = 1; + for (int i = 0; i < ndims; i++) + sr *= ranges[i][1] - ranges[i][0] + 1; + this->required_size = sr * data_size; +} + +bool S3VLChunkObj::checkFullWrite() { + for (int i = 0; i < ndims; i++) { + if (ranges[i][0] != 0 || ranges[i][1] != shape[i] - 1) + return false; + } + return true; +} + +std::string S3VLChunkObj::to_string() { + std::stringstream ss; + ss << uri << " " << dtype << " " << ndims << " " << data_size << std::endl; + for (auto &s : shape) + ss << s << " "; + ss << std::endl; + for (int i = 0; i < ndims; i++) { + ss << shape[i] << " [" << ranges[i][0] << ", " << ranges[i][1] << "]" + << std::endl; + } + return ss.str(); +} diff --git a/arraymorph/src/s3vl/dataset_callbacks.cc b/arraymorph/src/s3vl/dataset_callbacks.cc new file mode 100644 index 0000000..16e7fd5 --- /dev/null +++ b/arraymorph/src/s3vl/dataset_callbacks.cc @@ -0,0 +1,209 @@ +#include "arraymorph/core/logger.h" +#include "arraymorph/core/operators.h" +#include "arraymorph/s3vl/dataset_callbacks.h" +#include "arraymorph/core/constants.h" +#include +#include +#include +#include +#include +#include + +const hid_t get_native_type(hid_t tid) { + hid_t native_list[] = { + H5T_NATIVE_CHAR, H5T_NATIVE_SHORT, H5T_NATIVE_INT, + H5T_NATIVE_LONG, H5T_NATIVE_LLONG, H5T_NATIVE_UCHAR, + H5T_NATIVE_USHORT, H5T_NATIVE_UINT, H5T_NATIVE_ULONG, + H5T_NATIVE_ULLONG, H5T_NATIVE_FLOAT, H5T_NATIVE_DOUBLE, + H5T_NATIVE_LDOUBLE, H5T_NATIVE_B8, H5T_NATIVE_B16, + H5T_NATIVE_B32, H5T_NATIVE_B64}; + hid_t tmp = H5Tget_native_type(tid, H5T_DIR_DEFAULT); + for (hid_t t : native_list) { + if (H5Tequal(t, tmp)) + return t; + } + return ARRAYMORPH_FAIL; +} + +void *S3VLDatasetCallbacks::S3VL_dataset_create( + void *obj, const H5VL_loc_params_t *loc_params, const char *name, + hid_t lcpl_id, hid_t type_id, hid_t space_id, hid_t dcpl_id, hid_t dapl_id, + hid_t dxpl_id, void **req) { + Logger::log("------ Create Dataset: ", name); + hid_t new_tid = get_native_type(type_id); + if (new_tid < 0) { + Logger::log("------ Unsupported data type"); + return NULL; + } + S3VLFileObj *file_obj = (S3VLFileObj *)obj; + std::string uri = file_obj->name + "/" + name; + + int ndims = H5Sget_simple_extent_ndims(space_id); + + hsize_t dims[ndims]; + hsize_t chunk_dims[ndims]; + hsize_t max_dims[ndims]; + + H5Sget_simple_extent_dims(space_id, dims, max_dims); + + H5D_layout_t layout = H5Pget_layout(dcpl_id); + + if (layout == H5D_CHUNKED) { + H5Pget_chunk(dcpl_id, ndims, chunk_dims); + } else { + memcpy(chunk_dims, dims, sizeof(hsize_t) * ndims); + } + int nchunks = 1; + for (int i = 0; i < ndims; i++) + nchunks *= (dims[i] - 1) / chunk_dims[i] + 1; + std::vector shape(dims, dims + ndims); + std::vector chunk_shape(chunk_dims, chunk_dims + ndims); + + S3VLDatasetObj *ret_obj = + new S3VLDatasetObj(name, uri, new_tid, ndims, shape, chunk_shape, nchunks, + BUCKET_NAME, global_cloud_client); + ret_obj->is_modified = true; + Logger::log("------ Create Metadata:"); + Logger::log(ret_obj->to_string()); + return (void *)ret_obj; +} +void *S3VLDatasetCallbacks::S3VL_dataset_open( + void *obj, const H5VL_loc_params_t *loc_params, const char *name, + hid_t dapl_id, hid_t dxpl_id, void **req) { + Logger::log("------ Open dataset ", name); + // string dset_name(name); + // cout << dset_name << " " << dset_name.size() << endl; + // if (dset_name == "/") { + // cout << "name is /" << endl; + // dset_name = "test"; + // // return NULL; + // } + S3VLFileObj *file_obj = (S3VLFileObj *)obj; + std::string dset_uri = file_obj->name + "/" + name + "/meta"; + + S3VLDatasetObj *dset_obj = + S3VLDatasetObj::getDatasetObj(global_cloud_client, BUCKET_NAME, dset_uri); + Logger::log("------ Get Metadata:"); + Logger::log(dset_obj->to_string()); + // hid_t type_id = dset_obj->dtype; + return (void *)dset_obj; +} +herr_t S3VLDatasetCallbacks::S3VL_dataset_read( + size_t count, void **dset, hid_t *mem_type_id, hid_t *mem_space_id, + hid_t *file_space_id, hid_t plist_id, void **buf, void **req) { + S3VLDatasetObj *dset_obj = (S3VLDatasetObj *)(dset[0]); + Logger::log("------ Read dataset"); + Logger::log("------ Read dataset ", dset_obj->uri); + // string lower_range = getenv("LOWER_RANGE"); + // string upper_range = getenv("UPPER_RANGE"); + // cout << lower_range << " " << upper_range << endl; + auto read_start = std::chrono::high_resolution_clock::now(); + std::cout << "read :" << dset_obj->uri << std::endl; + if (dset_obj->read(*mem_space_id, *file_space_id, buf[0])) { + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration duration = end - read_start; + std::cout << "VOL read time: " << duration.count() << " seconds" + << std::endl; + Logger::log("read successfully"); + return ARRAYMORPH_SUCCESS; + } + Logger::log("read failed"); + return ARRAYMORPH_FAIL; +} +herr_t S3VLDatasetCallbacks::S3VL_dataset_write( + size_t count, void **dset, hid_t *mem_type_id, hid_t *mem_space_id, + hid_t *file_space_id, hid_t plist_id, const void **buf, void **req) { + // TODO: update + S3VLDatasetObj *dset_obj = (S3VLDatasetObj *)(dset[0]); + Logger::log("------ Write dataset ", dset_obj->uri); + // vector mem_space = get_range_from_dataspace(mem_space_id); + // vector file_space = get_range_from_dataspace(file_space_id); + + if (dset_obj->write(*mem_space_id, *file_space_id, buf[0])) { + Logger::log("write successfully"); + return ARRAYMORPH_SUCCESS; + } + Logger::log("write failed"); + return ARRAYMORPH_FAIL; +} + +herr_t S3VLDatasetCallbacks::S3VL_dataset_get(void *dset, + H5VL_dataset_get_args_t *args, + hid_t dxpl_id, void **req) { + S3VLDatasetObj *dset_obj = (S3VLDatasetObj *)dset; + Logger::log("------ Get Space dataset: ", args->op_type); + if (args->op_type == H5VL_dataset_get_t::H5VL_DATASET_GET_DCPL) { + hid_t dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + args->args.get_dcpl.dcpl_id = H5Pcopy(dcpl_id); + } else if (args->op_type == H5VL_dataset_get_t::H5VL_DATASET_GET_SPACE) { + std::vector shape = dset_obj->shape; + // swap(shape[0], shape[1]); + hid_t space_id = H5Screate_simple(dset_obj->ndims, shape.data(), NULL); + args->args.get_space.space_id = space_id; + } else if (args->op_type == H5VL_dataset_get_t::H5VL_DATASET_GET_TYPE) { + hid_t type_id = H5Tcopy(dset_obj->dtype); + args->args.get_type.type_id = type_id; + } + return ARRAYMORPH_SUCCESS; +} +herr_t S3VLDatasetCallbacks::S3VL_dataset_close(void *dset, hid_t dxpl_id, + void **req) { + // TODO: update metadata + Logger::log("------ Close dataset"); + S3VLDatasetObj *dset_obj = (S3VLDatasetObj *)dset; + if (dset_obj->is_modified) + dset_obj->upload(); + + delete dset_obj; + return ARRAYMORPH_SUCCESS; +} + +void *S3VLDatasetCallbacks::S3VL_obj_open(void *obj, + const H5VL_loc_params_t *loc_params, + H5I_type_t *opened_type, + hid_t dxpl_id, void **req) { + Logger::log("------ Open object"); + std::string access_name = loc_params->loc_data.loc_by_name.name; + // cout << "open by name: " << access_name << endl; + // cout << "open type: " << *opened_type << endl; + // if (access_name == "/") { + // *opened_type = H5I_type_t::H5I_GROUP; + // vector test = {"1", "2", "3"}; + // return new S3VLObj(test, 0); + // } + // else { + *opened_type = H5I_type_t::H5I_DATASET; + return S3VLDatasetCallbacks::S3VL_dataset_open( + obj, loc_params, loc_params->loc_data.loc_by_name.name, dxpl_id, dxpl_id, + req); + // } +} + +herr_t S3VLDatasetCallbacks::S3VL_obj_get(void *obj, + const H5VL_loc_params_t *loc_params, + H5VL_object_get_args_t *args, + hid_t dxpl_id, void **req) { + Logger::log("------ Object get"); + // cout << args->op_type << endl; + // cout << loc_params->obj_type << " " << loc_params->type << " " << + // loc_params->loc_data.loc_by_name.name << endl;; + if (args->op_type == H5VL_object_get_t::H5VL_OBJECT_GET_INFO) + args->args.get_info.oinfo->type = H5O_type_t::H5O_TYPE_GROUP; + return ARRAYMORPH_SUCCESS; +} +void *S3VLDatasetCallbacks::S3VL_wrap_object(void *obj, H5I_type_t obj_type, + void *wrap_ctx) { + Logger::log("------ Wrap object"); + return obj; +} + +void *S3VLDatasetCallbacks::S3VL_get_object(const void *obj) { + Logger::log("------ Get object"); + return NULL; +} + +herr_t S3VLDatasetCallbacks::S3VL_dataset_specific( + void *obj, H5VL_dataset_specific_args_t *args, hid_t dxpl_id, void **req) { + Logger::log("------ Specific dataset: ", args->op_type); + return ARRAYMORPH_SUCCESS; +} diff --git a/arraymorph/src/s3vl/dataset_obj.cc b/arraymorph/src/s3vl/dataset_obj.cc new file mode 100644 index 0000000..6033f8e --- /dev/null +++ b/arraymorph/src/s3vl/dataset_obj.cc @@ -0,0 +1,541 @@ +#include "arraymorph/s3vl/dataset_obj.h" +#include "arraymorph/core/logger.h" +#include "arraymorph/core/utils.h" +#include +#include +#include +#include +#include +#include +#include +#include + +uint64_t transfer_size; +int lambda_num, range_num, total_num; + +S3VLDatasetObj::S3VLDatasetObj(const std::string &name, const std::string &uri, + hid_t dtype, int ndims, + std::vector &shape, + std::vector &chunk_shape, int chunk_num, + const std::string &bucket_name, + const CloudClient &client) + : name(name), uri(uri), dtype(dtype), ndims(ndims), shape(shape), + chunk_shape(chunk_shape), chunk_num(chunk_num), bucket_name(bucket_name), + client(client) { + this->data_size = H5Tget_size(this->dtype); + Logger::log("Datasize: ", this->data_size); + // data_size = 4; + num_per_dim.resize(ndims); + reduc_per_dim.resize(ndims); + reduc_per_dim[ndims - 1] = 1; + for (int i = 0; i < ndims; i++) + num_per_dim[i] = (shape[i] - 1) / chunk_shape[i] + 1; + for (int i = ndims - 2; i >= 0; i--) + reduc_per_dim[i] = reduc_per_dim[i + 1] * num_per_dim[i + 1]; + assert(reduc_per_dim[0] * num_per_dim[0] == chunk_num); + + element_per_chunk = 1; + for (auto &s : chunk_shape) + element_per_chunk *= s; +} + +std::vector S3VLDatasetObj::getChunkOffsets(int chunk_idx) { + std::vector idx_per_dim(ndims); + int tmp = chunk_idx; + + for (int i = 0; i < ndims; i++) { + idx_per_dim[i] = tmp / reduc_per_dim[i] * chunk_shape[i]; + tmp %= reduc_per_dim[i]; + } + + // for (int i = ndims - 1; i >= 0; i--) { + // idx_per_dim[i] = tmp / reduc_per_dim[i] * chunk_shape[i]; + // tmp %= reduc_per_dim[i]; + // } + return idx_per_dim; +} + +std::vector> +S3VLDatasetObj::getChunkRanges(int chunk_idx) { + std::vector offsets_per_dim = getChunkOffsets(chunk_idx); + std::vector> re(ndims); + for (int i = 0; i < ndims; i++) + re[i] = {offsets_per_dim[i], offsets_per_dim[i] + chunk_shape[i] - 1}; + return re; +} + +std::vector> +S3VLDatasetObj::selectionFromSpace(hid_t space_id) { + hsize_t start[ndims]; + hsize_t end[ndims]; + std::vector> ranges(ndims); + H5Sget_select_bounds(space_id, start, end); + for (int i = 0; i < ndims; i++) + ranges[i] = {start[i], end[i]}; + // if (ndims >= 2) + // swap(ranges[0], ranges[1]); + return ranges; +} + +void processAzure(std::vector> &chunk_objs, + const std::vector &azure_plans, void *buf, + BlobContainerClient *client, const std::string &bucket_name) { + std::vector> futures; + size_t azure_thread_num = THREAD_NUM; + futures.reserve(azure_thread_num); + size_t cur_batch_size = 0; + + for (int i = 0; i < azure_plans.size(); i++) { + const CPlan &p = azure_plans[i]; + if (cur_batch_size != 0 && + (cur_batch_size + p.num_requests > azure_thread_num)) { + while (OperationTracker::getInstance().get() < cur_batch_size) + ; + cur_batch_size = 0; + for (auto &fut : futures) + fut.wait(); + futures.clear(); + } + for (auto &s : p.segments) { + std::vector> mapping; + for (auto it = s->mapping_start; it != s->mapping_end; ++it) + mapping.push_back({(*it)[0], (*it)[1], (*it)[2]}); + for (auto &m : mapping) + m[0] -= s->start_offset; + auto context = std::make_shared(buf, mapping); + futures.push_back(std::async(std::launch::async, Operators::AzureGetRange, + client, chunk_objs[i]->uri, s->start_offset, + s->end_offset, context)); + cur_batch_size++; + transfer_size += s->end_offset - s->start_offset + 1; + } + } + for (auto &fut : futures) + fut.wait(); + futures.clear(); +} + +void processS3(std::vector> &chunk_objs, + const std::vector &s3_plans, void *buf, + Aws::S3::S3Client *s3_client, const std::string &bucket_name) { + size_t s3_thread_num = THREAD_NUM; + size_t cur_batch_size = 0; + OperationTracker::getInstance().reset(); + for (int i = 0; i < s3_plans.size(); i++) { + const CPlan &p = s3_plans[i]; + if (cur_batch_size != 0 && + (cur_batch_size + p.num_requests > s3_thread_num)) { + while (OperationTracker::getInstance().get() < cur_batch_size) + ; + cur_batch_size = 0; + OperationTracker::getInstance().reset(); + } + for (auto &s : p.segments) { + std::vector> mapping; + for (auto it = s->mapping_start; it != s->mapping_end; ++it) + mapping.push_back({(*it)[0], (*it)[1], (*it)[2]}); + for (auto &m : mapping) + m[0] -= s->start_offset; + auto context = std::make_shared(buf, mapping); + Operators::S3GetByteRangeAsync(s3_client, bucket_name, chunk_objs[i]->uri, + s->start_offset, s->end_offset, context); + cur_batch_size++; + transfer_size += s->end_offset - s->start_offset + 1; + } + } + while (OperationTracker::getInstance().get() < cur_batch_size) + ; +} + +herr_t S3VLDatasetObj::read(hid_t mem_space_id, hid_t file_space_id, + void *buf) { + + // string lambda_merge_path = getenv("AWS_LAMBDA_MERGE_ACCESS_POINT"); + std::vector> ranges; + if (file_space_id != H5S_ALL) { + ranges = selectionFromSpace(file_space_id); + } else { + for (int i = 0; i < ndims; i++) + ranges.push_back({0, shape[i] - 1}); + } + + auto chunk_objs = generateChunks(ranges); + int num = chunk_objs.size(); + std::vector>> segments(num); + std::vector out_offsets; + hsize_t out_row_size; + std::vector>> global_mapping(num); + if (mem_space_id == H5S_ALL) { + // memspace == dataspace + out_offsets = calSerialOffsets(ranges, shape); + // out_row_size = ranges[0][1] - ranges[0][0] + 1; + out_row_size = ranges[ndims - 1][1] - ranges[ndims - 1][0] + 1; + } else { + std::vector> out_ranges = + selectionFromSpace(mem_space_id); + hsize_t dims_out[ndims]; + H5Sget_simple_extent_dims(mem_space_id, dims_out, NULL); + std::vector out_shape(dims_out, dims_out + ndims); + // if (ndims >= 2) + // swap(out_shape[0], out_shape[1]); + out_offsets = calSerialOffsets(out_ranges, out_shape); + // out_row_size = out_ranges[0][1] - out_ranges[0][0] + 1; + out_row_size = out_ranges[ndims - 1][1] - out_ranges[ndims - 1][0] + 1; + } + for (int i = 0; i < num; i++) { + // hsize_t input_row_size = chunk_objs[i]->ranges[0][1] - + // chunk_objs[i]->ranges[0][0] + 1; + hsize_t input_row_size = chunk_objs[i]->ranges[ndims - 1][1] - + chunk_objs[i]->ranges[ndims - 1][0] + 1; + global_mapping[i] = mapHyperslab(chunk_objs[i]->local_offsets, + chunk_objs[i]->global_offsets, out_offsets, + input_row_size, out_row_size, data_size); + segments[i] = generateSegments(global_mapping[i], chunk_objs[i]->size); + } + + transfer_size = 0; + lambda_num = 0; + range_num = 0; + + // cout << "start plan" << endl; + struct timeval start_opt, end_opt; + gettimeofday(&start_opt, NULL); + std::vector plans; + plans.reserve(chunk_objs.size()); + + for (int i = 0; i < chunk_objs.size(); i++) { + plans.emplace_back(i, GET, segments[i].size(), std::move(segments[i])); + } + gettimeofday(&end_opt, NULL); + double opt_t = (1000000 * (end_opt.tv_sec - start_opt.tv_sec) + + end_opt.tv_usec - start_opt.tv_usec) / + 1000000.0; + assert(plans.size() == num); +#ifdef PROFILE_ENABLE + std::cout << "query processer time: " << opt_t << std::endl; + std::cout << "chunk num:" << num << std::endl; +#endif + // cout << "get plans" << endl; +#ifdef LOG_ENABLE + Logger::log("------ Plans:"); + for (int i = 0; i < num; i++) { + Logger::log("chunk: ", chunk_objs[i]->uri); + Logger::log("plan: ", plans[i].qp); + } +#endif + assert(num == plans.size()); + + // std::sort(plans.begin(), plans.end(), [](const CPlan &a, const CPlan &b) { + // return a.qp < b.qp; + // }); + +#ifdef PROFILE_ENABLE + std::cout << "Plans: " << std::endl; + std::cout << "total num: " << plans.size() << std::endl; +#endif + if (SP == AZURE_BLOB) { + auto azure_client = + std::get_if>(&client); + processAzure(chunk_objs, plans, buf, azure_client->get(), bucket_name); + } else { + auto s3_client = std::get_if>(&client); + processS3(chunk_objs, plans, buf, s3_client->get(), bucket_name); + } +#ifdef PROFILE_ENABLE + std::cout << "transfer_size: " << transfer_size << std::endl; +#endif + return ARRAYMORPH_SUCCESS; +} + +herr_t S3VLDatasetObj::write(hid_t mem_space_id, hid_t file_space_id, + const void *buf) { + std::vector> ranges; + if (file_space_id != H5S_ALL) { + ranges = selectionFromSpace(file_space_id); + } else { + for (int i = 0; i < ndims; i++) + ranges.push_back({0, shape[i] - 1}); + } + + auto chunk_objs = generateChunks(ranges); + int num = chunk_objs.size(); + std::vector>> mappings(num); + std::vector source_offsets; + hsize_t source_row_size; + if (mem_space_id == H5S_ALL) { + // memspace == dataspace + source_offsets = calSerialOffsets(ranges, shape); + // source_row_size = ranges[0][1] - ranges[0][0] + 1; + source_row_size = ranges[ndims - 1][1] - ranges[ndims - 1][0] + 1; + } else { + std::vector> source_ranges = + selectionFromSpace(mem_space_id); + hsize_t dims_source[ndims]; + H5Sget_simple_extent_dims(mem_space_id, dims_source, NULL); + std::vector source_shape(dims_source, dims_source + ndims); + // if (ndims >= 2) + // swap(source_shape[0], source_shape[1]); + source_offsets = calSerialOffsets(source_ranges, source_shape); + // source_row_size = source_ranges[0][1] - source_ranges[0][0] + 1; + source_row_size = + source_ranges[ndims - 1][1] - source_ranges[ndims - 1][0] + 1; + } + + for (int i = 0; i < num; i++) { + // hsize_t dest_row_size = chunk_objs[i]->ranges[0][1] - + // chunk_objs[i]->ranges[0][0] + 1; + hsize_t dest_row_size = chunk_objs[i]->ranges[ndims - 1][1] - + chunk_objs[i]->ranges[ndims - 1][0] + 1; + mappings[i] = mapHyperslab(chunk_objs[i]->local_offsets, + chunk_objs[i]->global_offsets, source_offsets, + dest_row_size, source_row_size, data_size); + } + + std::vector> futures; + futures.reserve(THREAD_NUM); + int cur_batch = 0; + + while (cur_batch < num) { + for (int idx = cur_batch; idx < std::min(num, cur_batch + THREAD_NUM); + idx++) { + size_t length = chunk_objs[idx]->size; + auto upload_buf = std::shared_ptr(new char[length], + std::default_delete()); + auto raw_buf = upload_buf.get(); +#ifdef DUMMY_WRITE + memset(raw_buf, 0, length); +#else + for (auto &m : mappings[idx]) { + memcpy(raw_buf + m[0], (char *)buf + m[1], m[2]); + } +#endif + if (SP == SPlan::AZURE_BLOB) { + auto azure_client = + std::get_if>(&client); + futures.push_back(std::async(std::launch::async, Operators::AzurePut, + azure_client->get(), chunk_objs[idx]->uri, + upload_buf, length)); + } else { + auto s3_client = + std::get_if>(&client); + futures.push_back(std::async(std::launch::async, Operators::S3PutBuf, + s3_client->get(), bucket_name, + chunk_objs[idx]->uri, upload_buf, length)); + } + } + cur_batch = std::min(num, cur_batch + THREAD_NUM); + for (auto &fut : futures) + fut.wait(); + futures.clear(); + } + return ARRAYMORPH_SUCCESS; +} + +std::vector> +S3VLDatasetObj::generateChunks(std::vector> ranges) { + assert(ranges.size() == ndims); + std::vector accessed_chunks; + // get accessed chunks + std::vector> chunk_ranges(ndims); + for (int i = 0; i < ndims; i++) + chunk_ranges[i] = {ranges[i][0] / chunk_shape[i], + ranges[i][1] / chunk_shape[i]}; + std::vector chunk_offsets = + calSerialOffsets(chunk_ranges, num_per_dim); + // for (int i = 0; i <= chunk_ranges[0][1] - chunk_ranges[0][0]; i++) + for (int i = 0; i <= chunk_ranges[ndims - 1][1] - chunk_ranges[ndims - 1][0]; + i++) + for (auto &n : chunk_offsets) + accessed_chunks.push_back(i + n); + + // iterate accessed chunks and generate queries + std::vector> chunk_objs; + chunk_objs.reserve(accessed_chunks.size()); + Logger::log("------ # of chunks ", accessed_chunks.size()); + for (auto &c : accessed_chunks) { + std::vector offsets = getChunkOffsets(c); + std::vector> local_ranges(ndims); + for (int i = 0; i < ndims; i++) { + hsize_t left = std::max(offsets[i], ranges[i][0]) - offsets[i]; + hsize_t right = + std::min(offsets[i] + chunk_shape[i] - 1, ranges[i][1]) - offsets[i]; + local_ranges[i] = {left, right}; + } + std::string chunk_uri = uri + "/" + std::to_string(c); + // get output serial offsets for each row + std::vector> global_ranges(ndims); + std::vector result_shape(ndims); + for (int i = 0; i < ndims; i++) { + global_ranges[i] = {local_ranges[i][0] + offsets[i] - ranges[i][0], + local_ranges[i][1] + offsets[i] - ranges[i][0]}; + result_shape[i] = ranges[i][1] - ranges[i][0] + 1; + } + std::vector result_serial_offsets = + calSerialOffsets(global_ranges, result_shape); + + // S3VLChunkObj *chunk = new S3VLChunkObj(chunk_uri, dtype, local_ranges, + // chunk_shape, result_serial_offsets); + auto chunk = std::make_shared( + chunk_uri, dtype, local_ranges, chunk_shape, result_serial_offsets); + // Logger::log("------ Generate Chunk"); + // Logger::log(chunk->to_string()); + // std::cout << chunk->to_string() << std::endl; + chunk_objs.push_back(chunk); + } + return chunk_objs; +} + +// read/write + +void S3VLDatasetObj::upload() { + Logger::log("------ Upload metadata " + uri); + int length; + char *buffer = toBuffer(&length); + std::shared_ptr upload_buf(buffer, std::default_delete()); + std::string meta_name = uri + "/meta"; + Result re{std::vector(buffer, buffer + length)}; + + if (SP == SPlan::S3) { + auto s3_client = std::get_if>(&client); + + if (!s3_client || !s3_client->get()) { + std::cerr << "S3 client not initialized correctly!" << std::endl; + return; + } + Operators::S3Put(s3_client->get(), bucket_name, meta_name, re); + } else { + auto azure_client = + std::get_if>(&client); + if (!azure_client || !azure_client->get()) { + std::cerr << "Azure client not initialized correctly!" << std::endl; + return; + } + Operators::AzurePut(azure_client->get(), meta_name, upload_buf, length); + } +} + +S3VLDatasetObj *S3VLDatasetObj::getDatasetObj(const CloudClient &client, + const std::string &bucket_name, + const std::string &uri) { + Result re; + if (SP == SPlan::S3) { + auto s3_client = std::get_if>(&client); + + if (!s3_client || !s3_client->get()) { + std::cerr << "S3 client not initialized correctly!" << std::endl; + return nullptr; + } + re = Operators::S3Get(s3_client->get(), bucket_name, uri); + + } else { + auto azure_client = + std::get_if>(&client); + if (!azure_client || !azure_client->get()) { + std::cerr << "Azure client not initialized correctly!" << std::endl; + return nullptr; + } + re = Operators::AzureGet(azure_client->get(), uri); + } + + if (re.data.empty()) { + std::cerr << "Didn't get metadata!" << std::endl; + return nullptr; + } + return S3VLDatasetObj::getDatasetObj(client, bucket_name, re.data); +} + +char *S3VLDatasetObj::toBuffer(int *length) { + int size = 8 + name.size() + uri.size() + sizeof(hid_t) + 4 + + 2 * ndims * sizeof(hsize_t) + 4; + *length = size; + char *buffer = new char[size]; + int c = 0; + int name_length = name.size(); + memcpy(buffer, &name_length, 4); + c += 4; + memcpy(buffer + c, name.c_str(), name.size()); + c += name.size(); + int uri_length = uri.size(); + memcpy(buffer + c, &uri_length, 4); + c += 4; + memcpy(buffer + c, uri.c_str(), uri.size()); + c += uri.size(); + memcpy(buffer + c, &dtype, sizeof(hid_t)); + c += sizeof(hid_t); + memcpy(buffer + c, &ndims, 4); + c += 4; + memcpy(buffer + c, shape.data(), sizeof(hsize_t) * ndims); + c += sizeof(hsize_t) * ndims; + memcpy(buffer + c, chunk_shape.data(), sizeof(hsize_t) * ndims); + c += sizeof(hsize_t) * ndims; + memcpy(buffer + c, &chunk_num, 4); + c += 4; + return buffer; +} + +S3VLDatasetObj *S3VLDatasetObj::getDatasetObj(const CloudClient &client, + const std::string &bucket_name, + std::vector &buffer) { + std::string name, uri; + int ndims, chunk_num; + hid_t dtype; + int c = 0; + + int name_length; + memcpy(&name_length, buffer.data() + c, sizeof(int)); + c += sizeof(int); + + name.assign(buffer.data() + c, name_length); // name + c += name_length; + + int uri_length; + memcpy(&uri_length, buffer.data() + c, sizeof(int)); + c += sizeof(int); + + uri.assign(buffer.data() + c, uri_length); // uri + c += uri_length; + + memcpy(&dtype, buffer.data() + c, sizeof(hid_t)); // dtype + c += sizeof(hid_t); + + memcpy(&ndims, buffer.data() + c, sizeof(int)); // ndims + c += sizeof(int); + + std::vector shape(ndims); // shape + memcpy(shape.data(), (hsize_t *)(buffer.data() + c), sizeof(hsize_t) * ndims); + c += sizeof(hsize_t) * ndims; + + std::vector chunk_shape(ndims); // chunk_shape + memcpy(chunk_shape.data(), (hsize_t *)(buffer.data() + c), + sizeof(hsize_t) * ndims); + c += sizeof(hsize_t) * ndims; + + memcpy(&chunk_num, buffer.data() + c, sizeof(int)); // chunk num + return new S3VLDatasetObj(name, uri, dtype, ndims, shape, chunk_shape, + chunk_num, bucket_name, client); +} + +std::string S3VLDatasetObj::to_string() { + std::stringstream ss; + ss << name << " " << uri << std::endl; + ss << dtype << " " << ndims << std::endl; + ss << chunk_num << " " << element_per_chunk << std::endl; + ; + for (int i = 0; i < ndims; i++) { + ss << shape[i] << " "; + } + ss << std::endl; + for (int i = 0; i < ndims; i++) { + ss << chunk_shape[i] << " "; + } + ss << std::endl; + for (int i = 0; i < ndims; i++) { + ss << num_per_dim[i] << " "; + } + ss << std::endl; + for (int i = 0; i < ndims; i++) { + ss << reduc_per_dim[i] << " "; + } + ss << std::endl; + return ss.str(); +} diff --git a/arraymorph/src/s3vl/file_callbacks.cc b/arraymorph/src/s3vl/file_callbacks.cc new file mode 100644 index 0000000..85e368b --- /dev/null +++ b/arraymorph/src/s3vl/file_callbacks.cc @@ -0,0 +1,140 @@ +#include "arraymorph/s3vl/file_callbacks.h" +#include "arraymorph/core/constants.h" +#include "arraymorph/core/logger.h" +#include "arraymorph/core/operators.h" +#include +#include +#include + +CloudClient get_client() { + Logger::log("Init cloud clients"); + // AWS connection + CloudClient client; + if (SP == SPlan::S3) { + std::string access_key = getenv("AWS_ACCESS_KEY_ID"); + std::string secret_key = getenv("AWS_SECRET_ACCESS_KEY"); + Aws::Auth::AWSCredentials cred(access_key, secret_key); + std::unique_ptr s3ClientConfig = + std::make_unique(); + + const char *use_tls = + getenv("AWS_USE_TLS"); // Is TLS necessary. Does not use it by default. + + // Env var is char pointer by default. Checking to see if value was + // provided, ('true' or 'false'). If so, cast to std::string_view to avoid + // heap allocation and check for value. + s3ClientConfig->scheme = use_tls && std::string_view(use_tls) == "true" + ? Scheme::HTTPS + : Scheme::HTTP; + const char *endpoint = getenv("AWS_ENDPOINT_URL_S3"); // Custom S3 endpoint + if (endpoint) { + s3ClientConfig->endpointOverride = endpoint; + } + + const char *region = + getenv("AWS_REGION"); // Region where bucket is located i.e. us-east-1 + if (region) { + s3ClientConfig->region = region; + } + + const char *signed_payloads = + getenv("AWS_SIGNED_PAYLOADS"); // Whether or not to sign each payload. + // Garage requires it to be on. May have + // affect on performance, needs to be + // tested. Off by default + auto payload_signing_policy = + *signed_payloads && std::string_view(signed_payloads) == "true" + ? Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always + : Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never; + + const char *path_style = + getenv("AWS_USE_PATH_STYLE"); // Some S3-compatible stores require path + // styles, 'bucket.endpoint' vs + // 'endpoint/bucket'(with path-style). + bool use_path_style = + *path_style && std::string_view(path_style) == "true" ? true : false; + + s3ClientConfig->maxConnections = s3Connections; + s3ClientConfig->requestTimeoutMs = requestTimeoutMs; + s3ClientConfig->connectTimeoutMs = connectTimeoutMs; + s3ClientConfig->retryStrategy = + std::make_shared(retries); +#ifdef POOLEXECUTOR + s3ClientConfig->executor = + Aws::MakeShared("test", + poolSize); +#endif + Logger::log("------ Create Client config: maxConnections=", + s3ClientConfig->maxConnections); + client = std::make_unique( + cred, std::move(*s3ClientConfig), payload_signing_policy, + use_path_style); + s3ClientConfig.reset(); + } + // Azure connection + else { + std::string azure_connection_string = + getenv("AZURE_STORAGE_CONNECTION_STRING"); + + Azure::Core::Http::Policies::RetryOptions retryOptions; + retryOptions.MaxRetries = retries; + retryOptions.RetryDelay = std::chrono::milliseconds(200); + retryOptions.MaxRetryDelay = std::chrono::seconds(2); + + Azure::Storage::Blobs::BlobClientOptions clientOptions; + clientOptions.Retry = retryOptions; + client = std::make_unique( + BlobContainerClient::CreateFromConnectionString( + azure_connection_string, BUCKET_NAME, clientOptions)); + } + return client; +} + +void *S3VLFileCallbacks::S3VL_file_create(const char *name, unsigned flags, + hid_t fcpl_id, hid_t fapl_id, + hid_t dxpl_id, void **req) { + S3VLFileObj *ret_obj = new S3VLFileObj(); + std::string path(name); + if (path.rfind("./", 0) == 0) { + path = path.substr(2); + } + ret_obj->name = path; + Logger::log("------ Create File:", path); + if (std::holds_alternative(global_cloud_client)) { + global_cloud_client = get_client(); + } + return (void *)ret_obj; +} +void *S3VLFileCallbacks::S3VL_file_open(const char *name, unsigned flags, + hid_t fapl_id, hid_t dxpl_id, + void **req) { + S3VLFileObj *ret_obj = new S3VLFileObj(); + std::string path(name); + Logger::log("------ Open File:", path); + if (path.rfind("./", 0) == 0) { + path = path.substr(2); + } + ret_obj->name = path; + if (std::holds_alternative(global_cloud_client)) { + global_cloud_client = get_client(); + } + return (void *)ret_obj; +} +herr_t S3VLFileCallbacks::S3VL_file_close(void *file, hid_t dxpl_id, + void **req) { + S3VLFileObj *file_obj = (S3VLFileObj *)file; + Logger::log("------ Close File: ", file_obj->name); + delete file_obj; + return ARRAYMORPH_SUCCESS; +} + +herr_t S3VLFileCallbacks::S3VL_file_get(void *file, H5VL_file_get_args_t *args, + hid_t dxpl_id, void **req) { + Logger::log("------ Get File"); + if (args->op_type == H5VL_file_get_t::H5VL_FILE_GET_FCPL) { + hid_t fcpl_id = H5Pcreate(H5P_FILE_CREATE); + args->args.get_fcpl.fcpl_id = H5Pcopy(fcpl_id); + // args->args.get_fcpl.fcpl_id = 1; + } + return ARRAYMORPH_SUCCESS; +} diff --git a/arraymorph/src/s3vl/group_callbacks.cc b/arraymorph/src/s3vl/group_callbacks.cc new file mode 100644 index 0000000..736c633 --- /dev/null +++ b/arraymorph/src/s3vl/group_callbacks.cc @@ -0,0 +1,115 @@ +#include "arraymorph/s3vl/group_callbacks.h" +#include "arraymorph/core/logger.h" +#include "arraymorph/core/operators.h" +#include "arraymorph/s3vl/dataset_callbacks.h" +#include +#include + +herr_t S3VLGroupCallbacks::S3VLgroup_get(void *obj, H5VL_group_get_args_t *args, + hid_t dxpl_id, void **req) { + Logger::log("------ Get Group"); + // cout << "is NULL: " << (obj == NULL) << endl; + // if (args->args.get_info.loc_params.obj_type == H5I_type_t::H5I_GROUP ) { + // S3VLObj *get_obj = (S3VLObj*)obj; + // cout << "obj info: " << get_obj->idx << " " << get_obj->keys[1] << + // endl; + // } + // cout << "get type: " << args->op_type << endl; + // cout << "get para: " << args->args.get_info.loc_params.obj_type << " " << + // args->args.get_info.loc_params.type << endl; if (args->op_type == + // H5VL_group_get_t::H5VL_GROUP_GET_INFO) { + // args->args.get_info.loc_params.type = + // H5VL_loc_type_t::H5VL_OBJECT_BY_IDX; auto ginfo = + // args->args.get_info.ginfo; ginfo->nlinks = 3; ginfo->mounted = true; + // } + // else if (args->op_type == H5VL_group_get_t::H5VL_GROUP_GET_GCPL) { + // hid_t gcpl_id = H5Pcreate(H5P_GROUP_CREATE); + // args->args.get_gcpl.gcpl_id = H5Pcopy(gcpl_id); + // } + return ARRAYMORPH_SUCCESS; +} + +void *S3VLGroupCallbacks::S3VLgroup_open(void *obj, + const H5VL_loc_params_t *loc_params, + const char *name, hid_t gapl_id, + hid_t dxpl_id, void **req) { + /* + TODO: This function is a simple way to list the dataset names using h5ls + */ + Logger::log("------ Open Group"); + Logger::log("name: ", name); + // if (strcmp(name, "/") == 0) { + // S3VLFileObj *f_obj = (S3VLFileObj*)obj; + // std::string keys_url = f_obj->name + "/keys"; + // std::string bucket_name = getenv("BUCKET_NAME"); + // Result keys; + // // aws connection + // if (SP == SPlan::S3) { + // std::string access_key = getenv("AWS_ACCESS_KEY_ID"); + // std::string secret_key = getenv("AWS_SECRET_ACCESS_KEY"); + // Aws::Auth::AWSCredentials cred(access_key, secret_key); + // if (s3Configured == false) { + // s3Configured = true; + // s3ClientConfig = new Aws::Client::ClientConfiguration(); + // s3ClientConfig->scheme = Scheme::HTTP; + // s3ClientConfig->maxConnections = s3Connections; + // s3ClientConfig->requestTimeoutMs = requestTimeoutMs; + // s3ClientConfig->connectTimeoutMs = connectTimeoutMs; + // #ifdef POOLEXECUTOR + // s3ClientConfig->executor = + // Aws::MakeShared("test", + // poolSize); + // #endif + // Logger::log("------ Create Client config: maxConnections=", + // s3ClientConfig->maxConnections); + // } + // std::unique_ptr client = + // std::make_unique(cred, *s3ClientConfig); + // keys = Operators::S3Get(client.get(), bucket_name, keys_url); + // // list dsets + // std::cout << "datasets:" << std::endl; + // std::cout.write(keys.data, keys.length); + // } + // } + return (void *)obj; +} + +herr_t S3VLGroupCallbacks::S3VLgroup_close(void *grp, hid_t dxpl_id, + void **req) { + Logger::log("------ Close Group"); + S3VLFileObj *f_obj = (S3VLFileObj *)grp; + // free(f_obj); + return ARRAYMORPH_SUCCESS; +} + +herr_t S3VLGroupCallbacks::S3VLlink_get(void *obj, + const H5VL_loc_params_t *loc_params, + H5VL_link_get_args_t *args, + hid_t dxpl_id, void **req) { + Logger::log("------ Get Link"); + // cout << "get type: " << args->op_type << endl; + // cout << "get para: " << args->args.get_info.loc_params.obj_type << " " << + // args->args.get_info.loc_params.type << endl; + return ARRAYMORPH_SUCCESS; +} + +herr_t S3VLGroupCallbacks::S3VLlink_specific( + void *obj, const H5VL_loc_params_t *loc_params, + H5VL_link_specific_args_t *args, hid_t dxpl_id, void **req) { + Logger::log("------ Specific Link"); + // S3VLGroupObj *get_obj = (S3VLGroupObj*)obj; + // cout << "obj info: " << get_obj->keys.size() << " " << get_obj->keys[1] << + // endl; + + // cout << "get type: " << args->op_type << endl; + // // cout << "get para: " << args->args.get_info.loc_params.obj_type << " " + // << args->args.get_info.loc_params.type << endl; if (args->op_type == + // H5VL_link_specific_t::H5VL_LINK_ITER) { + // auto it_args = args->args.iterate; + // cout << it_args.recursive << endl; + // cout << it_args.op << endl; + // // cout << *(it_args.idx_p) << endl; + // // cout << loc_params->loc_data.loc_by_name.name << endl; + // } + return ARRAYMORPH_SUCCESS; +} diff --git a/arraymorph/src/s3vl/vol_connector.cc b/arraymorph/src/s3vl/vol_connector.cc new file mode 100644 index 0000000..a619cc2 --- /dev/null +++ b/arraymorph/src/s3vl/vol_connector.cc @@ -0,0 +1,164 @@ +/* This connector's header */ +#include "arraymorph/core/constants.h" +#include "arraymorph/s3vl/dataset_callbacks.h" +#include "arraymorph/s3vl/file_callbacks.h" +#include "arraymorph/s3vl/group_callbacks.h" +#include "arraymorph/s3vl/initialize.h" +#include "arraymorph/s3vl/vol_connector.h" + +#include +#include +#include +#include +// #include "operators.h" +// #include "metadata.h" + +/* Introscpect */ + +static herr_t S3_introspect_opt_query(void *obj, H5VL_subclass_t cls, + int op_type, uint64_t *flags); + +static herr_t S3_get_conn_cls(void *obj, H5VL_get_conn_lvl_t lvl, + const struct H5VL_class_t **conn_cls); + +/* The VOL class struct */ +static const H5VL_class_t template_class_g = { + 3, /* VOL class struct version */ + S3_VOL_CONNECTOR_VALUE, /* value */ + S3_VOL_CONNECTOR_NAME, /* name */ + 1, /* version */ + 0, /* capability flags */ + S3VLINITIALIZE::s3VL_initialize_init, /* initialize */ + S3VLINITIALIZE::s3VL_initialize_close, /* terminate */ + { + /* info_cls */ + (size_t)0, /* size */ + NULL, /* copy */ + NULL, /* compare */ + NULL, /* free */ + NULL, /* to_str */ + NULL, /* from_str */ + }, + { + /* wrap_cls */ + S3VLDatasetCallbacks::S3VL_get_object, /* get_object */ + NULL, /* get_wrap_ctx */ + S3VLDatasetCallbacks::S3VL_wrap_object, /* wrap_object */ + NULL, /* unwrap_object */ + NULL, /* free_wrap_ctx */ + }, + { + /* attribute_cls */ + NULL, /* create */ + NULL, /* open */ + NULL, /* read */ + NULL, /* write */ + NULL, /* get */ + NULL, /* specific */ + NULL, /* optional */ + NULL /* close */ + }, + { + /* dataset_cls */ + S3VLDatasetCallbacks::S3VL_dataset_create, /* create */ + S3VLDatasetCallbacks::S3VL_dataset_open, /* open */ + S3VLDatasetCallbacks::S3VL_dataset_read, /* read */ + S3VLDatasetCallbacks::S3VL_dataset_write, /* write */ + S3VLDatasetCallbacks::S3VL_dataset_get, /* get */ + S3VLDatasetCallbacks::S3VL_dataset_specific, /* specific */ + NULL, /* optional */ + S3VLDatasetCallbacks::S3VL_dataset_close /* close */ + }, + { + /* datatype_cls */ + NULL, /* commit */ + NULL, /* open */ + NULL, /* get_size */ + NULL, /* specific */ + NULL, /* optional */ + NULL /* close */ + }, + { + /* file_cls */ + S3VLFileCallbacks::S3VL_file_create, /* create */ + S3VLFileCallbacks::S3VL_file_open, /* open */ + S3VLFileCallbacks::S3VL_file_get, /* get */ + NULL, /* specific */ + NULL, /* optional */ + S3VLFileCallbacks::S3VL_file_close /* close */ + }, + { + /* group_cls */ + NULL, /* create */ + S3VLGroupCallbacks::S3VLgroup_open, /* open */ + S3VLGroupCallbacks::S3VLgroup_get, /* get */ + // NULL, + NULL, /* specific */ + NULL, /* optional */ + S3VLGroupCallbacks::S3VLgroup_close // NULL /* close */ + }, + { + /* link_cls */ + NULL, /* create */ + NULL, /* copy */ + NULL, /* move */ + S3VLGroupCallbacks::S3VLlink_get, /* get */ + NULL, // S3VLGroupCallbacks::S3VLlink_specific, /* specific */ + NULL /* optional */ + }, + { + /* object_cls */ + S3VLDatasetCallbacks::S3VL_obj_open, /* open */ + NULL, /* copy */ + S3VLDatasetCallbacks::S3VL_obj_get, /* get */ + NULL, /* specific */ + NULL /* optional */ + }, + { + /* introscpect_cls */ + S3_get_conn_cls, /* get_conn_cls */ + NULL, /* get_cap_flags */ + S3_introspect_opt_query /* opt_query */ + }, + { + /* request_cls */ + NULL, /* wait */ + NULL, /* notify */ + NULL, /* cancel */ + NULL, /* specific */ + NULL, /* optional */ + NULL /* free */ + }, + { + /* blob_cls */ + NULL, /* put */ + NULL, /* get */ + NULL, /* specific */ + NULL /* optional */ + }, + { + /* token_cls */ + NULL, /* cmp */ + NULL, /* to_str */ + NULL /* from_str */ + }, + NULL /* optional */ +}; + +/* These two functions are necessary to load this plugin using + * the HDF5 library. + */ + +static herr_t S3_introspect_opt_query(void *obj, H5VL_subclass_t cls, + int op_type, uint64_t *flags) { + return ARRAYMORPH_SUCCESS; +} + +static herr_t S3_get_conn_cls(void *obj, H5VL_get_conn_lvl_t lvl, + const struct H5VL_class_t **conn_cls) { + *conn_cls = new H5VL_class_t; + return ARRAYMORPH_SUCCESS; +} + +H5PL_type_t H5PLget_plugin_type(void) { return H5PL_TYPE_VOL; } +const void *H5PLget_plugin_info(void) { return &template_class_g; } diff --git a/arraymorph/src/s3vl_chunk_obj.cc b/arraymorph/src/s3vl_chunk_obj.cc deleted file mode 100644 index 9afc594..0000000 --- a/arraymorph/src/s3vl_chunk_obj.cc +++ /dev/null @@ -1,48 +0,0 @@ -#include "s3vl_chunk_obj.h" -#include -#include -#include -S3VLChunkObj::S3VLChunkObj(const std::string& name, hid_t dtype, std::vector>& ranges, - const std::vector& shape, std::vector& return_offsets): - uri(name), dtype(dtype), ranges(ranges), shape(shape), global_offsets(return_offsets) { - this->data_size = H5Tget_size(dtype); - // this->data_size = 4; - this->ndims = shape.size(); - assert(this->ndims = ranges.size()); - - reduc_per_dim.resize(this->ndims); - // reduc_per_dim[0] = 1; - // for (int i = 1; i < this->ndims; i++) - // reduc_per_dim[i] = reduc_per_dim[i - 1] * shape[i - 1]; - reduc_per_dim[this->ndims - 1] = 1; - for (int i = this->ndims - 2; i >= 0; i--) - reduc_per_dim[i] = reduc_per_dim[i + 1] * shape[i + 1]; - this->local_offsets = calSerialOffsets(ranges, shape); - // this->size = reduc_per_dim[ndims - 1] * shape[ndims - 1] * data_size; - this->size = reduc_per_dim[0] * shape[0] * data_size; - hsize_t sr = 1; - for (int i = 0; i < ndims; i++) - sr *= ranges[i][1] - ranges[i][0] + 1; - this->required_size = sr * data_size; -} - -bool S3VLChunkObj::checkFullWrite() { - for (int i = 0; i < ndims; i++) { - if (ranges[i][0] != 0 || ranges[i][1] != shape[i] - 1) - return false; - } - return true; -} - - -std::string S3VLChunkObj::to_string() { - std::stringstream ss; - ss << uri << " " << dtype << " " << ndims << " " << data_size << std::endl; - for (auto &s : shape) - ss << s << " "; - ss << std::endl; - for (int i = 0; i < ndims; i++) { - ss << shape[i] << " [" << ranges[i][0] << ", " << ranges[i][1] << "]" << std::endl; - } - return ss.str(); -} diff --git a/arraymorph/src/s3vl_chunk_obj.h b/arraymorph/src/s3vl_chunk_obj.h deleted file mode 100644 index 4797f82..0000000 --- a/arraymorph/src/s3vl_chunk_obj.h +++ /dev/null @@ -1,60 +0,0 @@ -#ifndef S3VL_CHUNK_OBJ -#define S3VL_CHUNK_OBJ -#include -#include -#include -#include -#include -#include -#include "constants.h" -#include "operators.h" -#include "logger.h" -#include "utils.h" -#include - - -typedef struct CPlan -{ - int chunk_id; - QPlan qp; - size_t num_requests; - std::vector> segments; - std::string lambda_query = ""; - - CPlan(int id, QPlan q, size_t reqs, std::vector> &&s) - : chunk_id(id), qp(q), num_requests(reqs), segments(std::move(s)) {} - - CPlan(const CPlan&) = delete; - CPlan& operator=(const CPlan&) = delete; - - CPlan(CPlan&&) = default; - CPlan& operator=(CPlan&&) = default; - - -} CPlan; - -class S3VLChunkObj -{ -public: - S3VLChunkObj(const std::string& uri, hid_t dtype, std::vector>& ranges, - const std::vector& shape, std::vector& return_offsets); - ~S3VLChunkObj(){}; - - std::string to_string(); - bool checkFullWrite(); - - const std::string uri; - hid_t dtype; - std::vector> ranges; - const std::vector shape; - std::vector reduc_per_dim; - int ndims; - - int data_size; - hsize_t size; - hsize_t required_size; - std::vector global_offsets; - std::vector local_offsets; -}; - -#endif \ No newline at end of file diff --git a/arraymorph/src/s3vl_dataset_callbacks.cc b/arraymorph/src/s3vl_dataset_callbacks.cc deleted file mode 100644 index 922ee60..0000000 --- a/arraymorph/src/s3vl_dataset_callbacks.cc +++ /dev/null @@ -1,206 +0,0 @@ -#include "s3vl_dataset_callbacks.h" -#include "operators.h" -#include "logger.h" -#include -#include -#include -#include -#include "constants.h" -#include -#include -#include - -const hid_t get_native_type(hid_t tid) { - hid_t native_list[] = { - H5T_NATIVE_CHAR, - H5T_NATIVE_SHORT, - H5T_NATIVE_INT, - H5T_NATIVE_LONG, - H5T_NATIVE_LLONG, - H5T_NATIVE_UCHAR, - H5T_NATIVE_USHORT, - H5T_NATIVE_UINT, - H5T_NATIVE_ULONG, - H5T_NATIVE_ULLONG, - H5T_NATIVE_FLOAT, - H5T_NATIVE_DOUBLE, - H5T_NATIVE_LDOUBLE, - H5T_NATIVE_B8, - H5T_NATIVE_B16, - H5T_NATIVE_B32, - H5T_NATIVE_B64 - }; - hid_t tmp = H5Tget_native_type(tid, H5T_DIR_DEFAULT); - for (hid_t t: native_list) { - if (H5Tequal(t, tmp)) - return t; - } - return ARRAYMORPH_FAIL; -} - - -void* S3VLDatasetCallbacks::S3VL_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const char *name, - hid_t lcpl_id, hid_t type_id, hid_t space_id, hid_t dcpl_id, hid_t dapl_id, hid_t dxpl_id, void **req) { - Logger::log("------ Create Dataset: ", name); - hid_t new_tid = get_native_type(type_id); - if (new_tid < 0) { - Logger::log("------ Unsupported data type"); - return NULL; - } - S3VLFileObj *file_obj = (S3VLFileObj*)obj; - std::string uri = file_obj->name + "/" + name; - - int ndims = H5Sget_simple_extent_ndims(space_id); - - hsize_t dims[ndims]; - hsize_t chunk_dims[ndims]; - hsize_t max_dims[ndims]; - - H5Sget_simple_extent_dims(space_id, dims, max_dims); - - H5D_layout_t layout = H5Pget_layout(dcpl_id); - - if (layout == H5D_CHUNKED) { - H5Pget_chunk(dcpl_id, ndims, chunk_dims); - } - else { - memcpy(chunk_dims, dims, sizeof(hsize_t) * ndims); - } - int nchunks = 1; - for (int i = 0; i < ndims; i++) - nchunks *= (dims[i] - 1) / chunk_dims[i] + 1; - std::vector shape(dims, dims + ndims); - std::vector chunk_shape(chunk_dims, chunk_dims + ndims); - - S3VLDatasetObj *ret_obj = new S3VLDatasetObj(name, uri, new_tid, ndims, shape, chunk_shape, nchunks, BUCKET_NAME, global_cloud_client); - ret_obj->is_modified = true; - Logger::log("------ Create Metadata:"); - Logger::log(ret_obj->to_string()); - return (void*)ret_obj; - -} -void* S3VLDatasetCallbacks::S3VL_dataset_open(void *obj, const H5VL_loc_params_t *loc_params, const char *name, hid_t dapl_id, - hid_t dxpl_id, void **req) { - Logger::log("------ Open dataset ", name); - // string dset_name(name); - // cout << dset_name << " " << dset_name.size() << endl; - // if (dset_name == "/") { - // cout << "name is /" << endl; - // dset_name = "test"; - // // return NULL; - // } - S3VLFileObj *file_obj = (S3VLFileObj*)obj; - std::string dset_uri = file_obj->name + "/" + name + "/meta"; - - - S3VLDatasetObj *dset_obj = S3VLDatasetObj::getDatasetObj(global_cloud_client, BUCKET_NAME, dset_uri); - Logger::log("------ Get Metadata:"); - Logger::log(dset_obj->to_string()); - // hid_t type_id = dset_obj->dtype; - return (void*)dset_obj; -} -herr_t S3VLDatasetCallbacks::S3VL_dataset_read(size_t count, void **dset, hid_t *mem_type_id, hid_t *mem_space_id, hid_t *file_space_id, - hid_t plist_id, void **buf, void **req) { - S3VLDatasetObj *dset_obj = (S3VLDatasetObj*)(dset[0]); - Logger::log("------ Read dataset"); - Logger::log("------ Read dataset ", dset_obj->uri); - // string lower_range = getenv("LOWER_RANGE"); - // string upper_range = getenv("UPPER_RANGE"); - // cout << lower_range << " " << upper_range << endl; - auto read_start = std::chrono::high_resolution_clock::now(); - std::cout << "read :" << dset_obj->uri << std::endl; - if (dset_obj->read(*mem_space_id, *file_space_id, buf[0])) { - auto end = std::chrono::high_resolution_clock::now(); - std::chrono::duration duration = end - read_start; - std::cout << "VOL read time: " << duration.count() << " seconds" << std::endl; - Logger::log("read successfully"); - return ARRAYMORPH_SUCCESS; - } - Logger::log("read failed"); - return ARRAYMORPH_FAIL; -} -herr_t S3VLDatasetCallbacks::S3VL_dataset_write(size_t count, void **dset, hid_t *mem_type_id, hid_t *mem_space_id, hid_t *file_space_id, - hid_t plist_id, const void **buf, void **req) { - // TODO: update - S3VLDatasetObj *dset_obj = (S3VLDatasetObj*)(dset[0]); - Logger::log("------ Write dataset ", dset_obj->uri); - // vector mem_space = get_range_from_dataspace(mem_space_id); - // vector file_space = get_range_from_dataspace(file_space_id); - - if (dset_obj->write(*mem_space_id, *file_space_id, buf[0])) { - Logger::log("write successfully"); - return ARRAYMORPH_SUCCESS; - } - Logger::log("write failed"); - return ARRAYMORPH_FAIL; -} - -herr_t S3VLDatasetCallbacks::S3VL_dataset_get(void *dset, H5VL_dataset_get_args_t *args, hid_t dxpl_id, void **req) { - S3VLDatasetObj *dset_obj = (S3VLDatasetObj*)dset; - Logger::log("------ Get Space dataset: ", args->op_type); - if (args->op_type == H5VL_dataset_get_t::H5VL_DATASET_GET_DCPL) { - hid_t dcpl_id = H5Pcreate(H5P_DATASET_CREATE); - args->args.get_dcpl.dcpl_id = H5Pcopy(dcpl_id); - } - else if (args->op_type == H5VL_dataset_get_t::H5VL_DATASET_GET_SPACE) { - std::vector shape = dset_obj->shape; - // swap(shape[0], shape[1]); - hid_t space_id = H5Screate_simple(dset_obj->ndims, shape.data(), NULL); - args->args.get_space.space_id = space_id; - } - else if (args->op_type == H5VL_dataset_get_t::H5VL_DATASET_GET_TYPE) { - hid_t type_id = H5Tcopy(dset_obj->dtype); - args->args.get_type.type_id = type_id; - } - return ARRAYMORPH_SUCCESS; -} -herr_t S3VLDatasetCallbacks::S3VL_dataset_close(void *dset, hid_t dxpl_id, void **req) { - // TODO: update metadata - Logger::log("------ Close dataset"); - S3VLDatasetObj *dset_obj = (S3VLDatasetObj*)dset; - if (dset_obj->is_modified) - dset_obj->upload(); - - delete dset_obj; - return ARRAYMORPH_SUCCESS; -} - -void* S3VLDatasetCallbacks::S3VL_obj_open(void* obj, const H5VL_loc_params_t *loc_params, H5I_type_t *opened_type, hid_t dxpl_id, void **req) { - Logger::log("------ Open object"); - std::string access_name = loc_params->loc_data.loc_by_name.name; - // cout << "open by name: " << access_name << endl; - // cout << "open type: " << *opened_type << endl; - // if (access_name == "/") { - // *opened_type = H5I_type_t::H5I_GROUP; - // vector test = {"1", "2", "3"}; - // return new S3VLObj(test, 0); - // } - // else { - *opened_type = H5I_type_t::H5I_DATASET; - return S3VLDatasetCallbacks::S3VL_dataset_open(obj, loc_params, loc_params->loc_data.loc_by_name.name, dxpl_id, dxpl_id, req); - // } -} - -herr_t S3VLDatasetCallbacks::S3VL_obj_get(void *obj, const H5VL_loc_params_t *loc_params, H5VL_object_get_args_t *args, hid_t dxpl_id, void **req) { - Logger::log("------ Object get"); - // cout << args->op_type << endl; - // cout << loc_params->obj_type << " " << loc_params->type << " " << loc_params->loc_data.loc_by_name.name << endl;; - if (args->op_type == H5VL_object_get_t::H5VL_OBJECT_GET_INFO) - args->args.get_info.oinfo->type = H5O_type_t::H5O_TYPE_GROUP; - return ARRAYMORPH_SUCCESS; -} -void* S3VLDatasetCallbacks::S3VL_wrap_object(void *obj, H5I_type_t obj_type, void* wrap_ctx) { - Logger::log("------ Wrap object"); - return obj; -} - -void* S3VLDatasetCallbacks::S3VL_get_object(const void *obj) { - Logger::log("------ Get object"); - return NULL; -} - -herr_t S3VLDatasetCallbacks::S3VL_dataset_specific(void *obj, H5VL_dataset_specific_args_t *args, hid_t dxpl_id, void **req ) { - Logger::log("------ Specific dataset: ", args->op_type); - return ARRAYMORPH_SUCCESS; -} - diff --git a/arraymorph/src/s3vl_dataset_callbacks.h b/arraymorph/src/s3vl_dataset_callbacks.h deleted file mode 100644 index cc7a1a1..0000000 --- a/arraymorph/src/s3vl_dataset_callbacks.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef S3VL_DATASET_CALLBACKS - -#include -#include -#include "s3vl_dataset_obj.h" -#include "s3vl_file_callbacks.h" -#include "operators.h" - - - -class S3VLDatasetCallbacks{ -public: - static void *S3VL_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const char *name, hid_t lcpl_id, hid_t type_id, hid_t space_id, hid_t dcpl_id, hid_t dapl_id, hid_t dxpl_id, void **req); - static void *S3VL_dataset_open(void *obj, const H5VL_loc_params_t *loc_params, const char *name, hid_t dapl_id, hid_t dxpl_id, void **req); - static void *S3VL_obj_open(void* obj, const H5VL_loc_params_t *loc_params, H5I_type_t *opened_type, hid_t dxpl_id, void **req); - static herr_t S3VL_obj_get(void *obj, const H5VL_loc_params_t *loc_params, H5VL_object_get_args_t *args, hid_t dxpl_id, void **req); - static void *S3VL_wrap_object(void *obj, H5I_type_t obj_type, void* wrap_ctx); - static void *S3VL_get_object(const void *obj); - static herr_t S3VL_dataset_read(size_t count, void **dset, hid_t *mem_type_id, hid_t *mem_space_id, hid_t *file_space_id, hid_t plist_id, void **buf, void **req); - static herr_t S3VL_dataset_write(size_t count, void **dset, hid_t *mem_type_id, hid_t *mem_space_id, hid_t *file_space_id, hid_t plist_id, const void **buf, void **req); - static herr_t S3VL_dataset_get(void *dset, H5VL_dataset_get_args_t *args, hid_t dxpl_id, void **req); - static herr_t S3VL_dataset_close(void *dset, hid_t dxpl_id, void **req); - static herr_t S3VL_dataset_specific(void *obj, H5VL_dataset_specific_args_t *args, hid_t dxpl_id, void **req ); -}; -#define S3VL_DATASET_CALLBACKS -#endif \ No newline at end of file diff --git a/arraymorph/src/s3vl_dataset_obj.cc b/arraymorph/src/s3vl_dataset_obj.cc deleted file mode 100644 index dd8189f..0000000 --- a/arraymorph/src/s3vl_dataset_obj.cc +++ /dev/null @@ -1,493 +0,0 @@ -#include "s3vl_dataset_obj.h" -#include "utils.h" -#include "logger.h" -#include -#include -#include -#include -#include -#include -#include -#include - -uint64_t transfer_size; -int lambda_num, range_num, total_num; - -S3VLDatasetObj::S3VLDatasetObj(const std::string& name, const std::string& uri, hid_t dtype, int ndims, std::vector& shape, std::vector& chunk_shape, int chunk_num, const std::string& bucket_name, const CloudClient& client - ) : name(name), uri(uri), dtype(dtype), ndims(ndims), shape(shape), chunk_shape(chunk_shape), chunk_num(chunk_num), bucket_name(bucket_name), client(client) - { - this->data_size = H5Tget_size(this->dtype); - Logger::log("Datasize: ", this->data_size); - // data_size = 4; - num_per_dim.resize(ndims); - reduc_per_dim.resize(ndims); - reduc_per_dim[ndims - 1] = 1; - for (int i = 0; i < ndims; i++) - num_per_dim[i] = (shape[i] - 1) / chunk_shape[i] + 1; - for (int i = ndims - 2; i >= 0; i--) - reduc_per_dim[i] = reduc_per_dim[i + 1] * num_per_dim[i + 1]; - assert(reduc_per_dim[0] * num_per_dim[0] == chunk_num); - - element_per_chunk = 1; - for (auto &s: chunk_shape) - element_per_chunk *= s; -} - -std::vector S3VLDatasetObj::getChunkOffsets(int chunk_idx) { - std::vector idx_per_dim(ndims); - int tmp = chunk_idx; - - for (int i = 0; i < ndims; i++) { - idx_per_dim[i] = tmp / reduc_per_dim[i] * chunk_shape[i]; - tmp %= reduc_per_dim[i]; - } - - // for (int i = ndims - 1; i >= 0; i--) { - // idx_per_dim[i] = tmp / reduc_per_dim[i] * chunk_shape[i]; - // tmp %= reduc_per_dim[i]; - // } - return idx_per_dim; -} - -std::vector> S3VLDatasetObj::getChunkRanges(int chunk_idx) { - std::vector offsets_per_dim = getChunkOffsets(chunk_idx); - std::vector> re(ndims); - for (int i = 0; i < ndims; i++) - re[i] = {offsets_per_dim[i], offsets_per_dim[i] + chunk_shape[i] - 1}; - return re; -} - -std::vector> S3VLDatasetObj::selectionFromSpace(hid_t space_id) { - hsize_t start[ndims]; - hsize_t end[ndims]; - std::vector> ranges(ndims); - H5Sget_select_bounds(space_id, start, end); - for (int i = 0; i < ndims; i++) - ranges[i] = {start[i], end[i]}; - // if (ndims >= 2) - // swap(ranges[0], ranges[1]); - return ranges; -} - - -void processAzure(std::vector> &chunk_objs, const std::vector &azure_plans, - void* buf, BlobContainerClient *client, const std::string& bucket_name) { - std::vector> futures; - size_t azure_thread_num = THREAD_NUM; - futures.reserve(azure_thread_num); - size_t cur_batch_size = 0; - - for (int i = 0; i < azure_plans.size(); i++) { - const CPlan &p = azure_plans[i]; - if (cur_batch_size !=0 && (cur_batch_size + p.num_requests > azure_thread_num)) { - while (OperationTracker::getInstance().get() < cur_batch_size); - cur_batch_size = 0; - for (auto& fut : futures) fut.wait(); - futures.clear(); - } - for (auto &s : p.segments) { - std::vector> mapping; - for (auto it = s->mapping_start; it != s->mapping_end; ++it) - mapping.push_back({(*it)[0], (*it)[1], (*it)[2]}); - for (auto &m : mapping) - m[0] -= s->start_offset; - auto context = std::make_shared(buf, mapping); - futures.push_back(std::async(std::launch::async, Operators::AzureGetRange, client, chunk_objs[i]->uri, s->start_offset, s->end_offset, context)); - cur_batch_size++; - transfer_size += s->end_offset - s->start_offset + 1; - } - } - for (auto& fut : futures) fut.wait(); - futures.clear(); -} - -void processS3(std::vector> &chunk_objs, const std::vector &s3_plans, - void* buf, Aws::S3::S3Client *s3_client, const std::string& bucket_name) { - size_t s3_thread_num = THREAD_NUM; - size_t cur_batch_size = 0; - OperationTracker::getInstance().reset(); - for (int i = 0; i < s3_plans.size(); i++) { - const CPlan &p = s3_plans[i]; - if (cur_batch_size !=0 && (cur_batch_size + p.num_requests > s3_thread_num)) { - while (OperationTracker::getInstance().get() < cur_batch_size); - cur_batch_size = 0; - OperationTracker::getInstance().reset(); - } - for (auto &s : p.segments) { - std::vector> mapping; - for (auto it = s->mapping_start; it != s->mapping_end; ++it) - mapping.push_back({(*it)[0], (*it)[1], (*it)[2]}); - for (auto &m : mapping) - m[0] -= s->start_offset; - auto context = std::make_shared(buf, mapping); - Operators::S3GetByteRangeAsync(s3_client, bucket_name, chunk_objs[i]->uri, s->start_offset, s->end_offset, context); - cur_batch_size++; - transfer_size += s->end_offset - s->start_offset + 1; - } - } - while (OperationTracker::getInstance().get() < cur_batch_size); -} - -herr_t S3VLDatasetObj::read(hid_t mem_space_id, hid_t file_space_id, void* buf) { - - // string lambda_merge_path = getenv("AWS_LAMBDA_MERGE_ACCESS_POINT"); - std::vector> ranges; - if (file_space_id != H5S_ALL) { - ranges = selectionFromSpace(file_space_id); - } - else { - for (int i = 0;i < ndims; i++) - ranges.push_back({0, shape[i] - 1}); - } - - auto chunk_objs = generateChunks(ranges); - int num = chunk_objs.size(); - std::vector>> segments(num); - std::vector out_offsets; - hsize_t out_row_size; - std::vector>> global_mapping(num); - if (mem_space_id == H5S_ALL) { - // memspace == dataspace - out_offsets = calSerialOffsets(ranges, shape); - // out_row_size = ranges[0][1] - ranges[0][0] + 1; - out_row_size = ranges[ndims - 1][1] - ranges[ndims - 1][0] + 1; - } - else { - std::vector> out_ranges = selectionFromSpace(mem_space_id); - hsize_t dims_out[ndims]; - H5Sget_simple_extent_dims(mem_space_id, dims_out, NULL); - std::vector out_shape(dims_out, dims_out + ndims); - // if (ndims >= 2) - // swap(out_shape[0], out_shape[1]); - out_offsets = calSerialOffsets(out_ranges, out_shape); - // out_row_size = out_ranges[0][1] - out_ranges[0][0] + 1; - out_row_size = out_ranges[ndims - 1][1] - out_ranges[ndims - 1][0] + 1; - } - for (int i = 0; i < num; i++) { - // hsize_t input_row_size = chunk_objs[i]->ranges[0][1] - chunk_objs[i]->ranges[0][0] + 1; - hsize_t input_row_size = chunk_objs[i]->ranges[ndims - 1][1] - chunk_objs[i]->ranges[ndims - 1][0] + 1; - global_mapping[i] = mapHyperslab(chunk_objs[i]->local_offsets, chunk_objs[i]->global_offsets, out_offsets, - input_row_size, out_row_size, data_size); - segments[i] = generateSegments(global_mapping[i], chunk_objs[i]->size); - } - - transfer_size = 0; - lambda_num = 0; - range_num = 0; - - // cout << "start plan" << endl; - struct timeval start_opt, end_opt; - gettimeofday(&start_opt, NULL); - std::vector plans; - plans.reserve(chunk_objs.size()); - - for (int i = 0; i < chunk_objs.size(); i++) { - plans.emplace_back(i, GET, segments[i].size(), std::move(segments[i])); - } - gettimeofday(&end_opt, NULL); - double opt_t = (1000000 * ( end_opt.tv_sec - start_opt.tv_sec ) - + end_opt.tv_usec -start_opt.tv_usec) /1000000.0; - assert(plans.size() == num); -#ifdef PROFILE_ENABLE - std::cout << "query processer time: " << opt_t << std::endl; - std::cout << "chunk num:" << num << std::endl; -#endif - // cout << "get plans" << endl; -#ifdef LOG_ENABLE - Logger::log("------ Plans:"); - for (int i = 0; i < num; i++) { - Logger::log("chunk: ", chunk_objs[i]->uri); - Logger::log("plan: ", plans[i].qp); - } -#endif - assert(num == plans.size()); - - // std::sort(plans.begin(), plans.end(), [](const CPlan &a, const CPlan &b) { - // return a.qp < b.qp; - // }); - -#ifdef PROFILE_ENABLE - std::cout << "Plans: " << std::endl; - std::cout << "total num: " << plans.size() << std::endl; -#endif - if (SP == AZURE_BLOB) { - auto azure_client = std::get_if>(&client); - processAzure(chunk_objs, plans, buf, azure_client->get(), bucket_name); - } - else { - auto s3_client = std::get_if>(&client); - processS3(chunk_objs, plans, buf, s3_client->get(), bucket_name); - } -#ifdef PROFILE_ENABLE - std::cout << "transfer_size: " << transfer_size << std::endl; -#endif - return ARRAYMORPH_SUCCESS; -} - -herr_t S3VLDatasetObj::write(hid_t mem_space_id, hid_t file_space_id, const void* buf) { - std::vector> ranges; - if (file_space_id != H5S_ALL) { - ranges = selectionFromSpace(file_space_id); - } - else { - for (int i = 0;i < ndims; i++) - ranges.push_back({0, shape[i] - 1}); - } - - auto chunk_objs = generateChunks(ranges); - int num = chunk_objs.size(); - std::vector>> mappings(num); - std::vector source_offsets; - hsize_t source_row_size; - if (mem_space_id == H5S_ALL) { - // memspace == dataspace - source_offsets = calSerialOffsets(ranges, shape); - // source_row_size = ranges[0][1] - ranges[0][0] + 1; - source_row_size = ranges[ndims - 1][1] - ranges[ndims - 1][0] + 1; - } - else { - std::vector> source_ranges = selectionFromSpace(mem_space_id); - hsize_t dims_source[ndims]; - H5Sget_simple_extent_dims(mem_space_id, dims_source, NULL); - std::vector source_shape(dims_source, dims_source + ndims); - // if (ndims >= 2) - // swap(source_shape[0], source_shape[1]); - source_offsets = calSerialOffsets(source_ranges, source_shape); - // source_row_size = source_ranges[0][1] - source_ranges[0][0] + 1; - source_row_size = source_ranges[ndims - 1][1] - source_ranges[ndims - 1][0] + 1; - } - - for (int i = 0; i < num; i++) { - // hsize_t dest_row_size = chunk_objs[i]->ranges[0][1] - chunk_objs[i]->ranges[0][0] + 1; - hsize_t dest_row_size = chunk_objs[i]->ranges[ndims - 1][1] - chunk_objs[i]->ranges[ndims - 1][0] + 1; - mappings[i] = mapHyperslab(chunk_objs[i]->local_offsets, chunk_objs[i]->global_offsets, source_offsets, - dest_row_size, source_row_size, data_size); - } - - std::vector> futures; - futures.reserve(THREAD_NUM); - int cur_batch = 0; - - while(cur_batch < num) { - for (int idx = cur_batch; idx < std::min(num, cur_batch + THREAD_NUM); idx++) { - size_t length = chunk_objs[idx]->size; - auto upload_buf = std::shared_ptr(new char[length], std::default_delete()); - auto raw_buf = upload_buf.get(); -#ifdef DUMMY_WRITE - memset(raw_buf, 0, length); -#else - for (auto &m: mappings[idx]){ - memcpy(raw_buf + m[0], (char*)buf + m[1], m[2]); - } -#endif - if (SP == SPlan::AZURE_BLOB) { - auto azure_client = std::get_if>(&client); - futures.push_back(std::async(std::launch::async, Operators::AzurePut, azure_client->get(), chunk_objs[idx]->uri, upload_buf, length)); - } - else { - auto s3_client = std::get_if>(&client); - futures.push_back(std::async(std::launch::async, Operators::S3PutBuf, s3_client->get(), bucket_name, chunk_objs[idx]->uri, upload_buf, length)); - } - } - cur_batch = std::min(num, cur_batch + THREAD_NUM); - for (auto& fut : futures) fut.wait(); - futures.clear(); - } - return ARRAYMORPH_SUCCESS; -} - -std::vector> S3VLDatasetObj::generateChunks(std::vector> ranges) { - assert(ranges.size() == ndims); - std::vector accessed_chunks; - // get accessed chunks - std::vector> chunk_ranges(ndims); - for (int i = 0; i < ndims; i++) - chunk_ranges[i] = {ranges[i][0] / chunk_shape[i], ranges[i][1] / chunk_shape[i]}; - std::vector chunk_offsets = calSerialOffsets(chunk_ranges, num_per_dim); - // for (int i = 0; i <= chunk_ranges[0][1] - chunk_ranges[0][0]; i++) - for (int i = 0; i <= chunk_ranges[ndims - 1][1] - chunk_ranges[ndims - 1][0]; i++) - for (auto &n : chunk_offsets) - accessed_chunks.push_back(i + n); - - // iterate accessed chunks and generate queries - std::vector> chunk_objs; - chunk_objs.reserve(accessed_chunks.size()); - Logger::log("------ # of chunks ", accessed_chunks.size()); - for (auto &c : accessed_chunks) { - std::vector offsets = getChunkOffsets(c); - std::vector> local_ranges(ndims); - for (int i = 0; i < ndims; i++) { - hsize_t left = std::max(offsets[i], ranges[i][0]) - offsets[i]; - hsize_t right = std::min(offsets[i] + chunk_shape[i] - 1, ranges[i][1]) - offsets[i]; - local_ranges[i] = {left, right}; - } - std::string chunk_uri = uri + "/" + std::to_string(c); - // get output serial offsets for each row - std::vector> global_ranges(ndims); - std::vector result_shape(ndims); - for (int i = 0; i < ndims; i++) { - global_ranges[i] = {local_ranges[i][0] + offsets[i] - ranges[i][0], - local_ranges[i][1] + offsets[i] - ranges[i][0]}; - result_shape[i] = ranges[i][1] - ranges[i][0] + 1; - } - std::vector result_serial_offsets = calSerialOffsets(global_ranges, result_shape); - - - // S3VLChunkObj *chunk = new S3VLChunkObj(chunk_uri, dtype, local_ranges, chunk_shape, result_serial_offsets); - auto chunk = std::make_shared(chunk_uri, dtype, local_ranges, chunk_shape, result_serial_offsets); - // Logger::log("------ Generate Chunk"); - // Logger::log(chunk->to_string()); - // std::cout << chunk->to_string() << std::endl; - chunk_objs.push_back(chunk); - } - return chunk_objs; -} - - -// read/write - -void S3VLDatasetObj::upload() { - Logger::log("------ Upload metadata " + uri); - int length; - char* buffer = toBuffer(&length); - std::shared_ptr upload_buf(buffer, std::default_delete()); - std::string meta_name = uri + "/meta"; - Result re{std::vector(buffer, buffer + length)}; - - if (SP == SPlan::S3) { - auto s3_client = std::get_if>(&client); - - if (!s3_client || ! s3_client->get()){ - std::cerr << "S3 client not initialized correctly!" << std::endl; - return ; - } - Operators::S3Put(s3_client->get(), bucket_name, meta_name, re); - } - else { - auto azure_client = std::get_if>(&client); - if (!azure_client || !azure_client->get()) { - std::cerr << "Azure client not initialized correctly!" << std::endl; - return; - } - Operators::AzurePut(azure_client->get(), meta_name, upload_buf, length); - } -} - -S3VLDatasetObj* S3VLDatasetObj::getDatasetObj(const CloudClient& client, const std::string& bucket_name, const std::string& uri) { - Result re; - if (SP == SPlan::S3) { - auto s3_client = std::get_if>(&client); - - if (!s3_client || ! s3_client->get()){ - std::cerr << "S3 client not initialized correctly!" << std::endl; - return nullptr; - } - re = Operators::S3Get(s3_client->get(), bucket_name, uri); - - } - else { - auto azure_client = std::get_if>(&client); - if (!azure_client || !azure_client->get()) { - std::cerr << "Azure client not initialized correctly!" << std::endl; - return nullptr; - } - re = Operators::AzureGet(azure_client->get(), uri); - } - - if (re.data.empty()) { - std::cerr << "Didn't get metadata!" << std::endl; - return nullptr; - - } - return S3VLDatasetObj::getDatasetObj(client, bucket_name, re.data); -} - -char* S3VLDatasetObj::toBuffer(int *length) { - int size = 8 + name.size() + uri.size() + sizeof(hid_t) + 4 + 2 * ndims * sizeof(hsize_t) + 4; - *length = size; - char * buffer = new char[size]; - int c = 0; - int name_length = name.size(); - memcpy(buffer, &name_length, 4); - c += 4; - memcpy(buffer + c, name.c_str(), name.size()); - c += name.size(); - int uri_length = uri.size(); - memcpy(buffer + c, &uri_length, 4); - c += 4; - memcpy(buffer + c, uri.c_str(), uri.size()); - c += uri.size(); - memcpy(buffer + c, &dtype, sizeof(hid_t)); - c += sizeof(hid_t); - memcpy(buffer + c, &ndims, 4); - c += 4; - memcpy(buffer + c, shape.data(), sizeof(hsize_t) * ndims); - c += sizeof(hsize_t) * ndims; - memcpy(buffer + c, chunk_shape.data(), sizeof(hsize_t) * ndims); - c += sizeof(hsize_t) * ndims; - memcpy(buffer + c, &chunk_num, 4); - c += 4; - return buffer; -} - -S3VLDatasetObj* S3VLDatasetObj::getDatasetObj(const CloudClient& client, const std::string& bucket_name, std::vector& buffer) { - std::string name, uri; - int ndims, chunk_num; - hid_t dtype; - int c = 0; - - int name_length; - memcpy(&name_length, buffer.data() + c, sizeof(int)); - c += sizeof(int); - - name.assign(buffer.data() + c, name_length); // name - c += name_length; - - int uri_length; - memcpy(&uri_length, buffer.data() + c, sizeof(int)); - c += sizeof(int); - - uri.assign(buffer.data() + c, uri_length); // uri - c += uri_length; - - memcpy(&dtype, buffer.data() + c, sizeof(hid_t)); // dtype - c += sizeof(hid_t); - - memcpy(&ndims, buffer.data() + c, sizeof(int)); //ndims - c += sizeof(int); - - std::vector shape(ndims); // shape - memcpy(shape.data(), (hsize_t*)(buffer.data() + c), sizeof(hsize_t) * ndims); - c += sizeof(hsize_t) * ndims; - - std::vector chunk_shape(ndims); // chunk_shape - memcpy(chunk_shape.data(), (hsize_t*)(buffer.data() + c), sizeof(hsize_t) * ndims); - c += sizeof(hsize_t) * ndims; - - memcpy(&chunk_num, buffer.data() + c, sizeof(int)); // chunk num - return new S3VLDatasetObj(name, uri, dtype, ndims, shape, chunk_shape, chunk_num, bucket_name, client); -} - -std::string S3VLDatasetObj::to_string() { - std::stringstream ss; - ss << name << " " << uri << std::endl; - ss << dtype << " " << ndims << std::endl; - ss << chunk_num << " " << element_per_chunk << std::endl;; - for (int i = 0; i < ndims; i++) { - ss << shape[i] << " "; - } - ss << std::endl; - for (int i = 0; i < ndims; i++) { - ss << chunk_shape[i] << " "; - } - ss << std::endl; - for (int i = 0; i < ndims; i++) { - ss << num_per_dim[i] << " "; - } - ss << std::endl; - for (int i = 0; i < ndims; i++) { - ss << reduc_per_dim[i] << " "; - } - ss << std::endl; - return ss.str(); -} diff --git a/arraymorph/src/s3vl_dataset_obj.h b/arraymorph/src/s3vl_dataset_obj.h deleted file mode 100644 index 52dfca2..0000000 --- a/arraymorph/src/s3vl_dataset_obj.h +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef S3VL_DATASET_OBJ -#define S3VL_DATASET_OBJ -#include -#include -#include -#include -#include -#include -#include "constants.h" -#include -#include "operators.h" -#include "s3vl_chunk_obj.h" - -class S3VLDatasetObj -{ -public: - S3VLDatasetObj(const std::string& name, const std::string& uri, hid_t dtype, int ndims, std::vector& shape, std::vector& chunk_shape, - int chunk_num, const std::string& bucket_name, const CloudClient& client); - ~S3VLDatasetObj(){}; - - static S3VLDatasetObj* getDatasetObj(const CloudClient& client, const std::string& bucket_name, const std::string& uri); - static S3VLDatasetObj* getDatasetObj(const CloudClient& client, const std::string& bucket_name, std::vector &buffer); - char* toBuffer(int *length); - std::vector> generateChunks(std::vector> ranges); - std::string to_string(); - std::vector getChunkOffsets(int chunk_idx); - std::vector> getChunkRanges(int chunk_idx); - std::vector> selectionFromSpace(hid_t space_id); - // QPlan getQueryPlan(FileFormat format, vector> ranges); - - void upload(); - herr_t write(hid_t mem_space_id, hid_t file_space_id, const void* buf); - herr_t read(hid_t mem_space_id, hid_t file_space_id, void* buf); - - const std::string name; - const std::string uri; - hid_t dtype; - const int ndims; - const std::vector shape; - const std::vector chunk_shape; - const int chunk_num; - const std::string bucket_name; - - hsize_t data_size; - std::vector num_per_dim; - std::vector reduc_per_dim; - hsize_t element_per_chunk; - bool is_modified{false}; - - const CloudClient& client; -}; - - -#endif \ No newline at end of file diff --git a/arraymorph/src/s3vl_file_callbacks.cc b/arraymorph/src/s3vl_file_callbacks.cc deleted file mode 100644 index adc039c..0000000 --- a/arraymorph/src/s3vl_file_callbacks.cc +++ /dev/null @@ -1,86 +0,0 @@ -#include "s3vl_file_callbacks.h" -#include -#include "logger.h" -#include "constants.h" -#include "operators.h" - -CloudClient get_client() { - Logger::log("Init cloud clients"); - // AWS connection - CloudClient client; - if (SP == SPlan::S3) { - std::string access_key = getenv("AWS_ACCESS_KEY_ID"); - std::string secret_key = getenv("AWS_SECRET_ACCESS_KEY"); - Aws::Auth::AWSCredentials cred(access_key, secret_key); - std::unique_ptr s3ClientConfig = std::make_unique(); - s3ClientConfig->scheme = Scheme::HTTPS; - s3ClientConfig->maxConnections = s3Connections; - s3ClientConfig->requestTimeoutMs = requestTimeoutMs; - s3ClientConfig->connectTimeoutMs = connectTimeoutMs; - s3ClientConfig->retryStrategy = std::make_shared(retries); -#ifdef POOLEXECUTOR - s3ClientConfig->executor = Aws::MakeShared("test", poolSize); -#endif - Logger::log("------ Create Client config: maxConnections=", s3ClientConfig->maxConnections); - client = std::make_unique(cred, std::move(*s3ClientConfig), Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, true); - s3ClientConfig.reset(); - } - // Azure connection - else { - std::string azure_connection_string = getenv("AZURE_STORAGE_CONNECTION_STRING"); - - Azure::Core::Http::Policies::RetryOptions retryOptions; - retryOptions.MaxRetries = retries; - retryOptions.RetryDelay = std::chrono::milliseconds(200); - retryOptions.MaxRetryDelay = std::chrono::seconds(2); - - Azure::Storage::Blobs::BlobClientOptions clientOptions; - clientOptions.Retry = retryOptions; - client = std::make_unique(BlobContainerClient::CreateFromConnectionString(azure_connection_string, BUCKET_NAME, clientOptions)); - } - return client; -} - -void* S3VLFileCallbacks::S3VL_file_create(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, hid_t dxpl_id, void **req) { - S3VLFileObj *ret_obj = new S3VLFileObj(); - std::string path(name); - if (path.rfind("./", 0) == 0) { - path = path.substr(2); - } - ret_obj->name = path; - Logger::log("------ Create File:", path); - if (std::holds_alternative(global_cloud_client)) { - global_cloud_client = get_client(); - } - return (void*)ret_obj; -} -void* S3VLFileCallbacks::S3VL_file_open(const char *name, unsigned flags, hid_t fapl_id, hid_t dxpl_id, void **req) { - S3VLFileObj *ret_obj = new S3VLFileObj(); - std::string path(name); - Logger::log("------ Open File:", path); - if (path.rfind("./", 0) == 0) { - path = path.substr(2); - } - ret_obj->name = path; - if (std::holds_alternative(global_cloud_client)) { - global_cloud_client = get_client(); - } - return (void*)ret_obj; - -} -herr_t S3VLFileCallbacks::S3VL_file_close(void *file, hid_t dxpl_id, void **req) { - S3VLFileObj *file_obj = (S3VLFileObj*)file; - Logger::log("------ Close File: ", file_obj->name); - delete file_obj; - return ARRAYMORPH_SUCCESS; -} - -herr_t S3VLFileCallbacks::S3VL_file_get(void *file, H5VL_file_get_args_t *args, hid_t dxpl_id, void **req) { - Logger::log("------ Get File"); - if (args->op_type == H5VL_file_get_t::H5VL_FILE_GET_FCPL) { - hid_t fcpl_id = H5Pcreate(H5P_FILE_CREATE); - args->args.get_fcpl.fcpl_id = H5Pcopy(fcpl_id); - // args->args.get_fcpl.fcpl_id = 1; - } - return ARRAYMORPH_SUCCESS; -} diff --git a/arraymorph/src/s3vl_file_callbacks.h b/arraymorph/src/s3vl_file_callbacks.h deleted file mode 100644 index 4671c6e..0000000 --- a/arraymorph/src/s3vl_file_callbacks.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef S3VL_FILE_CALLBACKS -#include -#include - - -typedef struct S3VLFileObj { - std::string name; -} S3VLFileObj; - -class S3VLFileCallbacks{ -public: - static void *S3VL_file_create(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, hid_t dxpl_id, void **req); - static void *S3VL_file_open(const char *name, unsigned flags, hid_t fapl_id, hid_t dxpl_id, void **req); - static herr_t S3VL_file_get(void *file, H5VL_file_get_args_t *args, hid_t dxpl_id, void **req); - static herr_t S3VL_file_close(void *file, hid_t dxpl_id, void **req); -}; -#define S3VL_FILE_CALLBACKS -#endif \ No newline at end of file diff --git a/arraymorph/src/s3vl_group_callbacks.cc b/arraymorph/src/s3vl_group_callbacks.cc deleted file mode 100644 index 6a00c24..0000000 --- a/arraymorph/src/s3vl_group_callbacks.cc +++ /dev/null @@ -1,99 +0,0 @@ -#include "s3vl_group_callbacks.h" -#include "s3vl_dataset_callbacks.h" -#include -#include "operators.h" -#include "logger.h" -#include - -herr_t S3VLGroupCallbacks::S3VLgroup_get(void * obj, H5VL_group_get_args_t *args, hid_t dxpl_id, void **req) { - Logger::log("------ Get Group"); - // cout << "is NULL: " << (obj == NULL) << endl; - // if (args->args.get_info.loc_params.obj_type == H5I_type_t::H5I_GROUP ) { - // S3VLObj *get_obj = (S3VLObj*)obj; - // cout << "obj info: " << get_obj->idx << " " << get_obj->keys[1] << endl; - // } - // cout << "get type: " << args->op_type << endl; - // cout << "get para: " << args->args.get_info.loc_params.obj_type << " " << args->args.get_info.loc_params.type << endl; - // if (args->op_type == H5VL_group_get_t::H5VL_GROUP_GET_INFO) { - // args->args.get_info.loc_params.type = H5VL_loc_type_t::H5VL_OBJECT_BY_IDX; - // auto ginfo = args->args.get_info.ginfo; - // ginfo->nlinks = 3; - // ginfo->mounted = true; - // } - // else if (args->op_type == H5VL_group_get_t::H5VL_GROUP_GET_GCPL) { - // hid_t gcpl_id = H5Pcreate(H5P_GROUP_CREATE); - // args->args.get_gcpl.gcpl_id = H5Pcopy(gcpl_id); - // } - return ARRAYMORPH_SUCCESS; -} - -void* S3VLGroupCallbacks::S3VLgroup_open(void *obj, const H5VL_loc_params_t *loc_params, const char *name, hid_t gapl_id, hid_t dxpl_id, -void **req) { - /* - TODO: This function is a simple way to list the dataset names using h5ls - */ - Logger::log("------ Open Group"); - Logger::log("name: ", name); -// if (strcmp(name, "/") == 0) { -// S3VLFileObj *f_obj = (S3VLFileObj*)obj; -// std::string keys_url = f_obj->name + "/keys"; -// std::string bucket_name = getenv("BUCKET_NAME"); -// Result keys; -// // aws connection -// if (SP == SPlan::S3) { -// std::string access_key = getenv("AWS_ACCESS_KEY_ID"); -// std::string secret_key = getenv("AWS_SECRET_ACCESS_KEY"); -// Aws::Auth::AWSCredentials cred(access_key, secret_key); -// if (s3Configured == false) { -// s3Configured = true; -// s3ClientConfig = new Aws::Client::ClientConfiguration(); -// s3ClientConfig->scheme = Scheme::HTTP; -// s3ClientConfig->maxConnections = s3Connections; -// s3ClientConfig->requestTimeoutMs = requestTimeoutMs; -// s3ClientConfig->connectTimeoutMs = connectTimeoutMs; -// #ifdef POOLEXECUTOR -// s3ClientConfig->executor = Aws::MakeShared("test", poolSize); -// #endif -// Logger::log("------ Create Client config: maxConnections=", s3ClientConfig->maxConnections); -// } -// std::unique_ptr client = std::make_unique(cred, *s3ClientConfig); -// keys = Operators::S3Get(client.get(), bucket_name, keys_url); -// // list dsets -// std::cout << "datasets:" << std::endl; -// std::cout.write(keys.data, keys.length); -// } -// } - return (void*)obj; -} - -herr_t S3VLGroupCallbacks::S3VLgroup_close(void *grp, hid_t dxpl_id, void **req) { - Logger::log("------ Close Group"); - S3VLFileObj *f_obj = (S3VLFileObj*)grp; - // free(f_obj); - return ARRAYMORPH_SUCCESS; -} - -herr_t S3VLGroupCallbacks::S3VLlink_get(void * obj, const H5VL_loc_params_t *loc_params, H5VL_link_get_args_t *args, hid_t dxpl_id, void **req) { - Logger::log("------ Get Link"); - // cout << "get type: " << args->op_type << endl; - // cout << "get para: " << args->args.get_info.loc_params.obj_type << " " << args->args.get_info.loc_params.type << endl; - return ARRAYMORPH_SUCCESS; -} - - -herr_t S3VLGroupCallbacks::S3VLlink_specific(void * obj, const H5VL_loc_params_t *loc_params, H5VL_link_specific_args_t *args, hid_t dxpl_id, void **req) { - Logger::log("------ Specific Link"); - // S3VLGroupObj *get_obj = (S3VLGroupObj*)obj; - // cout << "obj info: " << get_obj->keys.size() << " " << get_obj->keys[1] << endl; - - // cout << "get type: " << args->op_type << endl; - // // cout << "get para: " << args->args.get_info.loc_params.obj_type << " " << args->args.get_info.loc_params.type << endl; - // if (args->op_type == H5VL_link_specific_t::H5VL_LINK_ITER) { - // auto it_args = args->args.iterate; - // cout << it_args.recursive << endl; - // cout << it_args.op << endl; - // // cout << *(it_args.idx_p) << endl; - // // cout << loc_params->loc_data.loc_by_name.name << endl; - // } - return ARRAYMORPH_SUCCESS; -} \ No newline at end of file diff --git a/arraymorph/src/s3vl_group_callbacks.h b/arraymorph/src/s3vl_group_callbacks.h deleted file mode 100644 index 13b8a91..0000000 --- a/arraymorph/src/s3vl_group_callbacks.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef S3VL_GROUP_CALLBACKS -#include -#include -#include - -typedef struct S3VLGroupObj { - std::vector keys; -} S3VLGroupObj; - -class S3VLGroupCallbacks{ -public: - static void* S3VLgroup_open(void *obj, const H5VL_loc_params_t *loc_params, const char *name, hid_t gapl_id, hid_t dxpl_id, void **req); - static herr_t S3VLgroup_close(void *grp, hid_t dxpl_id, void **req); - static herr_t S3VLgroup_get(void * obj, H5VL_group_get_args_t *args, hid_t dxpl_id, void **req); - static herr_t S3VLlink_get(void * obj, const H5VL_loc_params_t *loc_params, H5VL_link_get_args_t *args, hid_t dxpl_id, void **req); - static herr_t S3VLlink_specific(void * obj, const H5VL_loc_params_t *loc_params, H5VL_link_specific_args_t *args, hid_t dxpl_id, void **req); -}; -#define S3VL_GROUP_CALLBACKS -#endif \ No newline at end of file diff --git a/arraymorph/src/s3vl_initialize.h b/arraymorph/src/s3vl_initialize.h deleted file mode 100644 index 6482735..0000000 --- a/arraymorph/src/s3vl_initialize.h +++ /dev/null @@ -1,66 +0,0 @@ -#ifndef S3VL_INITIALIZE -#define S3VL_INITIALIZE - -#include -#include -#include "constants.h" -#include "s3vl_vol_connector.h" -#include -#include -#include "logger.h" -class S3VLINITIALIZE -{ -public: - static herr_t s3VL_initialize_init(hid_t vipl_id); - static herr_t s3VL_initialize_close(); -}; - -std::optional getEnv(const char* var) { - const char* val = std::getenv(var); - return val ? std::optional(val) : std::nullopt; -} - -inline herr_t S3VLINITIALIZE::s3VL_initialize_init(hid_t vipl_id) { - Aws::SDKOptions options; - options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Off; - Aws::InitAPI(options); - Logger::log("------ Init VOL"); - std::optional platform = getEnv("STORAGE_PLATFORM"); - if (platform.has_value()) { - std::string platform_str = platform.value(); - if (platform_str == "S3") - Logger::log("------ Using S3"); - else if (platform_str == "Azure") { - Logger::log("------ Using Azure"); - SP = SPlan::AZURE_BLOB; - } - else { - Logger::log("------ Unsupported platform"); - return ARRAYMORPH_FAIL; - } - } - else { - Logger::log("------ Using default platform S3"); - } - std::optional bucket_name = getEnv("BUCKET_NAME"); - if (bucket_name.has_value()) { - BUCKET_NAME = bucket_name.value(); - Logger::log("------ Using bucket", BUCKET_NAME); - } - else { - Logger::log("------ Bucekt not set"); - return ARRAYMORPH_FAIL; - } - return S3_VOL_CONNECTOR_VALUE; -} - -inline herr_t S3VLINITIALIZE::s3VL_initialize_close() { - Logger::log("------ Close VOL"); - // Aws::ShutdownAPI(opt); - return ARRAYMORPH_SUCCESS; -} - - - - -#endif diff --git a/arraymorph/src/s3vl_vol_connector.cc b/arraymorph/src/s3vl_vol_connector.cc deleted file mode 100644 index 35f47a5..0000000 --- a/arraymorph/src/s3vl_vol_connector.cc +++ /dev/null @@ -1,149 +0,0 @@ -/* This connector's header */ -#include "s3vl_vol_connector.h" -#include "s3vl_file_callbacks.h" -#include "s3vl_dataset_callbacks.h" -#include "s3vl_group_callbacks.h" -#include "s3vl_initialize.h" -#include "constants.h" - -#include -#include -#include -#include -// #include "operators.h" -// #include "metadata.h" - -/* Introscpect */ - -static herr_t S3_introspect_opt_query(void *obj, H5VL_subclass_t cls, int op_type, uint64_t *flags); - -static herr_t S3_get_conn_cls(void *obj, H5VL_get_conn_lvl_t lvl, const struct H5VL_class_t **conn_cls); - - -/* The VOL class struct */ -static const H5VL_class_t template_class_g = { - 3, /* VOL class struct version */ - S3_VOL_CONNECTOR_VALUE, /* value */ - S3_VOL_CONNECTOR_NAME, /* name */ - 1, /* version */ - 0, /* capability flags */ - S3VLINITIALIZE::s3VL_initialize_init, /* initialize */ - S3VLINITIALIZE::s3VL_initialize_close, /* terminate */ - { /* info_cls */ - (size_t)0, /* size */ - NULL, /* copy */ - NULL, /* compare */ - NULL, /* free */ - NULL, /* to_str */ - NULL, /* from_str */ - }, - { /* wrap_cls */ - S3VLDatasetCallbacks::S3VL_get_object, /* get_object */ - NULL, /* get_wrap_ctx */ - S3VLDatasetCallbacks::S3VL_wrap_object, /* wrap_object */ - NULL, /* unwrap_object */ - NULL, /* free_wrap_ctx */ - }, - { /* attribute_cls */ - NULL, /* create */ - NULL, /* open */ - NULL, /* read */ - NULL, /* write */ - NULL, /* get */ - NULL, /* specific */ - NULL, /* optional */ - NULL /* close */ - }, - { /* dataset_cls */ - S3VLDatasetCallbacks::S3VL_dataset_create, /* create */ - S3VLDatasetCallbacks::S3VL_dataset_open, /* open */ - S3VLDatasetCallbacks::S3VL_dataset_read, /* read */ - S3VLDatasetCallbacks::S3VL_dataset_write, /* write */ - S3VLDatasetCallbacks::S3VL_dataset_get, /* get */ - S3VLDatasetCallbacks::S3VL_dataset_specific, /* specific */ - NULL, /* optional */ - S3VLDatasetCallbacks::S3VL_dataset_close /* close */ - }, - { /* datatype_cls */ - NULL, /* commit */ - NULL, /* open */ - NULL, /* get_size */ - NULL, /* specific */ - NULL, /* optional */ - NULL /* close */ - }, - { /* file_cls */ - S3VLFileCallbacks::S3VL_file_create, /* create */ - S3VLFileCallbacks::S3VL_file_open, /* open */ - S3VLFileCallbacks::S3VL_file_get, /* get */ - NULL, /* specific */ - NULL, /* optional */ - S3VLFileCallbacks::S3VL_file_close /* close */ - }, - { /* group_cls */ - NULL, /* create */ - S3VLGroupCallbacks::S3VLgroup_open, /* open */ - S3VLGroupCallbacks::S3VLgroup_get, /* get */ - // NULL, - NULL, /* specific */ - NULL, /* optional */ - S3VLGroupCallbacks::S3VLgroup_close // NULL /* close */ - }, - { /* link_cls */ - NULL, /* create */ - NULL, /* copy */ - NULL, /* move */ - S3VLGroupCallbacks::S3VLlink_get, /* get */ - NULL, //S3VLGroupCallbacks::S3VLlink_specific, /* specific */ - NULL /* optional */ - }, - { /* object_cls */ - S3VLDatasetCallbacks::S3VL_obj_open, /* open */ - NULL, /* copy */ - S3VLDatasetCallbacks::S3VL_obj_get, /* get */ - NULL, /* specific */ - NULL /* optional */ - }, - { /* introscpect_cls */ - S3_get_conn_cls, /* get_conn_cls */ - NULL, /* get_cap_flags */ - S3_introspect_opt_query /* opt_query */ - }, - { /* request_cls */ - NULL, /* wait */ - NULL, /* notify */ - NULL, /* cancel */ - NULL, /* specific */ - NULL, /* optional */ - NULL /* free */ - }, - { /* blob_cls */ - NULL, /* put */ - NULL, /* get */ - NULL, /* specific */ - NULL /* optional */ - }, - { /* token_cls */ - NULL, /* cmp */ - NULL, /* to_str */ - NULL /* from_str */ - }, - NULL /* optional */ -}; - -/* These two functions are necessary to load this plugin using - * the HDF5 library. - */ - -static herr_t S3_introspect_opt_query(void *obj, H5VL_subclass_t cls, int op_type, uint64_t *flags) { - return ARRAYMORPH_SUCCESS; -} - -static herr_t S3_get_conn_cls(void *obj, H5VL_get_conn_lvl_t lvl, const struct H5VL_class_t **conn_cls){ - *conn_cls = new H5VL_class_t; - return ARRAYMORPH_SUCCESS; -} - -H5PL_type_t H5PLget_plugin_type(void) {return H5PL_TYPE_VOL;} -const void *H5PLget_plugin_info(void) {return &template_class_g;} - diff --git a/arraymorph/src/utils.cc b/arraymorph/src/utils.cc deleted file mode 100644 index 106b532..0000000 --- a/arraymorph/src/utils.cc +++ /dev/null @@ -1,124 +0,0 @@ -#include "utils.h" -#include -#include -#include "logger.h" - -Segment::Segment(hsize_t start_offset, hsize_t end_offset, hsize_t required_data_size, const std::list>::iterator &mapping_start, const std::list>::iterator &mapping_end, hsize_t mapping_size) : start_offset(start_offset), end_offset(end_offset), required_data_size(required_data_size), mapping_start(mapping_start), mapping_end(mapping_end), mapping_size(mapping_size) { - assert(mapping_start != mapping_end); -} - -Segment::Segment(hsize_t start_offset, hsize_t end_offset, const std::list>::iterator &mapping_start, const std::list>::iterator &mapping_end, hsize_t mapping_size): start_offset(start_offset), end_offset(end_offset), mapping_start(mapping_start), mapping_end(mapping_end), mapping_size(mapping_size) { - assert(mapping_start != mapping_end); - required_data_size = 0; - for (auto it = mapping_start; it != mapping_end; ++it) - required_data_size += (*it)[2]; -} - -Segment::Segment(const std::list>::iterator &mapping_start, const std::list>::iterator &mapping_end, hsize_t mapping_size): mapping_start(mapping_start), mapping_end(mapping_end), mapping_size(mapping_size) { - assert(mapping_start != mapping_end); - start_offset = (*mapping_start)[0]; - auto last_mapping = std::prev(mapping_end); - end_offset = (*last_mapping)[0] + (*last_mapping)[2] - 1; - required_data_size = 0; - for (auto it = mapping_start; it != mapping_end; ++it) - required_data_size += (*it)[2]; -} - -std::string Segment::to_string() { - std::stringstream ss; - ss << "start_offset: " << start_offset << " " << "end_offset: " << end_offset << std::endl; - for (auto it = mapping_start; it != mapping_end; ++it) - ss << (*it)[0] << " " << (*it)[1] << " " << (*it)[2] << std::endl; - return ss.str(); -} - - -std::vector reduceAdd(std::vector& a, std::vector& b) { - std::vector re; - re.reserve(a.size() * b.size()); - for (auto &i : a) - for (auto &j: b) - re.push_back(i + j); - return re; -} -// calculate each row's start position in serial order -std::vector calSerialOffsets(std::vector> ranges, std::vector shape) { - assert(ranges.size() == shape.size()); - int dims = ranges.size(); - std::vector offsets_per_dim(dims); - std::vector> offsets(dims - 1); - - offsets_per_dim[dims - 1] = 1; - for (int i = dims - 2; i >= 0; i--) - offsets_per_dim[i] = offsets_per_dim[i + 1] * shape[i + 1]; - - for (int i = 0; i < dims - 1; i++) - for (int j = ranges[i][0]; j <= ranges[i][1]; j++) - offsets[i].push_back(j * offsets_per_dim[i]); - std::vector re = {0}; - for (int i = 0; i < dims - 1; i++) - re = reduceAdd(re, offsets[i]); - for (auto &o : re) - o += ranges[dims - 1][0]; - - return re; -} - -// mapping offsets between two hyperslab -std::list> mapHyperslab(std::vector &local, std::vector &global, std::vector &out, - hsize_t &input_rsize, hsize_t &out_rsize, hsize_t &data_size) { - std::list> mapping; - - for (int i = 0; i < local.size(); i++) { - hsize_t idx = global[i] / out_rsize; - hsize_t start_offset = out[idx] + global[i] % out_rsize; - hsize_t remaining = out[idx] + out_rsize - start_offset; - if (remaining >= input_rsize) { - mapping.push_back({local[i], start_offset, input_rsize}); - } - else { - mapping.push_back({local[i], start_offset, remaining}); - hsize_t cur = remaining; - int n = (input_rsize - remaining) / out_rsize; - for (int j = 1; j <= n; j++) { - mapping.push_back({local[i] + cur, out[idx + j], out_rsize}); - cur += out_rsize; - } - if ((input_rsize - remaining) % out_rsize > 0) { - mapping.push_back({local[i] + cur, out[idx + n + 1], (input_rsize - remaining) % out_rsize}); - } - - } - } - for (auto &m : mapping) - for (auto &o : m) - o *= data_size; - return mapping; - -} - - -std::vector> generateSegments(std::list>& mapping, hsize_t chunk_size) { - std::vector> segments; - size_t n = mapping.size(); - if (n == 0) - return segments; - segments.emplace_back(std::make_unique(0, chunk_size - 1, mapping.begin(), mapping.end(), mapping.size())); -#ifdef LOG_ENABLE - for (auto const & s : segments) - Logger::log(s->to_string()); -#endif - return segments; -} - - -std::string createQuery(hsize_t data_size, int ndims, const std::vector& shape, std::vector>& ranges) { - std::stringstream ss; - ss << "-" << data_size << "-" << ndims << "-"; - for (auto &s: shape) - ss << s << "-"; - for (auto &r: ranges) - ss << r[0] << "-" << r[1] << "-"; - std::string re = ss.str(); - return re.substr(0, re.size() - 1); -} diff --git a/arraymorph/src/utils.h b/arraymorph/src/utils.h deleted file mode 100644 index 996648d..0000000 --- a/arraymorph/src/utils.h +++ /dev/null @@ -1,41 +0,0 @@ -#ifndef UTILS -#define UTILS -#include -#include -#include -#include "constants.h" -#include -#include -#include - -class Segment { -public: - Segment(hsize_t start_offset, hsize_t end_offset, hsize_t required_data_size, const std::list>::iterator &mapping_start, const std::list>::iterator &mapping_end, hsize_t mapping_size); - Segment(hsize_t start_offset, hsize_t end_offset, const std::list>::iterator &mapping_start, const std::list>::iterator &mapping_end, hsize_t mapping_size); - Segment(const std::list>::iterator &mapping_start, const std::list>::iterator &mapping_end, hsize_t mapping_size); - std::string to_string(); - hsize_t start_offset; // inclusive - hsize_t end_offset; // inclusive - hsize_t required_data_size; - hsize_t mapping_size; - std::list>::iterator mapping_start; // inclusive - std::list>::iterator mapping_end; // exclusive -}; - -std::vector reduceAdd(std::vector& a, std::vector& b); - -// calculate each row's start position in serial order -std::vector calSerialOffsets(std::vector> ranges, std::vector shape); - -// mapping offsets between two hyperslab -std::list> mapHyperslab(std::vector &local, std::vector &global, std::vector &out, - hsize_t &input_rsize, hsize_t &out_rsize, hsize_t &data_size); - -// create Json query for Lambda function -std::string createQuery(hsize_t data_size, int ndims, const std::vector& shape, std::vector>& ranges); - -// generate list of segments for chunk-level query - -std::vector> generateSegments(std::list>& mapping, hsize_t chunk_size); - -#endif \ No newline at end of file diff --git a/env-example.txt b/env-example.txt new file mode 100644 index 0000000..f300413 --- /dev/null +++ b/env-example.txt @@ -0,0 +1,13 @@ +STORAGE_PLATFORM=S3 +BUCKET_NAME=playgrounds +AWS_ENDPOINT_URL_S3=garage-s3-endpoint +AWS_S3_ADDRESSING_STYLE="path" +AWS_ACCESS_KEY_ID=garage-access-key +AWS_SECRET_ACCESS_KEY=secret-access-key +AWS_REGION=garage +AWS_USE_PATH_STYLE=true +AWS_USE_TLS=true +AWS_SIGNED_PAYLOADS=true +HDF5_VOL_CONNECTOR=arraymorph +HDF5_PLUGIN_PATH=path-to-libarraymorph + From 4d0a0dfb6e9ea7dafd209acbe5945edb366321ef Mon Sep 17 00:00:00 2001 From: Carlos Guzman Date: Fri, 13 Feb 2026 16:00:33 -0600 Subject: [PATCH 02/16] Initiated a python project with uv --- main.py | 6 +++++ pyproject.toml | 9 ++++++++ uv.lock | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 main.py create mode 100644 pyproject.toml create mode 100644 uv.lock diff --git a/main.py b/main.py new file mode 100644 index 0000000..aa77d2f --- /dev/null +++ b/main.py @@ -0,0 +1,6 @@ +def main(): + print("Hello from arraymorph!") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6708d7d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,9 @@ +[project] +name = "arraymorph" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.14" +dependencies = [ + "h5py>=3.15.1", +] diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..f61dd0c --- /dev/null +++ b/uv.lock @@ -0,0 +1,62 @@ +version = 1 +revision = 3 +requires-python = ">=3.14" + +[[package]] +name = "arraymorph" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "h5py" }, +] + +[package.metadata] +requires-dist = [{ name = "h5py", specifier = ">=3.15.1" }] + +[[package]] +name = "h5py" +version = "3.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/6a/0d79de0b025aa85dc8864de8e97659c94cf3d23148394a954dc5ca52f8c8/h5py-3.15.1.tar.gz", hash = "sha256:c86e3ed45c4473564de55aa83b6fc9e5ead86578773dfbd93047380042e26b69", size = 426236, upload-time = "2025-10-16T10:35:27.404Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/2c/926eba1514e4d2e47d0e9eb16c784e717d8b066398ccfca9b283917b1bfb/h5py-3.15.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5f4fb0567eb8517c3ecd6b3c02c4f4e9da220c8932604960fd04e24ee1254763", size = 3380368, upload-time = "2025-10-16T10:35:03.117Z" }, + { url = "https://files.pythonhosted.org/packages/65/4b/d715ed454d3baa5f6ae1d30b7eca4c7a1c1084f6a2edead9e801a1541d62/h5py-3.15.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:954e480433e82d3872503104f9b285d369048c3a788b2b1a00e53d1c47c98dd2", size = 2833793, upload-time = "2025-10-16T10:35:05.623Z" }, + { url = "https://files.pythonhosted.org/packages/ef/d4/ef386c28e4579314610a8bffebbee3b69295b0237bc967340b7c653c6c10/h5py-3.15.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fd125c131889ebbef0849f4a0e29cf363b48aba42f228d08b4079913b576bb3a", size = 4903199, upload-time = "2025-10-16T10:35:08.972Z" }, + { url = "https://files.pythonhosted.org/packages/33/5d/65c619e195e0b5e54ea5a95c1bb600c8ff8715e0d09676e4cce56d89f492/h5py-3.15.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28a20e1a4082a479b3d7db2169f3a5034af010b90842e75ebbf2e9e49eb4183e", size = 5097224, upload-time = "2025-10-16T10:35:12.808Z" }, + { url = "https://files.pythonhosted.org/packages/30/30/5273218400bf2da01609e1292f562c94b461fcb73c7a9e27fdadd43abc0a/h5py-3.15.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa8df5267f545b4946df8ca0d93d23382191018e4cda2deda4c2cedf9a010e13", size = 4551207, upload-time = "2025-10-16T10:35:16.24Z" }, + { url = "https://files.pythonhosted.org/packages/d3/39/a7ef948ddf4d1c556b0b2b9559534777bccc318543b3f5a1efdf6b556c9c/h5py-3.15.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:99d374a21f7321a4c6ab327c4ab23bd925ad69821aeb53a1e75dd809d19f67fa", size = 5025426, upload-time = "2025-10-16T10:35:19.831Z" }, + { url = "https://files.pythonhosted.org/packages/b6/d8/7368679b8df6925b8415f9dcc9ab1dab01ddc384d2b2c24aac9191bd9ceb/h5py-3.15.1-cp314-cp314-win_amd64.whl", hash = "sha256:9c73d1d7cdb97d5b17ae385153472ce118bed607e43be11e9a9deefaa54e0734", size = 2865704, upload-time = "2025-10-16T10:35:22.658Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b7/4a806f85d62c20157e62e58e03b27513dc9c55499768530acc4f4c5ce4be/h5py-3.15.1-cp314-cp314-win_arm64.whl", hash = "sha256:a6d8c5a05a76aca9a494b4c53ce8a9c29023b7f64f625c6ce1841e92a362ccdf", size = 2465544, upload-time = "2025-10-16T10:35:25.695Z" }, +] + +[[package]] +name = "numpy" +version = "2.4.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/fd/0005efbd0af48e55eb3c7208af93f2862d4b1a56cd78e84309a2d959208d/numpy-2.4.2.tar.gz", hash = "sha256:659a6107e31a83c4e33f763942275fd278b21d095094044eb35569e86a21ddae", size = 20723651, upload-time = "2026-01-31T23:13:10.135Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/88/b7df6050bf18fdcfb7046286c6535cabbdd2064a3440fca3f069d319c16e/numpy-2.4.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:444be170853f1f9d528428eceb55f12918e4fda5d8805480f36a002f1415e09b", size = 16663092, upload-time = "2026-01-31T23:12:04.521Z" }, + { url = "https://files.pythonhosted.org/packages/25/7a/1fee4329abc705a469a4afe6e69b1ef7e915117747886327104a8493a955/numpy-2.4.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d1240d50adff70c2a88217698ca844723068533f3f5c5fa6ee2e3220e3bdb000", size = 14698770, upload-time = "2026-01-31T23:12:06.96Z" }, + { url = "https://files.pythonhosted.org/packages/fb/0b/f9e49ba6c923678ad5bc38181c08ac5e53b7a5754dbca8e581aa1a56b1ff/numpy-2.4.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:7cdde6de52fb6664b00b056341265441192d1291c130e99183ec0d4b110ff8b1", size = 5208562, upload-time = "2026-01-31T23:12:09.632Z" }, + { url = "https://files.pythonhosted.org/packages/7d/12/d7de8f6f53f9bb76997e5e4c069eda2051e3fe134e9181671c4391677bb2/numpy-2.4.2-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:cda077c2e5b780200b6b3e09d0b42205a3d1c68f30c6dceb90401c13bff8fe74", size = 6543710, upload-time = "2026-01-31T23:12:11.969Z" }, + { url = "https://files.pythonhosted.org/packages/09/63/c66418c2e0268a31a4cf8a8b512685748200f8e8e8ec6c507ce14e773529/numpy-2.4.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d30291931c915b2ab5717c2974bb95ee891a1cf22ebc16a8006bd59cd210d40a", size = 15677205, upload-time = "2026-01-31T23:12:14.33Z" }, + { url = "https://files.pythonhosted.org/packages/5d/6c/7f237821c9642fb2a04d2f1e88b4295677144ca93285fd76eff3bcba858d/numpy-2.4.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bba37bc29d4d85761deed3954a1bc62be7cf462b9510b51d367b769a8c8df325", size = 16611738, upload-time = "2026-01-31T23:12:16.525Z" }, + { url = "https://files.pythonhosted.org/packages/c2/a7/39c4cdda9f019b609b5c473899d87abff092fc908cfe4d1ecb2fcff453b0/numpy-2.4.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b2f0073ed0868db1dcd86e052d37279eef185b9c8db5bf61f30f46adac63c909", size = 17028888, upload-time = "2026-01-31T23:12:19.306Z" }, + { url = "https://files.pythonhosted.org/packages/da/b3/e84bb64bdfea967cc10950d71090ec2d84b49bc691df0025dddb7c26e8e3/numpy-2.4.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7f54844851cdb630ceb623dcec4db3240d1ac13d4990532446761baede94996a", size = 18339556, upload-time = "2026-01-31T23:12:21.816Z" }, + { url = "https://files.pythonhosted.org/packages/88/f5/954a291bc1192a27081706862ac62bb5920fbecfbaa302f64682aa90beed/numpy-2.4.2-cp314-cp314-win32.whl", hash = "sha256:12e26134a0331d8dbd9351620f037ec470b7c75929cb8a1537f6bfe411152a1a", size = 6006899, upload-time = "2026-01-31T23:12:24.14Z" }, + { url = "https://files.pythonhosted.org/packages/05/cb/eff72a91b2efdd1bc98b3b8759f6a1654aa87612fc86e3d87d6fe4f948c4/numpy-2.4.2-cp314-cp314-win_amd64.whl", hash = "sha256:068cdb2d0d644cdb45670810894f6a0600797a69c05f1ac478e8d31670b8ee75", size = 12443072, upload-time = "2026-01-31T23:12:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/37/75/62726948db36a56428fce4ba80a115716dc4fad6a3a4352487f8bb950966/numpy-2.4.2-cp314-cp314-win_arm64.whl", hash = "sha256:6ed0be1ee58eef41231a5c943d7d1375f093142702d5723ca2eb07db9b934b05", size = 10494886, upload-time = "2026-01-31T23:12:28.488Z" }, + { url = "https://files.pythonhosted.org/packages/36/2f/ee93744f1e0661dc267e4b21940870cabfae187c092e1433b77b09b50ac4/numpy-2.4.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:98f16a80e917003a12c0580f97b5f875853ebc33e2eaa4bccfc8201ac6869308", size = 14818567, upload-time = "2026-01-31T23:12:30.709Z" }, + { url = "https://files.pythonhosted.org/packages/a7/24/6535212add7d76ff938d8bdc654f53f88d35cddedf807a599e180dcb8e66/numpy-2.4.2-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:20abd069b9cda45874498b245c8015b18ace6de8546bf50dfa8cea1696ed06ef", size = 5328372, upload-time = "2026-01-31T23:12:32.962Z" }, + { url = "https://files.pythonhosted.org/packages/5e/9d/c48f0a035725f925634bf6b8994253b43f2047f6778a54147d7e213bc5a7/numpy-2.4.2-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:e98c97502435b53741540a5717a6749ac2ada901056c7db951d33e11c885cc7d", size = 6649306, upload-time = "2026-01-31T23:12:34.797Z" }, + { url = "https://files.pythonhosted.org/packages/81/05/7c73a9574cd4a53a25907bad38b59ac83919c0ddc8234ec157f344d57d9a/numpy-2.4.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:da6cad4e82cb893db4b69105c604d805e0c3ce11501a55b5e9f9083b47d2ffe8", size = 15722394, upload-time = "2026-01-31T23:12:36.565Z" }, + { url = "https://files.pythonhosted.org/packages/35/fa/4de10089f21fc7d18442c4a767ab156b25c2a6eaf187c0db6d9ecdaeb43f/numpy-2.4.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e4424677ce4b47fe73c8b5556d876571f7c6945d264201180db2dc34f676ab5", size = 16653343, upload-time = "2026-01-31T23:12:39.188Z" }, + { url = "https://files.pythonhosted.org/packages/b8/f9/d33e4ffc857f3763a57aa85650f2e82486832d7492280ac21ba9efda80da/numpy-2.4.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2b8f157c8a6f20eb657e240f8985cc135598b2b46985c5bccbde7616dc9c6b1e", size = 17078045, upload-time = "2026-01-31T23:12:42.041Z" }, + { url = "https://files.pythonhosted.org/packages/c8/b8/54bdb43b6225badbea6389fa038c4ef868c44f5890f95dd530a218706da3/numpy-2.4.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5daf6f3914a733336dab21a05cdec343144600e964d2fcdabaac0c0269874b2a", size = 18380024, upload-time = "2026-01-31T23:12:44.331Z" }, + { url = "https://files.pythonhosted.org/packages/a5/55/6e1a61ded7af8df04016d81b5b02daa59f2ea9252ee0397cb9f631efe9e5/numpy-2.4.2-cp314-cp314t-win32.whl", hash = "sha256:8c50dd1fc8826f5b26a5ee4d77ca55d88a895f4e4819c7ecc2a9f5905047a443", size = 6153937, upload-time = "2026-01-31T23:12:47.229Z" }, + { url = "https://files.pythonhosted.org/packages/45/aa/fa6118d1ed6d776b0983f3ceac9b1a5558e80df9365b1c3aa6d42bf9eee4/numpy-2.4.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fcf92bee92742edd401ba41135185866f7026c502617f422eb432cfeca4fe236", size = 12631844, upload-time = "2026-01-31T23:12:48.997Z" }, + { url = "https://files.pythonhosted.org/packages/32/0a/2ec5deea6dcd158f254a7b372fb09cfba5719419c8d66343bab35237b3fb/numpy-2.4.2-cp314-cp314t-win_arm64.whl", hash = "sha256:1f92f53998a17265194018d1cc321b2e96e900ca52d54c7c77837b71b9465181", size = 10565379, upload-time = "2026-01-31T23:12:51.345Z" }, +] From 69f0cdac9d0a5c9a110e35a302b71133433bc825 Mon Sep 17 00:00:00 2001 From: Carlos Guzman Date: Wed, 25 Feb 2026 10:46:27 -0600 Subject: [PATCH 03/16] chore: add .gitignore, untrack conda channel artifact Excludes build artifacts, Conan generated files, Python bytecode, distribution packages, secrets, and editor configs. Removes accidentally tracked local conda package from arraymorph_channel/. --- .actrc | 5 + ...uild-lib_array_morph-and-pypi-package.yaml | 99 +++++++++++ .gitignore | 94 +++++----- arraymorph/CMakeLists.txt | 68 -------- {arraymorph => lib}/.clangd | 0 lib/CMakeLists.txt | 102 +++++++++++ {arraymorph => lib}/CMakeUserPresets.json | 0 {arraymorph => lib}/README.md | 0 {arraymorph => lib}/activate.sh | 0 {arraymorph => lib}/conanfile.py | 0 {arraymorph => lib}/deactivate.sh | 0 .../include/arraymorph/core/constants.h | 0 .../include/arraymorph/core/logger.h | 0 .../include/arraymorph/core/operators.h | 0 .../include/arraymorph/core/utils.h | 0 .../include/arraymorph/s3vl/chunk_obj.h | 0 .../arraymorph/s3vl/dataset_callbacks.h | 0 .../include/arraymorph/s3vl/dataset_obj.h | 0 .../include/arraymorph/s3vl/file_callbacks.h | 0 .../include/arraymorph/s3vl/group_callbacks.h | 0 .../include/arraymorph/s3vl/initialize.h | 0 .../include/arraymorph/s3vl/vol_connector.h | 0 {arraymorph => lib}/justfile | 3 +- {arraymorph => lib}/meta.yaml | 0 .../scripts/extract_perspective.py | 0 {arraymorph => lib}/src/CMakeLists.txt | 22 +-- {arraymorph => lib}/src/core/constants.cc | 0 {arraymorph => lib}/src/core/operators.cc | 0 {arraymorph => lib}/src/core/utils.cc | 0 {arraymorph => lib}/src/s3vl/chunk_obj.cc | 0 .../src/s3vl/dataset_callbacks.cc | 0 {arraymorph => lib}/src/s3vl/dataset_obj.cc | 0 .../src/s3vl/file_callbacks.cc | 25 +-- .../src/s3vl/group_callbacks.cc | 0 {arraymorph => lib}/src/s3vl/vol_connector.cc | 0 src/arraymorph/__init__.py | 161 ++++++++++++++++++ 36 files changed, 448 insertions(+), 131 deletions(-) create mode 100644 .actrc create mode 100644 .github/workflows/build-lib_array_morph-and-pypi-package.yaml delete mode 100644 arraymorph/CMakeLists.txt rename {arraymorph => lib}/.clangd (100%) create mode 100644 lib/CMakeLists.txt rename {arraymorph => lib}/CMakeUserPresets.json (100%) rename {arraymorph => lib}/README.md (100%) rename {arraymorph => lib}/activate.sh (100%) rename {arraymorph => lib}/conanfile.py (100%) rename {arraymorph => lib}/deactivate.sh (100%) rename {arraymorph => lib}/include/arraymorph/core/constants.h (100%) rename {arraymorph => lib}/include/arraymorph/core/logger.h (100%) rename {arraymorph => lib}/include/arraymorph/core/operators.h (100%) rename {arraymorph => lib}/include/arraymorph/core/utils.h (100%) rename {arraymorph => lib}/include/arraymorph/s3vl/chunk_obj.h (100%) rename {arraymorph => lib}/include/arraymorph/s3vl/dataset_callbacks.h (100%) rename {arraymorph => lib}/include/arraymorph/s3vl/dataset_obj.h (100%) rename {arraymorph => lib}/include/arraymorph/s3vl/file_callbacks.h (100%) rename {arraymorph => lib}/include/arraymorph/s3vl/group_callbacks.h (100%) rename {arraymorph => lib}/include/arraymorph/s3vl/initialize.h (100%) rename {arraymorph => lib}/include/arraymorph/s3vl/vol_connector.h (100%) rename {arraymorph => lib}/justfile (80%) rename {arraymorph => lib}/meta.yaml (100%) rename {arraymorph => lib}/scripts/extract_perspective.py (100%) rename {arraymorph => lib}/src/CMakeLists.txt (67%) rename {arraymorph => lib}/src/core/constants.cc (100%) rename {arraymorph => lib}/src/core/operators.cc (100%) rename {arraymorph => lib}/src/core/utils.cc (100%) rename {arraymorph => lib}/src/s3vl/chunk_obj.cc (100%) rename {arraymorph => lib}/src/s3vl/dataset_callbacks.cc (100%) rename {arraymorph => lib}/src/s3vl/dataset_obj.cc (100%) rename {arraymorph => lib}/src/s3vl/file_callbacks.cc (87%) rename {arraymorph => lib}/src/s3vl/group_callbacks.cc (100%) rename {arraymorph => lib}/src/s3vl/vol_connector.cc (100%) create mode 100644 src/arraymorph/__init__.py diff --git a/.actrc b/.actrc new file mode 100644 index 0000000..04e905b --- /dev/null +++ b/.actrc @@ -0,0 +1,5 @@ +-P ubuntu-latest=catthehacker/ubuntu:act-latest +-P ubuntu-24.04-arm=catthehacker/ubuntu:act-latest +-P macos-latest=catthehacker/ubuntu:act-latest +-P macos-13=catthehacker/ubuntu:act-latest +--container-architecture linux/amd64 diff --git a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml new file mode 100644 index 0000000..6676732 --- /dev/null +++ b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml @@ -0,0 +1,99 @@ +name: Build, Test, and Publish + +on: + pull_request: + branches: [main] + release: + types: [published] + workflow_dispatch: # Manual trigger — builds and tests only, does not publish to PyPI + +jobs: + build_wheels: + name: Build (${{ matrix.os }} / ${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + # Linux x86_64 + - os: linux + arch: x86_64 + runner: ubuntu-latest + # Linux ARM64 + - os: linux + arch: aarch64 + runner: ubuntu-24.04-arm + # macOS Intel + - os: macos + arch: x86_64 + runner: macos-13 + # macOS Apple Silicon + - os: macos + arch: arm64 + runner: macos-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up uv + uses: astral-sh/setup-uv@v5 + + - name: Install cibuildwheel + run: uv tool install cibuildwheel + + - name: Build wheels + run: cibuildwheel --output-dir wheelhouse + env: + CIBW_ARCHS: ${{ matrix.arch }} + + - name: Upload wheel artifact + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.os }}-${{ matrix.arch }} + path: wheelhouse/*.whl + retention-days: 7 + + # Extract the native lib from the wheel and attach to the GitHub release + - name: Extract native library from wheel + if: github.event_name == 'release' + shell: bash + run: | + wheel_file=$(ls wheelhouse/*.whl | head -1) + mkdir -p extracted_lib + unzip -j "$wheel_file" "arraymorph/lib/lib_array_morph*" -d extracted_lib/ + lib_file=$(ls extracted_lib/lib_array_morph*) + ext="${lib_file##*.}" + cp "$lib_file" "lib_array_morph-${{ matrix.os }}-${{ matrix.arch }}.$ext" + echo "LIB_ARTIFACT=lib_array_morph-${{ matrix.os }}-${{ matrix.arch }}.$ext" >> $GITHUB_ENV + + - name: Attach native library to GitHub release + if: github.event_name == 'release' + uses: softprops/action-gh-release@v2 + with: + files: ${{ env.LIB_ARTIFACT }} + + publish: + name: Publish to PyPI + needs: build_wheels + runs-on: ubuntu-latest + if: github.event_name == 'release' + environment: + name: pypi + url: https://pypi.org/p/arraymorph + permissions: + id-token: write # Required for OIDC trusted publishing + + steps: + - name: Set up uv + uses: astral-sh/setup-uv@v5 + + - name: Download all wheels + uses: actions/download-artifact@v4 + with: + pattern: wheels-* + merge-multiple: true + path: dist/ + + - name: Publish to PyPI + run: uv publish dist/* diff --git a/.gitignore b/.gitignore index c648f1b..e739efe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,56 +1,74 @@ -# These are some examples of commonly ignored file patterns. -# You should customize this list as applicable to your project. -# Learn more about .gitignore: -# https://www.atlassian.com/git/tutorials/saving-changes/gitignore - -# Node artifact files -node_modules/ -dist/ - -# Compiled Java class files -*.class - -# Compiled Python bytecode +# Python *.py[cod] +*.egg-info/ +*.egg +.venv/ +.test-venv/ +__pycache__/ +dist/ +wheelhouse/ +*.whl -# Log files -*.log +# C++ / Build +build/ +lib/build/ +*.so +*.dylib +*.dll +*.a +*.o +CMakeCache.txt +CMakeFiles/ +cmake_install.cmake +compile_commands.json +CMakeUserPresets.json -# Package files -*.jar +# Conan generated +lib/activate.sh +lib/deactivate.sh -# Maven -target/ +# Distribution / packaging dist/ +arraymorph_channel/ -# JetBrains IDE -.idea/ +# HDF5 +*.h5 +*.hdf5 +.h5py_hdf5_info -# Unit test reports -TEST*.xml +# Environment / secrets +.env +.env.* +*.pem +*.key +*.cert +*.crt +.netrc +.secrets +.aws/ +credentials +.conan2/ +conan.conf -# Generated by MacOS +# Lock files (library, not app) +uv.lock + +# macOS .DS_Store -# Generated by Windows -Thumbs.db +# JetBrains +.idea/ + +# VSCode +.vscode/ -# Applications -*.app -*.exe -*.war +# Logs +*.log -# Large media files +# Large media *.mp4 *.tiff *.avi *.flv *.mov *.wmv - -# vscode -.vscode -# build -arraymorph/build - -.venv diff --git a/arraymorph/CMakeLists.txt b/arraymorph/CMakeLists.txt deleted file mode 100644 index 5759c01..0000000 --- a/arraymorph/CMakeLists.txt +++ /dev/null @@ -1,68 +0,0 @@ -cmake_minimum_required(VERSION 3.20) - -project(ArrayMorph - VERSION 0.1.0 - LANGUAGES C CXX -) - -set(CMAKE_CXX_STANDARD 20) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_POSITION_INDEPENDENT_CODE ON) # b_staticpic=true -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - -# Build type and optimization -if(NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Release) -endif() - -if(CMAKE_BUILD_TYPE STREQUAL "Release") - set(CMAKE_CXX_FLAGS_RELEASE "-O3") -endif() - -# Include directory -include_directories(${CMAKE_SOURCE_DIR}/include) - -# --- Find External Dependencies --- - -# AWS SDK -find_package(AWSSDK REQUIRED COMPONENTS core s3) - -# Azure SDK -find_package(AzureSDK CONFIG REQUIRED) - -# cURL and OpenSSL -find_package(CURL REQUIRED) -find_package(OpenSSL REQUIRED) - -if(NOT DEFINED ENV{H5PY_HDF5_DIR}) - message(FATAL_ERROR "H5PY_HDF5_DIR environment variable not set") -endif() - -# --- HDF5 with h5py workaround --- -set(H5PY_LIB_DIR "$ENV{H5PY_HDF5_DIR}") - -# Find HDF5 for headers (via Conan or system) -find_package(HDF5 REQUIRED COMPONENTS C) - -# Create custom HDF5 dependency: Conan headers + h5py binary -add_library(hdf5_custom INTERFACE) -target_include_directories(hdf5_custom INTERFACE ${HDF5_INCLUDE_DIRS}) -target_link_directories(hdf5_custom INTERFACE ${H5PY_LIB_DIR}) -target_link_libraries(hdf5_custom INTERFACE hdf5.310) - -# Print confirmation -message(STATUS "HDF5 found: ${HDF5_VERSION}") - -# Collect all dependencies into a list -set(ALL_DEPS - AWS::aws-sdk-cpp-core - AWS::aws-sdk-cpp-s3 - Azure::azure-storage-blobs - hdf5_custom - OpenSSL::SSL - OpenSSL::Crypto - CURL::libcurl -) - -# Enter src directory -add_subdirectory(src) diff --git a/arraymorph/.clangd b/lib/.clangd similarity index 100% rename from arraymorph/.clangd rename to lib/.clangd diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt new file mode 100644 index 0000000..897085b --- /dev/null +++ b/lib/CMakeLists.txt @@ -0,0 +1,102 @@ +cmake_minimum_required(VERSION 3.20) + +project(ArrayMorph + VERSION 0.1.0 + LANGUAGES C CXX +) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +# Build type and optimization +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release) +endif() + +if(CMAKE_BUILD_TYPE STREQUAL "Release") + set(CMAKE_CXX_FLAGS_RELEASE "-O3") +endif() + +# Include directory +include_directories(${CMAKE_SOURCE_DIR}/include) + +# --- Find External Dependencies --- + +# AWS SDK +find_package(AWSSDK REQUIRED COMPONENTS core s3) + +# Azure SDK +find_package(AzureSDK CONFIG REQUIRED) + +# cURL and OpenSSL +find_package(CURL REQUIRED) +find_package(OpenSSL REQUIRED) + +# --- HDF5: Conan headers + h5py runtime binary --- +# +# ArrayMorph is a VOL plugin that gets dlopen'd by HDF5 at runtime. +# We MUST link against the same HDF5 that h5py ships to avoid +# duplicate symbol conflicts. Conan provides headers, h5py provides +# the shared library. + +if(NOT DEFINED ENV{H5PY_HDF5_DIR}) + message(FATAL_ERROR + "H5PY_HDF5_DIR not set. Run:\n" + " export H5PY_HDF5_DIR=$(python -c \"" + "import h5py, os; " + "d=os.path.dirname(h5py.__file__); " + "print(os.path.join(d,'.dylibs') if os.path.exists(os.path.join(d,'.dylibs')) " + "else os.path.join(os.path.dirname(d),'h5py.libs'))\")" + ) +endif() + +set(H5PY_LIB_DIR "$ENV{H5PY_HDF5_DIR}") + +# Find HDF5 headers (via Conan) +find_package(HDF5 REQUIRED COMPONENTS C) + +# Locate the actual shared library in h5py's bundled directory +file(GLOB _h5py_hdf5_libs + "${H5PY_LIB_DIR}/libhdf5*.dylib" # macOS + "${H5PY_LIB_DIR}/libhdf5*.so*" # Linux +) + +if(NOT _h5py_hdf5_libs) + # List what's actually in the directory for debugging + file(GLOB _h5py_dir_contents "${H5PY_LIB_DIR}/*") + message(FATAL_ERROR + "No HDF5 shared library found in H5PY_HDF5_DIR=${H5PY_LIB_DIR}\n" + "Directory contents: ${_h5py_dir_contents}\n" + "Expected libhdf5*.dylib (macOS) or libhdf5*.so* (Linux)" + ) +endif() + +# Pick the first match +list(GET _h5py_hdf5_libs 0 _h5py_hdf5_lib) + +# Create imported target: Conan headers + h5py binary +add_library(hdf5_custom SHARED IMPORTED) +set_target_properties(hdf5_custom PROPERTIES + IMPORTED_LOCATION "${_h5py_hdf5_lib}" + INTERFACE_INCLUDE_DIRECTORIES "${HDF5_INCLUDE_DIRS}" +) + +message(STATUS "HDF5 version: ${HDF5_VERSION}") +message(STATUS "HDF5 headers: ${HDF5_INCLUDE_DIRS}") +message(STATUS "HDF5 binary: ${_h5py_hdf5_lib}") + +# Collect all dependencies into a list +set(ALL_DEPS + AWS::aws-sdk-cpp-core + AWS::aws-sdk-cpp-s3 + Azure::azure-storage-blobs + hdf5_custom + OpenSSL::SSL + OpenSSL::Crypto + CURL::libcurl +) + +# Enter src directory +add_subdirectory(src) diff --git a/arraymorph/CMakeUserPresets.json b/lib/CMakeUserPresets.json similarity index 100% rename from arraymorph/CMakeUserPresets.json rename to lib/CMakeUserPresets.json diff --git a/arraymorph/README.md b/lib/README.md similarity index 100% rename from arraymorph/README.md rename to lib/README.md diff --git a/arraymorph/activate.sh b/lib/activate.sh similarity index 100% rename from arraymorph/activate.sh rename to lib/activate.sh diff --git a/arraymorph/conanfile.py b/lib/conanfile.py similarity index 100% rename from arraymorph/conanfile.py rename to lib/conanfile.py diff --git a/arraymorph/deactivate.sh b/lib/deactivate.sh similarity index 100% rename from arraymorph/deactivate.sh rename to lib/deactivate.sh diff --git a/arraymorph/include/arraymorph/core/constants.h b/lib/include/arraymorph/core/constants.h similarity index 100% rename from arraymorph/include/arraymorph/core/constants.h rename to lib/include/arraymorph/core/constants.h diff --git a/arraymorph/include/arraymorph/core/logger.h b/lib/include/arraymorph/core/logger.h similarity index 100% rename from arraymorph/include/arraymorph/core/logger.h rename to lib/include/arraymorph/core/logger.h diff --git a/arraymorph/include/arraymorph/core/operators.h b/lib/include/arraymorph/core/operators.h similarity index 100% rename from arraymorph/include/arraymorph/core/operators.h rename to lib/include/arraymorph/core/operators.h diff --git a/arraymorph/include/arraymorph/core/utils.h b/lib/include/arraymorph/core/utils.h similarity index 100% rename from arraymorph/include/arraymorph/core/utils.h rename to lib/include/arraymorph/core/utils.h diff --git a/arraymorph/include/arraymorph/s3vl/chunk_obj.h b/lib/include/arraymorph/s3vl/chunk_obj.h similarity index 100% rename from arraymorph/include/arraymorph/s3vl/chunk_obj.h rename to lib/include/arraymorph/s3vl/chunk_obj.h diff --git a/arraymorph/include/arraymorph/s3vl/dataset_callbacks.h b/lib/include/arraymorph/s3vl/dataset_callbacks.h similarity index 100% rename from arraymorph/include/arraymorph/s3vl/dataset_callbacks.h rename to lib/include/arraymorph/s3vl/dataset_callbacks.h diff --git a/arraymorph/include/arraymorph/s3vl/dataset_obj.h b/lib/include/arraymorph/s3vl/dataset_obj.h similarity index 100% rename from arraymorph/include/arraymorph/s3vl/dataset_obj.h rename to lib/include/arraymorph/s3vl/dataset_obj.h diff --git a/arraymorph/include/arraymorph/s3vl/file_callbacks.h b/lib/include/arraymorph/s3vl/file_callbacks.h similarity index 100% rename from arraymorph/include/arraymorph/s3vl/file_callbacks.h rename to lib/include/arraymorph/s3vl/file_callbacks.h diff --git a/arraymorph/include/arraymorph/s3vl/group_callbacks.h b/lib/include/arraymorph/s3vl/group_callbacks.h similarity index 100% rename from arraymorph/include/arraymorph/s3vl/group_callbacks.h rename to lib/include/arraymorph/s3vl/group_callbacks.h diff --git a/arraymorph/include/arraymorph/s3vl/initialize.h b/lib/include/arraymorph/s3vl/initialize.h similarity index 100% rename from arraymorph/include/arraymorph/s3vl/initialize.h rename to lib/include/arraymorph/s3vl/initialize.h diff --git a/arraymorph/include/arraymorph/s3vl/vol_connector.h b/lib/include/arraymorph/s3vl/vol_connector.h similarity index 100% rename from arraymorph/include/arraymorph/s3vl/vol_connector.h rename to lib/include/arraymorph/s3vl/vol_connector.h diff --git a/arraymorph/justfile b/lib/justfile similarity index 80% rename from arraymorph/justfile rename to lib/justfile index f65169c..5349ac8 100644 --- a/arraymorph/justfile +++ b/lib/justfile @@ -6,10 +6,9 @@ COMPILER_STD := "gnu20" # Set this to the path where H5Py installs HDF5 binary. On macOS its .dylib, Linux .so, Windows .dll -H5PY_HDF5_DIR := "../.venv/lib/python3.14/site-packages/h5py/.dylibs" +H5PY_HDF5_DIR := `python -c "import h5py,os;d=os.path.dirname(h5py.__file__);print(os.path.join(d,'.dylibs') if os.path.exists(os.path.join(d,'.dylibs')) else os.path.join(os.path.dirname(d),'h5py.libs'))"` # Settings - set dotenv-load := true set export := true diff --git a/arraymorph/meta.yaml b/lib/meta.yaml similarity index 100% rename from arraymorph/meta.yaml rename to lib/meta.yaml diff --git a/arraymorph/scripts/extract_perspective.py b/lib/scripts/extract_perspective.py similarity index 100% rename from arraymorph/scripts/extract_perspective.py rename to lib/scripts/extract_perspective.py diff --git a/arraymorph/src/CMakeLists.txt b/lib/src/CMakeLists.txt similarity index 67% rename from arraymorph/src/CMakeLists.txt rename to lib/src/CMakeLists.txt index af80f5a..32dbbad 100644 --- a/arraymorph/src/CMakeLists.txt +++ b/lib/src/CMakeLists.txt @@ -1,5 +1,4 @@ # 1. Include directories -# In CMake, you typically set these per-target rather than globally set(PROJECT_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../include @@ -13,42 +12,43 @@ target_link_libraries(utils PUBLIC ${HDF5_LIBRARIES}) add_library(constants STATIC core/constants.cc) target_include_directories(constants PUBLIC ${PROJECT_INCLUDE_DIRS}) -# all_deps would be your list of dependencies (AWS, Azure, SSL, etc.) add_library(operators STATIC core/operators.cc) target_include_directories(operators PUBLIC ${PROJECT_INCLUDE_DIRS}) -target_link_libraries(operators PUBLIC constants ${ALL_DEPS}) +target_link_libraries(operators PRIVATE constants ${ALL_DEPS}) add_library(chunk_obj STATIC s3vl/chunk_obj.cc) target_include_directories(chunk_obj PUBLIC ${PROJECT_INCLUDE_DIRS}) -target_link_libraries(chunk_obj PUBLIC operators utils ${ALL_DEPS}) +target_link_libraries(chunk_obj PRIVATE operators utils ${ALL_DEPS}) add_library(dataset_obj STATIC s3vl/dataset_obj.cc) target_include_directories(dataset_obj PUBLIC ${PROJECT_INCLUDE_DIRS}) -target_link_libraries(dataset_obj PUBLIC chunk_obj ${ALL_DEPS}) +target_link_libraries(dataset_obj PRIVATE chunk_obj ${ALL_DEPS}) add_library(file_callbacks STATIC s3vl/file_callbacks.cc) target_include_directories(file_callbacks PUBLIC ${PROJECT_INCLUDE_DIRS}) -target_link_libraries(file_callbacks PUBLIC operators ${ALL_DEPS}) +target_link_libraries(file_callbacks PRIVATE operators ${ALL_DEPS}) add_library(dataset_callbacks STATIC s3vl/dataset_callbacks.cc) target_include_directories(dataset_callbacks PUBLIC ${PROJECT_INCLUDE_DIRS}) -target_link_libraries(dataset_callbacks PUBLIC file_callbacks dataset_obj ${ALL_DEPS}) +target_link_libraries(dataset_callbacks PRIVATE file_callbacks dataset_obj ${ALL_DEPS}) add_library(group_callbacks STATIC s3vl/group_callbacks.cc) target_include_directories(group_callbacks PUBLIC ${PROJECT_INCLUDE_DIRS}) -target_link_libraries(group_callbacks PUBLIC dataset_callbacks ${ALL_DEPS}) +target_link_libraries(group_callbacks PRIVATE dataset_callbacks ${ALL_DEPS}) # 3. The Final VOL Connector (shared library) add_library(array_morph SHARED s3vl/vol_connector.cc) set_target_properties(array_morph PROPERTIES PREFIX "lib_") target_include_directories(array_morph PUBLIC ${PROJECT_INCLUDE_DIRS}) target_link_libraries(array_morph PUBLIC - dataset_callbacks - file_callbacks group_callbacks ${ALL_DEPS} ) +# 4. Install into the Python package's lib/ directory +# scikit-build-core sets CMAKE_INSTALL_PREFIX to the wheel staging area +# so this puts the .so/.dylib at: arraymorph/lib/lib_array_morph.so install(TARGETS array_morph - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION arraymorph/lib # .so (Linux) and .dylib (macOS) + RUNTIME DESTINATION arraymorph/lib # .dll (Windows) ) diff --git a/arraymorph/src/core/constants.cc b/lib/src/core/constants.cc similarity index 100% rename from arraymorph/src/core/constants.cc rename to lib/src/core/constants.cc diff --git a/arraymorph/src/core/operators.cc b/lib/src/core/operators.cc similarity index 100% rename from arraymorph/src/core/operators.cc rename to lib/src/core/operators.cc diff --git a/arraymorph/src/core/utils.cc b/lib/src/core/utils.cc similarity index 100% rename from arraymorph/src/core/utils.cc rename to lib/src/core/utils.cc diff --git a/arraymorph/src/s3vl/chunk_obj.cc b/lib/src/s3vl/chunk_obj.cc similarity index 100% rename from arraymorph/src/s3vl/chunk_obj.cc rename to lib/src/s3vl/chunk_obj.cc diff --git a/arraymorph/src/s3vl/dataset_callbacks.cc b/lib/src/s3vl/dataset_callbacks.cc similarity index 100% rename from arraymorph/src/s3vl/dataset_callbacks.cc rename to lib/src/s3vl/dataset_callbacks.cc diff --git a/arraymorph/src/s3vl/dataset_obj.cc b/lib/src/s3vl/dataset_obj.cc similarity index 100% rename from arraymorph/src/s3vl/dataset_obj.cc rename to lib/src/s3vl/dataset_obj.cc diff --git a/arraymorph/src/s3vl/file_callbacks.cc b/lib/src/s3vl/file_callbacks.cc similarity index 87% rename from arraymorph/src/s3vl/file_callbacks.cc rename to lib/src/s3vl/file_callbacks.cc index 85e368b..4351bd2 100644 --- a/arraymorph/src/s3vl/file_callbacks.cc +++ b/lib/src/s3vl/file_callbacks.cc @@ -3,8 +3,9 @@ #include "arraymorph/core/logger.h" #include "arraymorph/core/operators.h" #include +#include #include -#include +#include CloudClient get_client() { Logger::log("Init cloud clients"); @@ -14,6 +15,7 @@ CloudClient get_client() { std::string access_key = getenv("AWS_ACCESS_KEY_ID"); std::string secret_key = getenv("AWS_SECRET_ACCESS_KEY"); Aws::Auth::AWSCredentials cred(access_key, secret_key); + std::unique_ptr s3ClientConfig = std::make_unique(); @@ -21,11 +23,9 @@ CloudClient get_client() { getenv("AWS_USE_TLS"); // Is TLS necessary. Does not use it by default. // Env var is char pointer by default. Checking to see if value was - // provided, ('true' or 'false'). If so, cast to std::string_view to avoid - // heap allocation and check for value. - s3ClientConfig->scheme = use_tls && std::string_view(use_tls) == "true" - ? Scheme::HTTPS - : Scheme::HTTP; + // provided, ('true' or 'false'). + s3ClientConfig->scheme = + use_tls && strcmp(use_tls, "true") == 0 ? Scheme::HTTPS : Scheme::HTTP; const char *endpoint = getenv("AWS_ENDPOINT_URL_S3"); // Custom S3 endpoint if (endpoint) { s3ClientConfig->endpointOverride = endpoint; @@ -43,16 +43,17 @@ CloudClient get_client() { // affect on performance, needs to be // tested. Off by default auto payload_signing_policy = - *signed_payloads && std::string_view(signed_payloads) == "true" + signed_payloads && strcmp(signed_payloads, "true") == 0 ? Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always : Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never; - const char *path_style = - getenv("AWS_USE_PATH_STYLE"); // Some S3-compatible stores require path - // styles, 'bucket.endpoint' vs - // 'endpoint/bucket'(with path-style). + const char *addressing_style = getenv( + "AWS_S3_ADDRESSING_STYLE"); // Some S3-compatible stores require path + // styles, 'bucket.endpoint' (virtual) vs + // 'endpoint/bucket'(with path-style). bool use_path_style = - *path_style && std::string_view(path_style) == "true" ? true : false; + addressing_style && strcmp(addressing_style, "path") == 0 ? true + : false; s3ClientConfig->maxConnections = s3Connections; s3ClientConfig->requestTimeoutMs = requestTimeoutMs; diff --git a/arraymorph/src/s3vl/group_callbacks.cc b/lib/src/s3vl/group_callbacks.cc similarity index 100% rename from arraymorph/src/s3vl/group_callbacks.cc rename to lib/src/s3vl/group_callbacks.cc diff --git a/arraymorph/src/s3vl/vol_connector.cc b/lib/src/s3vl/vol_connector.cc similarity index 100% rename from arraymorph/src/s3vl/vol_connector.cc rename to lib/src/s3vl/vol_connector.cc diff --git a/src/arraymorph/__init__.py b/src/arraymorph/__init__.py new file mode 100644 index 0000000..a466105 --- /dev/null +++ b/src/arraymorph/__init__.py @@ -0,0 +1,161 @@ +""" +ArrayMorph - HDF5 VOL connector for cloud object storage. + +Supports AWS S3 and Azure Blob Storage via HDF5's Virtual Object Layer. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +__version__ = "0.2.0" + +# The compiled VOL plugin lives next to this file after installation +_PLUGIN_DIR = str(Path(__file__).parent / "lib") + + +def get_plugin_path() -> str: + """Return the directory containing the ArrayMorph VOL plugin (.so/.dylib). + + Use this to set HDF5_PLUGIN_PATH: + >>> import arraymorph + >>> os.environ["HDF5_PLUGIN_PATH"] = arraymorph.get_plugin_path() + """ + return _PLUGIN_DIR + + +def enable() -> None: + """Configure HDF5 environment variables to use ArrayMorph. + + Sets HDF5_PLUGIN_PATH and HDF5_VOL_CONNECTOR so that any + subsequent h5py calls route through the ArrayMorph VOL connector. + + Usage: + >>> import arraymorph + >>> arraymorph.enable() + >>> import h5py + >>> f = h5py.File("s3://bucket/data.h5", "r") + """ + os.environ["HDF5_PLUGIN_PATH"] = _PLUGIN_DIR + os.environ["HDF5_VOL_CONNECTOR"] = "arraymorph" + + +def configure_s3( + bucket: str, + access_key: str = "", + secret_key: str = "", + endpoint: str | None = None, + region: str = "us-east-2", + use_tls: bool = False, + addressing_style: bool = False, + use_signed_payloads: bool = False, +) -> None: + """Configure AWS S3 credentials and client behavior for ArrayMorph. + + Sets the environment variables read by the VOL connector's S3 client + at initialization time. Call this before any h5py file operations. + + Args: + bucket: Name of the S3 bucket where HDF5 files are stored. + Maps to: BUCKET_NAME + access_key: Access key ID for authentication with the S3 service. + Maps to: AWS_ACCESS_KEY_ID + secret_key: Secret access key paired with access_key for authentication. + Maps to: AWS_SECRET_ACCESS_KEY + endpoint: Custom S3-compatible endpoint URL (e.g. 'http://localhost:3900'). + When None, the S3 client targets the default AWS endpoint. Required + for any non-AWS S3-compatible object store (MinIO, Ceph, etc.). + Maps to: AWS_ENDPOINT_URL_S3 + region: Region label used in SigV4 request signing. Must match the region + your bucket or S3-compatible store is configured with — a mismatch + produces signature validation errors. Defaults to 'us-east-2'. + Maps to: AWS_REGION + use_tls: Whether to use HTTPS (True) or HTTP (False) for S3 connections. + Set to False for object stores that do not have TLS configured. + Defaults to False. + Maps to: AWS_USE_TLS + addressing_style: URL addressing style for the S3 client. When True, + uses path-style ('endpoint/bucket/key'). When False, uses + virtual-hosted style ('bucket.endpoint/key'), which can cause the + S3 client to misinterpret the HDF5 filename as the bucket name. + Most S3-compatible stores require path-style addressing. + Defaults to False. + Maps to: AWS_S3_ADDRESSING_STYLE + use_signed_payloads: Whether to include the request body in the SigV4 + signature (PayloadSigningPolicy::Always). Some S3-compatible stores + require signed payloads and will reject requests with signature + validation errors if this is disabled. Defaults to False. + Maps to: AWS_SIGNED_PAYLOADS + + Example: + >>> import arraymorph + >>> arraymorph.configure_s3( + ... bucket="my-bucket", + ... access_key="my-access-key", + ... secret_key="my-secret-key", + ... endpoint="http://localhost:3900", + ... region="us-east-1", + ... use_tls=False, + ... addressing_style=True, + ... use_signed_payloads=True, + ... ) + >>> arraymorph.enable() + """ + if not (access_key and secret_key): + raise ValueError( + "configure_s3() requires both 'access_key' and 'secret_key'. " + "Set them explicitly or export AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY " + "before calling this function." + ) + + os.environ["AWS_ACCESS_KEY_ID"] = access_key + os.environ["AWS_SECRET_ACCESS_KEY"] = secret_key + os.environ["STORAGE_PLATFORM"] = "S3" + os.environ["BUCKET_NAME"] = bucket + os.environ["AWS_REGION"] = region + + if endpoint: + os.environ["AWS_ENDPOINT_URL_S3"] = endpoint + + os.environ["AWS_USE_TLS"] = str(use_tls).lower() + os.environ["AWS_S3_ADDRESSING_STYLE"] = "path" if addressing_style else "virtual" + os.environ["AWS_SIGNED_PAYLOADS"] = str(use_signed_payloads).lower() + + +def configure_azure( + container: str, + connection_string: str | None = None, +) -> None: + """Configure Azure Blob Storage credentials for ArrayMorph. + + Sets the environment variables read by the VOL connector's Azure client + at initialization time. Call this before any h5py file operations. + + Args: + container: Name of the Azure Blob Storage container where HDF5 files + are stored. Maps to: BUCKET_NAME + connection_string: Azure Storage connection string used to authenticate + and locate the storage account. If None, the connector will fall back + to the existing AZURE_STORAGE_CONNECTION_STRING environment variable. + Maps to: AZURE_STORAGE_CONNECTION_STRING + + Example: + >>> import arraymorph + >>> arraymorph.configure_azure( + ... container="my-container", + ... connection_string="DefaultEndpointsProtocol=https;AccountName=...", + ... ) + >>> arraymorph.enable() + """ + if not connection_string and not os.environ.get("AZURE_STORAGE_CONNECTION_STRING"): + raise ValueError( + "configure_azure() requires a 'connection_string'. " + "Set it explicitly or export AZURE_STORAGE_CONNECTION_STRING " + "before calling this function." + ) + + os.environ["STORAGE_PLATFORM"] = "Azure" + os.environ["BUCKET_NAME"] = container + if connection_string: + os.environ["AZURE_STORAGE_CONNECTION_STRING"] = connection_string From 53d2cf502af9a47bca496b27dbe401a8ef6a9a41 Mon Sep 17 00:00:00 2001 From: Carlos Guzman Date: Wed, 25 Feb 2026 10:50:41 -0600 Subject: [PATCH 04/16] Adding missing files --- .python-version | 1 + CONAN-INTEGRATION.md | 1 - env-example.txt | 2 - justfile | 63 ++++++++ pyproject.toml | 69 ++++++++- uv.lock | 336 ++++++++++++++++++++++++++++++++++++++++++- 6 files changed, 460 insertions(+), 12 deletions(-) create mode 100644 .python-version create mode 100644 justfile diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..cc1923a --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.8 diff --git a/CONAN-INTEGRATION.md b/CONAN-INTEGRATION.md index 9829db3..6bd2176 100644 --- a/CONAN-INTEGRATION.md +++ b/CONAN-INTEGRATION.md @@ -247,7 +247,6 @@ AWS_S3_ADDRESSING_STYLE="path" AWS_ACCESS_KEY_ID=garage-access-key AWS_SECRET_ACCESS_KEY=secret-access-key AWS_REGION=garage -AWS_USE_PATH_STYLE=true AWS_USE_TLS=true AWS_SIGNED_PAYLOADS=true HDF5_VOL_CONNECTOR=arraymorph diff --git a/env-example.txt b/env-example.txt index f300413..7e6bb00 100644 --- a/env-example.txt +++ b/env-example.txt @@ -5,9 +5,7 @@ AWS_S3_ADDRESSING_STYLE="path" AWS_ACCESS_KEY_ID=garage-access-key AWS_SECRET_ACCESS_KEY=secret-access-key AWS_REGION=garage -AWS_USE_PATH_STYLE=true AWS_USE_TLS=true -AWS_SIGNED_PAYLOADS=true HDF5_VOL_CONNECTOR=arraymorph HDF5_PLUGIN_PATH=path-to-libarraymorph diff --git a/justfile b/justfile new file mode 100644 index 0000000..ccd9e7b --- /dev/null +++ b/justfile @@ -0,0 +1,63 @@ + +# ArrayMorph — Top-Level Build Orchestration +# https://just.systems + +# --- Settings --- +set dotenv-load := true +set export := true + +# --- Variables --- +CONAN_BUILD_DIR := "lib/build/Release/generators" +CMAKE_TOOLCHAIN_FILE := justfile_directory() / CONAN_BUILD_DIR / "conan_toolchain.cmake" +H5PY_HDF5_DIR := `./.venv/bin/python -c "import h5py,os;d=os.path.dirname(h5py.__file__);print(os.path.join(d,'.dylibs') if os.path.exists(os.path.join(d,'.dylibs')) else os.path.join(os.path.dirname(d),'h5py.libs'))"` + +# --- Recipes --- + +# List available commands +default: + @just --list + +# Install C++ dependencies via Conan +deps: + cd lib && conan install . --build=missing -s build_type=Release + +# Build Python wheel (runs scikit-build-core + CMake) +wheel: + CMAKE_TOOLCHAIN_FILE={{ CMAKE_TOOLCHAIN_FILE }} \ + H5PY_HDF5_DIR={{ H5PY_HDF5_DIR }} \ + uv build + +# Install editable into current venv (for development iteration) +dev: + CMAKE_TOOLCHAIN_FILE={{ CMAKE_TOOLCHAIN_FILE }} \ + H5PY_HDF5_DIR={{ H5PY_HDF5_DIR }} \ + uv pip install -e . + +# Full build from scratch: deps → wheel +build: deps wheel + +# Test the built wheel in an isolated venv +test: + rm -rf .test-venv + uv venv .test-venv + source .test-venv/bin/activate.fish + uv pip install dist/arraymorph-0.2.0-*.whl + python -c "import arraymorph; print('Plugin:', arraymorph.get_plugin_path()); arraymorph.enable(); print('VOL enabled')" + rm -rf .test-venv + +# Full build + test +all: build test + +# Clean build artifacts +clean: + rm -rf lib/build dist *.egg-info .test-venv + +# Full clean rebuild +rebuild: clean build + +# Show current env var values (for debugging) +info: + @echo "CMAKE_TOOLCHAIN_FILE: {{ CMAKE_TOOLCHAIN_FILE }}" + @echo "H5PY_HDF5_DIR: {{ H5PY_HDF5_DIR }}" + @echo "Plugin lib: $(find lib/build -name 'lib_array_morph*' 2>/dev/null || echo 'not built')" + diff --git a/pyproject.toml b/pyproject.toml index 6708d7d..6d89eab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,68 @@ +[build-system] +requires = ["scikit-build-core", "cmake>=3.20", "ninja"] +build-backend = "scikit_build_core.build" + [project] name = "arraymorph" -version = "0.1.0" -description = "Add your description here" +version = "0.2.0" +description = "HDF5 VOL connector for cloud object storage (AWS S3, Azure Blob)" readme = "README.md" -requires-python = ">=3.14" -dependencies = [ - "h5py>=3.15.1", +license = { text = "MIT" } +authors = [{ name = "ICICLE AI Institute" }] +requires-python = ">=3.8" +dependencies = ["h5py>=3.11.0"] + +[project.urls] +Homepage = "https://github.com/ICICLE-ai/ArrayMorph" +Repository = "https://github.com/ICICLE-ai/ArrayMorph" +Issues = "https://github.com/ICICLE-ai/ArrayMorph/issues" + + +# --- scikit-build-core settings --- + +[tool.scikit-build] +# CMakeLists.txt is inside arraymorph/ subdirectory +cmake.source-dir = "lib" +cmake.build-type = "Release" +cmake.verbose = true + +# Install the Python package from src/ +wheel.packages = ["src/arraymorph"] + +# Pass h5py HDF5 location to CMake +# This gets set by the before-build script in cibuildwheel +[tool.scikit-build.cmake.define] +CMAKE_POSITION_INDEPENDENT_CODE = "ON" +CMAKE_TOOLCHAIN_FILE = {env = "CMAKE_TOOLCHAIN_FILE", default = ""} + +# --- cibuildwheel settings --- + +[tool.cibuildwheel] +# ARRAYMORPH_USE_CONAN signals CMake to use the Conan toolchain. +# H5PY_HDF5_DIR is intentionally NOT set here — h5py must be installed +# first (in before-build) before we can discover its bundled HDF5 path. +environment = { ARRAYMORPH_USE_CONAN = "1" } + +# Only build CPython, skip 32-bit and musl +skip = ["pp*", "*-win32", "*-manylinux_i686", "*-musllinux*"] + +# Before building: +# 1. Install h5py so we can locate its bundled HDF5 shared library +# 2. Set H5PY_HDF5_DIR so CMake knows which HDF5 binary to link against +# 3. Install conan and resolve C++ dependencies +before-build = [ + "pip install h5py", + "export H5PY_HDF5_DIR=$(python -c \"import h5py,os;d=os.path.dirname(h5py.__file__);print(os.path.join(d,'.dylibs') if os.path.exists(os.path.join(d,'.dylibs')) else os.path.join(os.path.dirname(d),'h5py.libs'))\")", + "pip install conan", + "cd {project}/arraymorph && conan install . --build=missing -of build", ] + + +[tool.cibuildwheel.linux] +# manylinux2014 has the toolchain we need +manylinux-x86_64-image = "manylinux_2_28" +manylinux-aarch64-image = "manylinux_2_28" + +[tool.cibuildwheel.macos] +# Build for both Intel and Apple Silicon +archs = ["x86_64", "arm64"] diff --git a/uv.lock b/uv.lock index f61dd0c..e24214f 100644 --- a/uv.lock +++ b/uv.lock @@ -1,27 +1,144 @@ version = 1 revision = 3 -requires-python = ">=3.14" +requires-python = ">=3.8" +resolution-markers = [ + "python_full_version >= '3.11'", + "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", + "python_full_version < '3.9'", +] [[package]] name = "arraymorph" version = "0.1.0" source = { virtual = "." } dependencies = [ - { name = "h5py" }, + { name = "h5py", version = "3.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "h5py", version = "3.14.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "h5py", version = "3.15.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, ] [package.metadata] -requires-dist = [{ name = "h5py", specifier = ">=3.15.1" }] +requires-dist = [{ name = "h5py", specifier = ">=3.11.0" }] + +[[package]] +name = "h5py" +version = "3.11.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +dependencies = [ + { name = "numpy", version = "1.24.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/8f/e557819155a282da36fb21f8de4730cfd10a964b52b3ae8d20157ac1c668/h5py-3.11.0.tar.gz", hash = "sha256:7b7e8f78072a2edec87c9836f25f34203fd492a4475709a18b417a33cfb21fa9", size = 406519, upload-time = "2024-04-10T10:52:39.585Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/25/a1cc81b3a742b73f9409bafe4762c9de0940cce0955d4b6754698fd5ce44/h5py-3.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1625fd24ad6cfc9c1ccd44a66dac2396e7ee74940776792772819fc69f3a3731", size = 3477113, upload-time = "2024-04-10T10:48:28.323Z" }, + { url = "https://files.pythonhosted.org/packages/d4/03/bbb9a992fb43d3ce46687b7c14107f0fa56e6c8704c9ca945a9392cbc8ce/h5py-3.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c072655ad1d5fe9ef462445d3e77a8166cbfa5e599045f8aa3c19b75315f10e5", size = 2939879, upload-time = "2024-04-10T10:48:38.094Z" }, + { url = "https://files.pythonhosted.org/packages/94/00/94bf8573e7487b7c37f2b613fc381880d48ec2311f2e859b8a5817deb4df/h5py-3.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77b19a40788e3e362b54af4dcf9e6fde59ca016db2c61360aa30b47c7b7cef00", size = 5306122, upload-time = "2024-04-10T10:48:51.581Z" }, + { url = "https://files.pythonhosted.org/packages/bb/0d/fbadb9c69e2a31f641bc24e8d21671129ef3b73f0c61bb16b094fadf1385/h5py-3.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:ef4e2f338fc763f50a8113890f455e1a70acd42a4d083370ceb80c463d803972", size = 2968816, upload-time = "2024-04-10T10:49:07.091Z" }, + { url = "https://files.pythonhosted.org/packages/a0/52/38bb74cc4362738cc7ef819503fc54d70f0c3a7378519ccb0ac309389122/h5py-3.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bbd732a08187a9e2a6ecf9e8af713f1d68256ee0f7c8b652a32795670fb481ba", size = 3489913, upload-time = "2024-04-10T10:49:15.92Z" }, + { url = "https://files.pythonhosted.org/packages/f0/af/dfbea0c69fe725e9e77259d42f4e14eb582eb094200aaf697feb36f513d8/h5py-3.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75bd7b3d93fbeee40860fd70cdc88df4464e06b70a5ad9ce1446f5f32eb84007", size = 2946912, upload-time = "2024-04-10T10:49:25.757Z" }, + { url = "https://files.pythonhosted.org/packages/af/26/f231ee425c8df93c1abbead3d90ea4a5ff3d6aa49e0edfd3b4c017e74844/h5py-3.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52c416f8eb0daae39dabe71415cb531f95dce2d81e1f61a74537a50c63b28ab3", size = 5420165, upload-time = "2024-04-10T10:49:57.203Z" }, + { url = "https://files.pythonhosted.org/packages/d8/5e/b7b83cfe60504cc4d24746aed04353af7ea8ec104e597e5ae71b8d0390cb/h5py-3.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:083e0329ae534a264940d6513f47f5ada617da536d8dccbafc3026aefc33c90e", size = 2979079, upload-time = "2024-04-10T10:50:11.4Z" }, + { url = "https://files.pythonhosted.org/packages/58/a9/2655d4b8355d0ee783dc89dd40b5f0780e6f54a4c9b60721dc235fd6c457/h5py-3.11.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a76cae64080210389a571c7d13c94a1a6cf8cb75153044fd1f822a962c97aeab", size = 3466468, upload-time = "2024-04-10T10:50:30.236Z" }, + { url = "https://files.pythonhosted.org/packages/9d/3f/cf80ef55e0a9b18aae96c763fbd275c54d0723e0f2cc54f954f87cc5c69a/h5py-3.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3736fe21da2b7d8a13fe8fe415f1272d2a1ccdeff4849c1421d2fb30fd533bc", size = 2943214, upload-time = "2024-04-10T10:50:42.532Z" }, + { url = "https://files.pythonhosted.org/packages/db/7e/fedac8bb8c4729409e2dec5e4136a289116d701d54f69ce73c5617afc5f0/h5py-3.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa6ae84a14103e8dc19266ef4c3e5d7c00b68f21d07f2966f0ca7bdb6c2761fb", size = 5378375, upload-time = "2024-04-10T10:50:55.591Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b2/0ee327933ffa37af1fc7915df7fc067e6009adcd8445d55ad07a9bec11b5/h5py-3.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:21dbdc5343f53b2e25404673c4f00a3335aef25521bd5fa8c707ec3833934892", size = 2970991, upload-time = "2024-04-10T10:51:01.555Z" }, + { url = "https://files.pythonhosted.org/packages/33/97/c1a8f28329ad794d18fc61bf251268ac03959bf93b82fdd7701ac6931fed/h5py-3.11.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:754c0c2e373d13d6309f408325343b642eb0f40f1a6ad21779cfa9502209e150", size = 3470228, upload-time = "2024-04-10T10:51:14.716Z" }, + { url = "https://files.pythonhosted.org/packages/a4/1d/fd0b88c51c37bc8aeedecc4f4b48397f7ce13c87073aaf6912faec06e9f6/h5py-3.11.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:731839240c59ba219d4cb3bc5880d438248533366f102402cfa0621b71796b62", size = 2935809, upload-time = "2024-04-10T10:51:33.125Z" }, + { url = "https://files.pythonhosted.org/packages/86/43/fd0bd74462b3c3fb35d98568935d3e5a435c8ec24d45ef408ac8869166af/h5py-3.11.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ec9df3dd2018904c4cc06331951e274f3f3fd091e6d6cc350aaa90fa9b42a76", size = 5309045, upload-time = "2024-04-10T10:51:44.58Z" }, + { url = "https://files.pythonhosted.org/packages/15/9a/b5456e1acc4abb382938d4a730600823bfe77a4bbfd29140ccbf01ba5596/h5py-3.11.0-cp38-cp38-win_amd64.whl", hash = "sha256:55106b04e2c83dfb73dc8732e9abad69d83a436b5b82b773481d95d17b9685e1", size = 2989172, upload-time = "2024-04-10T10:51:56.815Z" }, + { url = "https://files.pythonhosted.org/packages/c2/1f/36a84945616881bd47e6c40dcdca7e929bc811725d78d001eddba6864185/h5py-3.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f4e025e852754ca833401777c25888acb96889ee2c27e7e629a19aee288833f0", size = 3490090, upload-time = "2024-04-10T10:52:08.237Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/e213586de5ea56f1747a843e725c62eef350512be57452186996ba660d52/h5py-3.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6c4b760082626120031d7902cd983d8c1f424cdba2809f1067511ef283629d4b", size = 2951710, upload-time = "2024-04-10T10:52:20.066Z" }, + { url = "https://files.pythonhosted.org/packages/71/28/69a881e01f198ccdb65c36f7adcfef22bfe85e38ffbfdf833af24f58eb5e/h5py-3.11.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67462d0669f8f5459529de179f7771bd697389fcb3faab54d63bf788599a48ea", size = 5326481, upload-time = "2024-04-10T10:52:30.029Z" }, + { url = "https://files.pythonhosted.org/packages/c3/61/0b35ad9aac0ab0a33365879556fdb824fc83013df69b247386690db59015/h5py-3.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:d9c944d364688f827dc889cf83f1fca311caf4fa50b19f009d1f2b525edd33a3", size = 2978689, upload-time = "2024-04-10T10:52:37.283Z" }, +] + +[[package]] +name = "h5py" +version = "3.14.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.9.*'", +] +dependencies = [ + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/57/dfb3c5c3f1bf5f5ef2e59a22dec4ff1f3d7408b55bfcefcfb0ea69ef21c6/h5py-3.14.0.tar.gz", hash = "sha256:2372116b2e0d5d3e5e705b7f663f7c8d96fa79a4052d250484ef91d24d6a08f4", size = 424323, upload-time = "2025-06-06T14:06:15.01Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/89/06cbb421e01dea2e338b3154326523c05d9698f89a01f9d9b65e1ec3fb18/h5py-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:24df6b2622f426857bda88683b16630014588a0e4155cba44e872eb011c4eaed", size = 3332522, upload-time = "2025-06-06T14:04:13.775Z" }, + { url = "https://files.pythonhosted.org/packages/c3/e7/6c860b002329e408348735bfd0459e7b12f712c83d357abeef3ef404eaa9/h5py-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6ff2389961ee5872de697054dd5a033b04284afc3fb52dc51d94561ece2c10c6", size = 2831051, upload-time = "2025-06-06T14:04:18.206Z" }, + { url = "https://files.pythonhosted.org/packages/fa/cd/3dd38cdb7cc9266dc4d85f27f0261680cb62f553f1523167ad7454e32b11/h5py-3.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:016e89d3be4c44f8d5e115fab60548e518ecd9efe9fa5c5324505a90773e6f03", size = 4324677, upload-time = "2025-06-06T14:04:23.438Z" }, + { url = "https://files.pythonhosted.org/packages/b1/45/e1a754dc7cd465ba35e438e28557119221ac89b20aaebef48282654e3dc7/h5py-3.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1223b902ef0b5d90bcc8a4778218d6d6cd0f5561861611eda59fa6c52b922f4d", size = 4557272, upload-time = "2025-06-06T14:04:28.863Z" }, + { url = "https://files.pythonhosted.org/packages/5c/06/f9506c1531645829d302c420851b78bb717af808dde11212c113585fae42/h5py-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:852b81f71df4bb9e27d407b43071d1da330d6a7094a588efa50ef02553fa7ce4", size = 2866734, upload-time = "2025-06-06T14:04:33.5Z" }, + { url = "https://files.pythonhosted.org/packages/61/1b/ad24a8ce846cf0519695c10491e99969d9d203b9632c4fcd5004b1641c2e/h5py-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f30dbc58f2a0efeec6c8836c97f6c94afd769023f44e2bb0ed7b17a16ec46088", size = 3352382, upload-time = "2025-06-06T14:04:37.95Z" }, + { url = "https://files.pythonhosted.org/packages/36/5b/a066e459ca48b47cc73a5c668e9924d9619da9e3c500d9fb9c29c03858ec/h5py-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:543877d7f3d8f8a9828ed5df6a0b78ca3d8846244b9702e99ed0d53610b583a8", size = 2852492, upload-time = "2025-06-06T14:04:42.092Z" }, + { url = "https://files.pythonhosted.org/packages/08/0c/5e6aaf221557314bc15ba0e0da92e40b24af97ab162076c8ae009320a42b/h5py-3.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c497600c0496548810047257e36360ff551df8b59156d3a4181072eed47d8ad", size = 4298002, upload-time = "2025-06-06T14:04:47.106Z" }, + { url = "https://files.pythonhosted.org/packages/21/d4/d461649cafd5137088fb7f8e78fdc6621bb0c4ff2c090a389f68e8edc136/h5py-3.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:723a40ee6505bd354bfd26385f2dae7bbfa87655f4e61bab175a49d72ebfc06b", size = 4516618, upload-time = "2025-06-06T14:04:52.467Z" }, + { url = "https://files.pythonhosted.org/packages/db/0c/6c3f879a0f8e891625817637fad902da6e764e36919ed091dc77529004ac/h5py-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d2744b520440a996f2dae97f901caa8a953afc055db4673a993f2d87d7f38713", size = 2874888, upload-time = "2025-06-06T14:04:56.95Z" }, + { url = "https://files.pythonhosted.org/packages/3e/77/8f651053c1843391e38a189ccf50df7e261ef8cd8bfd8baba0cbe694f7c3/h5py-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e0045115d83272090b0717c555a31398c2c089b87d212ceba800d3dc5d952e23", size = 3312740, upload-time = "2025-06-06T14:05:01.193Z" }, + { url = "https://files.pythonhosted.org/packages/ff/10/20436a6cf419b31124e59fefc78d74cb061ccb22213226a583928a65d715/h5py-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6da62509b7e1d71a7d110478aa25d245dd32c8d9a1daee9d2a42dba8717b047a", size = 2829207, upload-time = "2025-06-06T14:05:05.061Z" }, + { url = "https://files.pythonhosted.org/packages/3f/19/c8bfe8543bfdd7ccfafd46d8cfd96fce53d6c33e9c7921f375530ee1d39a/h5py-3.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:554ef0ced3571366d4d383427c00c966c360e178b5fb5ee5bb31a435c424db0c", size = 4708455, upload-time = "2025-06-06T14:05:11.528Z" }, + { url = "https://files.pythonhosted.org/packages/86/f9/f00de11c82c88bfc1ef22633557bfba9e271e0cb3189ad704183fc4a2644/h5py-3.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cbd41f4e3761f150aa5b662df991868ca533872c95467216f2bec5fcad84882", size = 4929422, upload-time = "2025-06-06T14:05:18.399Z" }, + { url = "https://files.pythonhosted.org/packages/7a/6d/6426d5d456f593c94b96fa942a9b3988ce4d65ebaf57d7273e452a7222e8/h5py-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:bf4897d67e613ecf5bdfbdab39a1158a64df105827da70ea1d90243d796d367f", size = 2862845, upload-time = "2025-06-06T14:05:23.699Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c2/7efe82d09ca10afd77cd7c286e42342d520c049a8c43650194928bcc635c/h5py-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:aa4b7bbce683379b7bf80aaba68e17e23396100336a8d500206520052be2f812", size = 3289245, upload-time = "2025-06-06T14:05:28.24Z" }, + { url = "https://files.pythonhosted.org/packages/4f/31/f570fab1239b0d9441024b92b6ad03bb414ffa69101a985e4c83d37608bd/h5py-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ef9603a501a04fcd0ba28dd8f0995303d26a77a980a1f9474b3417543d4c6174", size = 2807335, upload-time = "2025-06-06T14:05:31.997Z" }, + { url = "https://files.pythonhosted.org/packages/0d/ce/3a21d87896bc7e3e9255e0ad5583ae31ae9e6b4b00e0bcb2a67e2b6acdbc/h5py-3.14.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8cbaf6910fa3983c46172666b0b8da7b7bd90d764399ca983236f2400436eeb", size = 4700675, upload-time = "2025-06-06T14:05:37.38Z" }, + { url = "https://files.pythonhosted.org/packages/e7/ec/86f59025306dcc6deee5fda54d980d077075b8d9889aac80f158bd585f1b/h5py-3.14.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d90e6445ab7c146d7f7981b11895d70bc1dd91278a4f9f9028bc0c95e4a53f13", size = 4921632, upload-time = "2025-06-06T14:05:43.464Z" }, + { url = "https://files.pythonhosted.org/packages/3f/6d/0084ed0b78d4fd3e7530c32491f2884140d9b06365dac8a08de726421d4a/h5py-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:ae18e3de237a7a830adb76aaa68ad438d85fe6e19e0d99944a3ce46b772c69b3", size = 2852929, upload-time = "2025-06-06T14:05:47.659Z" }, + { url = "https://files.pythonhosted.org/packages/ec/ac/9ea82488c8790ee5b6ad1a807cd7dc3b9dadfece1cd0e0e369f68a7a8937/h5py-3.14.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f5cc1601e78027cedfec6dd50efb4802f018551754191aeb58d948bd3ec3bd7a", size = 3345097, upload-time = "2025-06-06T14:05:51.984Z" }, + { url = "https://files.pythonhosted.org/packages/6c/bc/a172ecaaf287e3af2f837f23b470b0a2229c79555a0da9ac8b5cc5bed078/h5py-3.14.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5e59d2136a8b302afd25acdf7a89b634e0eb7c66b1a211ef2d0457853768a2ef", size = 2843320, upload-time = "2025-06-06T14:05:55.754Z" }, + { url = "https://files.pythonhosted.org/packages/66/40/b423b57696514e05aa7bb06150ef96667d0e0006cc6de7ab52c71734ab51/h5py-3.14.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:573c33ad056ac7c1ab6d567b6db9df3ffc401045e3f605736218f96c1e0490c6", size = 4326368, upload-time = "2025-06-06T14:06:00.782Z" }, + { url = "https://files.pythonhosted.org/packages/f7/07/e088f89f04fdbe57ddf9de377f857158d3daa38cf5d0fb20ef9bd489e313/h5py-3.14.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccbe17dc187c0c64178f1a10aa274ed3a57d055117588942b8a08793cc448216", size = 4559686, upload-time = "2025-06-06T14:06:07.416Z" }, + { url = "https://files.pythonhosted.org/packages/b4/e4/fb8032d0e5480b1db9b419b5b50737b61bb3c7187c49d809975d62129fb0/h5py-3.14.0-cp39-cp39-win_amd64.whl", hash = "sha256:4f025cf30ae738c4c4e38c7439a761a71ccfcce04c2b87b2a2ac64e8c5171d43", size = 2877166, upload-time = "2025-06-06T14:06:13.05Z" }, +] [[package]] name = "h5py" version = "3.15.1" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.11'", + "python_full_version == '3.10.*'", +] dependencies = [ - { name = "numpy" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4d/6a/0d79de0b025aa85dc8864de8e97659c94cf3d23148394a954dc5ca52f8c8/h5py-3.15.1.tar.gz", hash = "sha256:c86e3ed45c4473564de55aa83b6fc9e5ead86578773dfbd93047380042e26b69", size = 426236, upload-time = "2025-10-16T10:35:27.404Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/86/30/8fa61698b438dd751fa46a359792e801191dadab560d0a5f1c709443ef8e/h5py-3.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:67e59f6c2f19a32973a40f43d9a088ae324fe228c8366e25ebc57ceebf093a6b", size = 3414477, upload-time = "2025-10-16T10:33:24.201Z" }, + { url = "https://files.pythonhosted.org/packages/16/16/db2f63302937337c4e9e51d97a5984b769bdb7488e3d37632a6ac297f8ef/h5py-3.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e2f471688402c3404fa4e13466e373e622fd4b74b47b56cfdff7cc688209422", size = 2850298, upload-time = "2025-10-16T10:33:27.747Z" }, + { url = "https://files.pythonhosted.org/packages/fc/2e/f1bb7de9b05112bfd14d5206090f0f92f1e75bbb412fbec5d4653c3d44dd/h5py-3.15.1-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c45802bcb711e128a6839cb6c01e9ac648dc55df045c9542a675c771f15c8d5", size = 4523605, upload-time = "2025-10-16T10:33:31.168Z" }, + { url = "https://files.pythonhosted.org/packages/05/8a/63f4b08f3628171ce8da1a04681a65ee7ac338fde3cb3e9e3c9f7818e4da/h5py-3.15.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64ce3f6470adb87c06e3a8dd1b90e973699f1759ad79bfa70c230939bff356c9", size = 4735346, upload-time = "2025-10-16T10:33:34.759Z" }, + { url = "https://files.pythonhosted.org/packages/74/48/f16d12d9de22277605bcc11c0dcab5e35f06a54be4798faa2636b5d44b3c/h5py-3.15.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4411c1867b9899a25e983fff56d820a66f52ac326bbe10c7cdf7d832c9dcd883", size = 4175305, upload-time = "2025-10-16T10:33:38.83Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/47cdbff65b2ce53c27458c6df63a232d7bb1644b97df37b2342442342c84/h5py-3.15.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2cbc4104d3d4aca9d6db8c0c694555e255805bfeacf9eb1349bda871e26cacbe", size = 4653602, upload-time = "2025-10-16T10:33:42.188Z" }, + { url = "https://files.pythonhosted.org/packages/c3/28/dc08de359c2f43a67baa529cb70d7f9599848750031975eed92d6ae78e1d/h5py-3.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:01f55111ca516f5568ae7a7fc8247dfce607de331b4467ee8a9a6ed14e5422c7", size = 2873601, upload-time = "2025-10-16T10:33:45.323Z" }, + { url = "https://files.pythonhosted.org/packages/41/fd/8349b48b15b47768042cff06ad6e1c229f0a4bd89225bf6b6894fea27e6d/h5py-3.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5aaa330bcbf2830150c50897ea5dcbed30b5b6d56897289846ac5b9e529ec243", size = 3434135, upload-time = "2025-10-16T10:33:47.954Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b0/1c628e26a0b95858f54aba17e1599e7f6cd241727596cc2580b72cb0a9bf/h5py-3.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c970fb80001fffabb0109eaf95116c8e7c0d3ca2de854e0901e8a04c1f098509", size = 2870958, upload-time = "2025-10-16T10:33:50.907Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e3/c255cafc9b85e6ea04e2ad1bba1416baa1d7f57fc98a214be1144087690c/h5py-3.15.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:80e5bb5b9508d5d9da09f81fd00abbb3f85da8143e56b1585d59bc8ceb1dba8b", size = 4504770, upload-time = "2025-10-16T10:33:54.357Z" }, + { url = "https://files.pythonhosted.org/packages/8b/23/4ab1108e87851ccc69694b03b817d92e142966a6c4abd99e17db77f2c066/h5py-3.15.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5b849ba619a066196169763c33f9f0f02e381156d61c03e000bb0100f9950faf", size = 4700329, upload-time = "2025-10-16T10:33:57.616Z" }, + { url = "https://files.pythonhosted.org/packages/a4/e4/932a3a8516e4e475b90969bf250b1924dbe3612a02b897e426613aed68f4/h5py-3.15.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e7f6c841efd4e6e5b7e82222eaf90819927b6d256ab0f3aca29675601f654f3c", size = 4152456, upload-time = "2025-10-16T10:34:00.843Z" }, + { url = "https://files.pythonhosted.org/packages/2a/0a/f74d589883b13737021b2049ac796328f188dbb60c2ed35b101f5b95a3fc/h5py-3.15.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ca8a3a22458956ee7b40d8e39c9a9dc01f82933e4c030c964f8b875592f4d831", size = 4617295, upload-time = "2025-10-16T10:34:04.154Z" }, + { url = "https://files.pythonhosted.org/packages/23/95/499b4e56452ef8b6c95a271af0dde08dac4ddb70515a75f346d4f400579b/h5py-3.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:550e51131376889656feec4aff2170efc054a7fe79eb1da3bb92e1625d1ac878", size = 2882129, upload-time = "2025-10-16T10:34:06.886Z" }, + { url = "https://files.pythonhosted.org/packages/ce/bb/cfcc70b8a42222ba3ad4478bcef1791181ea908e2adbd7d53c66395edad5/h5py-3.15.1-cp311-cp311-win_arm64.whl", hash = "sha256:b39239947cb36a819147fc19e86b618dcb0953d1cd969f5ed71fc0de60392427", size = 2477121, upload-time = "2025-10-16T10:34:09.579Z" }, + { url = "https://files.pythonhosted.org/packages/62/b8/c0d9aa013ecfa8b7057946c080c0c07f6fa41e231d2e9bd306a2f8110bdc/h5py-3.15.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:316dd0f119734f324ca7ed10b5627a2de4ea42cc4dfbcedbee026aaa361c238c", size = 3399089, upload-time = "2025-10-16T10:34:12.135Z" }, + { url = "https://files.pythonhosted.org/packages/a4/5e/3c6f6e0430813c7aefe784d00c6711166f46225f5d229546eb53032c3707/h5py-3.15.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b51469890e58e85d5242e43aab29f5e9c7e526b951caab354f3ded4ac88e7b76", size = 2847803, upload-time = "2025-10-16T10:34:14.564Z" }, + { url = "https://files.pythonhosted.org/packages/00/69/ba36273b888a4a48d78f9268d2aee05787e4438557450a8442946ab8f3ec/h5py-3.15.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a33bfd5dfcea037196f7778534b1ff7e36a7f40a89e648c8f2967292eb6898e", size = 4914884, upload-time = "2025-10-16T10:34:18.452Z" }, + { url = "https://files.pythonhosted.org/packages/3a/30/d1c94066343a98bb2cea40120873193a4fed68c4ad7f8935c11caf74c681/h5py-3.15.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:25c8843fec43b2cc368aa15afa1cdf83fc5e17b1c4e10cd3771ef6c39b72e5ce", size = 5109965, upload-time = "2025-10-16T10:34:21.853Z" }, + { url = "https://files.pythonhosted.org/packages/81/3d/d28172116eafc3bc9f5991b3cb3fd2c8a95f5984f50880adfdf991de9087/h5py-3.15.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a308fd8681a864c04423c0324527237a0484e2611e3441f8089fd00ed56a8171", size = 4561870, upload-time = "2025-10-16T10:34:26.69Z" }, + { url = "https://files.pythonhosted.org/packages/a5/83/393a7226024238b0f51965a7156004eaae1fcf84aa4bfecf7e582676271b/h5py-3.15.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f4a016df3f4a8a14d573b496e4d1964deb380e26031fc85fb40e417e9131888a", size = 5037161, upload-time = "2025-10-16T10:34:30.383Z" }, + { url = "https://files.pythonhosted.org/packages/cf/51/329e7436bf87ca6b0fe06dd0a3795c34bebe4ed8d6c44450a20565d57832/h5py-3.15.1-cp312-cp312-win_amd64.whl", hash = "sha256:59b25cf02411bf12e14f803fef0b80886444c7fe21a5ad17c6a28d3f08098a1e", size = 2874165, upload-time = "2025-10-16T10:34:33.461Z" }, + { url = "https://files.pythonhosted.org/packages/09/a8/2d02b10a66747c54446e932171dd89b8b4126c0111b440e6bc05a7c852ec/h5py-3.15.1-cp312-cp312-win_arm64.whl", hash = "sha256:61d5a58a9851e01ee61c932bbbb1c98fe20aba0a5674776600fb9a361c0aa652", size = 2458214, upload-time = "2025-10-16T10:34:35.733Z" }, + { url = "https://files.pythonhosted.org/packages/88/b3/40207e0192415cbff7ea1d37b9f24b33f6d38a5a2f5d18a678de78f967ae/h5py-3.15.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c8440fd8bee9500c235ecb7aa1917a0389a2adb80c209fa1cc485bd70e0d94a5", size = 3376511, upload-time = "2025-10-16T10:34:38.596Z" }, + { url = "https://files.pythonhosted.org/packages/31/96/ba99a003c763998035b0de4c299598125df5fc6c9ccf834f152ddd60e0fb/h5py-3.15.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ab2219dbc6fcdb6932f76b548e2b16f34a1f52b7666e998157a4dfc02e2c4123", size = 2826143, upload-time = "2025-10-16T10:34:41.342Z" }, + { url = "https://files.pythonhosted.org/packages/6a/c2/fc6375d07ea3962df7afad7d863fe4bde18bb88530678c20d4c90c18de1d/h5py-3.15.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8cb02c3a96255149ed3ac811eeea25b655d959c6dd5ce702c9a95ff11859eb5", size = 4908316, upload-time = "2025-10-16T10:34:44.619Z" }, + { url = "https://files.pythonhosted.org/packages/d9/69/4402ea66272dacc10b298cca18ed73e1c0791ff2ae9ed218d3859f9698ac/h5py-3.15.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:121b2b7a4c1915d63737483b7bff14ef253020f617c2fb2811f67a4bed9ac5e8", size = 5103710, upload-time = "2025-10-16T10:34:48.639Z" }, + { url = "https://files.pythonhosted.org/packages/e0/f6/11f1e2432d57d71322c02a97a5567829a75f223a8c821764a0e71a65cde8/h5py-3.15.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:59b0d63b318bf3cc06687def2b45afd75926bbc006f7b8cd2b1a231299fc8599", size = 4556042, upload-time = "2025-10-16T10:34:51.841Z" }, + { url = "https://files.pythonhosted.org/packages/18/88/3eda3ef16bfe7a7dbc3d8d6836bbaa7986feb5ff091395e140dc13927bcc/h5py-3.15.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e02fe77a03f652500d8bff288cbf3675f742fc0411f5a628fa37116507dc7cc0", size = 5030639, upload-time = "2025-10-16T10:34:55.257Z" }, + { url = "https://files.pythonhosted.org/packages/e5/ea/fbb258a98863f99befb10ed727152b4ae659f322e1d9c0576f8a62754e81/h5py-3.15.1-cp313-cp313-win_amd64.whl", hash = "sha256:dea78b092fd80a083563ed79a3171258d4a4d307492e7cf8b2313d464c82ba52", size = 2864363, upload-time = "2025-10-16T10:34:58.099Z" }, + { url = "https://files.pythonhosted.org/packages/5d/c9/35021cc9cd2b2915a7da3026e3d77a05bed1144a414ff840953b33937fb9/h5py-3.15.1-cp313-cp313-win_arm64.whl", hash = "sha256:c256254a8a81e2bddc0d376e23e2a6d2dc8a1e8a2261835ed8c1281a0744cd97", size = 2449570, upload-time = "2025-10-16T10:35:00.473Z" }, { url = "https://files.pythonhosted.org/packages/a0/2c/926eba1514e4d2e47d0e9eb16c784e717d8b066398ccfca9b283917b1bfb/h5py-3.15.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5f4fb0567eb8517c3ecd6b3c02c4f4e9da220c8932604960fd04e24ee1254763", size = 3380368, upload-time = "2025-10-16T10:35:03.117Z" }, { url = "https://files.pythonhosted.org/packages/65/4b/d715ed454d3baa5f6ae1d30b7eca4c7a1c1084f6a2edead9e801a1541d62/h5py-3.15.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:954e480433e82d3872503104f9b285d369048c3a788b2b1a00e53d1c47c98dd2", size = 2833793, upload-time = "2025-10-16T10:35:05.623Z" }, { url = "https://files.pythonhosted.org/packages/ef/d4/ef386c28e4579314610a8bffebbee3b69295b0237bc967340b7c653c6c10/h5py-3.15.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fd125c131889ebbef0849f4a0e29cf363b48aba42f228d08b4079913b576bb3a", size = 4903199, upload-time = "2025-10-16T10:35:08.972Z" }, @@ -32,12 +149,216 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d3/b7/4a806f85d62c20157e62e58e03b27513dc9c55499768530acc4f4c5ce4be/h5py-3.15.1-cp314-cp314-win_arm64.whl", hash = "sha256:a6d8c5a05a76aca9a494b4c53ce8a9c29023b7f64f625c6ce1841e92a362ccdf", size = 2465544, upload-time = "2025-10-16T10:35:25.695Z" }, ] +[[package]] +name = "numpy" +version = "1.24.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/a4/9b/027bec52c633f6556dba6b722d9a0befb40498b9ceddd29cbe67a45a127c/numpy-1.24.4.tar.gz", hash = "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463", size = 10911229, upload-time = "2023-06-26T13:39:33.218Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/80/6cdfb3e275d95155a34659163b83c09e3a3ff9f1456880bec6cc63d71083/numpy-1.24.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0bfb52d2169d58c1cdb8cc1f16989101639b34c7d3ce60ed70b19c63eba0b64", size = 19789140, upload-time = "2023-06-26T13:22:33.184Z" }, + { url = "https://files.pythonhosted.org/packages/64/5f/3f01d753e2175cfade1013eea08db99ba1ee4bdb147ebcf3623b75d12aa7/numpy-1.24.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1", size = 13854297, upload-time = "2023-06-26T13:22:59.541Z" }, + { url = "https://files.pythonhosted.org/packages/5a/b3/2f9c21d799fa07053ffa151faccdceeb69beec5a010576b8991f614021f7/numpy-1.24.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79fc682a374c4a8ed08b331bef9c5f582585d1048fa6d80bc6c35bc384eee9b4", size = 13995611, upload-time = "2023-06-26T13:23:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/10/be/ae5bf4737cb79ba437879915791f6f26d92583c738d7d960ad94e5c36adf/numpy-1.24.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ffe43c74893dbf38c2b0a1f5428760a1a9c98285553c89e12d70a96a7f3a4d6", size = 17282357, upload-time = "2023-06-26T13:23:51.446Z" }, + { url = "https://files.pythonhosted.org/packages/c0/64/908c1087be6285f40e4b3e79454552a701664a079321cff519d8c7051d06/numpy-1.24.4-cp310-cp310-win32.whl", hash = "sha256:4c21decb6ea94057331e111a5bed9a79d335658c27ce2adb580fb4d54f2ad9bc", size = 12429222, upload-time = "2023-06-26T13:24:13.849Z" }, + { url = "https://files.pythonhosted.org/packages/22/55/3d5a7c1142e0d9329ad27cece17933b0e2ab4e54ddc5c1861fbfeb3f7693/numpy-1.24.4-cp310-cp310-win_amd64.whl", hash = "sha256:b4bea75e47d9586d31e892a7401f76e909712a0fd510f58f5337bea9572c571e", size = 14841514, upload-time = "2023-06-26T13:24:38.129Z" }, + { url = "https://files.pythonhosted.org/packages/a9/cc/5ed2280a27e5dab12994c884f1f4d8c3bd4d885d02ae9e52a9d213a6a5e2/numpy-1.24.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f136bab9c2cfd8da131132c2cf6cc27331dd6fae65f95f69dcd4ae3c3639c810", size = 19775508, upload-time = "2023-06-26T13:25:08.882Z" }, + { url = "https://files.pythonhosted.org/packages/c0/bc/77635c657a3668cf652806210b8662e1aff84b818a55ba88257abf6637a8/numpy-1.24.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2926dac25b313635e4d6cf4dc4e51c8c0ebfed60b801c799ffc4c32bf3d1254", size = 13840033, upload-time = "2023-06-26T13:25:33.417Z" }, + { url = "https://files.pythonhosted.org/packages/a7/4c/96cdaa34f54c05e97c1c50f39f98d608f96f0677a6589e64e53104e22904/numpy-1.24.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:222e40d0e2548690405b0b3c7b21d1169117391c2e82c378467ef9ab4c8f0da7", size = 13991951, upload-time = "2023-06-26T13:25:55.725Z" }, + { url = "https://files.pythonhosted.org/packages/22/97/dfb1a31bb46686f09e68ea6ac5c63fdee0d22d7b23b8f3f7ea07712869ef/numpy-1.24.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7215847ce88a85ce39baf9e89070cb860c98fdddacbaa6c0da3ffb31b3350bd5", size = 17278923, upload-time = "2023-06-26T13:26:25.658Z" }, + { url = "https://files.pythonhosted.org/packages/35/e2/76a11e54139654a324d107da1d98f99e7aa2a7ef97cfd7c631fba7dbde71/numpy-1.24.4-cp311-cp311-win32.whl", hash = "sha256:4979217d7de511a8d57f4b4b5b2b965f707768440c17cb70fbf254c4b225238d", size = 12422446, upload-time = "2023-06-26T13:26:49.302Z" }, + { url = "https://files.pythonhosted.org/packages/d8/ec/ebef2f7d7c28503f958f0f8b992e7ce606fb74f9e891199329d5f5f87404/numpy-1.24.4-cp311-cp311-win_amd64.whl", hash = "sha256:b7b1fc9864d7d39e28f41d089bfd6353cb5f27ecd9905348c24187a768c79694", size = 14834466, upload-time = "2023-06-26T13:27:16.029Z" }, + { url = "https://files.pythonhosted.org/packages/11/10/943cfb579f1a02909ff96464c69893b1d25be3731b5d3652c2e0cf1281ea/numpy-1.24.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1452241c290f3e2a312c137a9999cdbf63f78864d63c79039bda65ee86943f61", size = 19780722, upload-time = "2023-06-26T13:27:49.573Z" }, + { url = "https://files.pythonhosted.org/packages/a7/ae/f53b7b265fdc701e663fbb322a8e9d4b14d9cb7b2385f45ddfabfc4327e4/numpy-1.24.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:04640dab83f7c6c85abf9cd729c5b65f1ebd0ccf9de90b270cd61935eef0197f", size = 13843102, upload-time = "2023-06-26T13:28:12.288Z" }, + { url = "https://files.pythonhosted.org/packages/25/6f/2586a50ad72e8dbb1d8381f837008a0321a3516dfd7cb57fc8cf7e4bb06b/numpy-1.24.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5425b114831d1e77e4b5d812b69d11d962e104095a5b9c3b641a218abcc050e", size = 14039616, upload-time = "2023-06-26T13:28:35.659Z" }, + { url = "https://files.pythonhosted.org/packages/98/5d/5738903efe0ecb73e51eb44feafba32bdba2081263d40c5043568ff60faf/numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd80e219fd4c71fc3699fc1dadac5dcf4fd882bfc6f7ec53d30fa197b8ee22dc", size = 17316263, upload-time = "2023-06-26T13:29:09.272Z" }, + { url = "https://files.pythonhosted.org/packages/d1/57/8d328f0b91c733aa9aa7ee540dbc49b58796c862b4fbcb1146c701e888da/numpy-1.24.4-cp38-cp38-win32.whl", hash = "sha256:4602244f345453db537be5314d3983dbf5834a9701b7723ec28923e2889e0bb2", size = 12455660, upload-time = "2023-06-26T13:29:33.434Z" }, + { url = "https://files.pythonhosted.org/packages/69/65/0d47953afa0ad569d12de5f65d964321c208492064c38fe3b0b9744f8d44/numpy-1.24.4-cp38-cp38-win_amd64.whl", hash = "sha256:692f2e0f55794943c5bfff12b3f56f99af76f902fc47487bdfe97856de51a706", size = 14868112, upload-time = "2023-06-26T13:29:58.385Z" }, + { url = "https://files.pythonhosted.org/packages/9a/cd/d5b0402b801c8a8b56b04c1e85c6165efab298d2f0ab741c2406516ede3a/numpy-1.24.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2541312fbf09977f3b3ad449c4e5f4bb55d0dbf79226d7724211acc905049400", size = 19816549, upload-time = "2023-06-26T13:30:36.976Z" }, + { url = "https://files.pythonhosted.org/packages/14/27/638aaa446f39113a3ed38b37a66243e21b38110d021bfcb940c383e120f2/numpy-1.24.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9667575fb6d13c95f1b36aca12c5ee3356bf001b714fc354eb5465ce1609e62f", size = 13879950, upload-time = "2023-06-26T13:31:01.787Z" }, + { url = "https://files.pythonhosted.org/packages/8f/27/91894916e50627476cff1a4e4363ab6179d01077d71b9afed41d9e1f18bf/numpy-1.24.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a86ed21e4f87050382c7bc96571755193c4c1392490744ac73d660e8f564a9", size = 14030228, upload-time = "2023-06-26T13:31:26.696Z" }, + { url = "https://files.pythonhosted.org/packages/7a/7c/d7b2a0417af6428440c0ad7cb9799073e507b1a465f827d058b826236964/numpy-1.24.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d11efb4dbecbdf22508d55e48d9c8384db795e1b7b51ea735289ff96613ff74d", size = 17311170, upload-time = "2023-06-26T13:31:56.615Z" }, + { url = "https://files.pythonhosted.org/packages/18/9d/e02ace5d7dfccee796c37b995c63322674daf88ae2f4a4724c5dd0afcc91/numpy-1.24.4-cp39-cp39-win32.whl", hash = "sha256:6620c0acd41dbcb368610bb2f4d83145674040025e5536954782467100aa8835", size = 12454918, upload-time = "2023-06-26T13:32:16.8Z" }, + { url = "https://files.pythonhosted.org/packages/63/38/6cc19d6b8bfa1d1a459daf2b3fe325453153ca7019976274b6f33d8b5663/numpy-1.24.4-cp39-cp39-win_amd64.whl", hash = "sha256:befe2bf740fd8373cf56149a5c23a0f601e82869598d41f8e188a0e9869926f8", size = 14867441, upload-time = "2023-06-26T13:32:40.521Z" }, + { url = "https://files.pythonhosted.org/packages/a4/fd/8dff40e25e937c94257455c237b9b6bf5a30d42dd1cc11555533be099492/numpy-1.24.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:31f13e25b4e304632a4619d0e0777662c2ffea99fcae2029556b17d8ff958aef", size = 19156590, upload-time = "2023-06-26T13:33:10.36Z" }, + { url = "https://files.pythonhosted.org/packages/42/e7/4bf953c6e05df90c6d351af69966384fed8e988d0e8c54dad7103b59f3ba/numpy-1.24.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95f7ac6540e95bc440ad77f56e520da5bf877f87dca58bd095288dce8940532a", size = 16705744, upload-time = "2023-06-26T13:33:36.703Z" }, + { url = "https://files.pythonhosted.org/packages/fc/dd/9106005eb477d022b60b3817ed5937a43dad8fd1f20b0610ea8a32fcb407/numpy-1.24.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e98f220aa76ca2a977fe435f5b04d7b3470c0a2e6312907b37ba6068f26787f2", size = 14734290, upload-time = "2023-06-26T13:34:05.409Z" }, +] + +[[package]] +name = "numpy" +version = "2.0.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.9.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/75/10dd1f8116a8b796cb2c737b674e02d02e80454bda953fa7e65d8c12b016/numpy-2.0.2.tar.gz", hash = "sha256:883c987dee1880e2a864ab0dc9892292582510604156762362d9326444636e78", size = 18902015, upload-time = "2024-08-26T20:19:40.945Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/21/91/3495b3237510f79f5d81f2508f9f13fea78ebfdf07538fc7444badda173d/numpy-2.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:51129a29dbe56f9ca83438b706e2e69a39892b5eda6cedcb6b0c9fdc9b0d3ece", size = 21165245, upload-time = "2024-08-26T20:04:14.625Z" }, + { url = "https://files.pythonhosted.org/packages/05/33/26178c7d437a87082d11019292dce6d3fe6f0e9026b7b2309cbf3e489b1d/numpy-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f15975dfec0cf2239224d80e32c3170b1d168335eaedee69da84fbe9f1f9cd04", size = 13738540, upload-time = "2024-08-26T20:04:36.784Z" }, + { url = "https://files.pythonhosted.org/packages/ec/31/cc46e13bf07644efc7a4bf68df2df5fb2a1a88d0cd0da9ddc84dc0033e51/numpy-2.0.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:8c5713284ce4e282544c68d1c3b2c7161d38c256d2eefc93c1d683cf47683e66", size = 5300623, upload-time = "2024-08-26T20:04:46.491Z" }, + { url = "https://files.pythonhosted.org/packages/6e/16/7bfcebf27bb4f9d7ec67332ffebee4d1bf085c84246552d52dbb548600e7/numpy-2.0.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:becfae3ddd30736fe1889a37f1f580e245ba79a5855bff5f2a29cb3ccc22dd7b", size = 6901774, upload-time = "2024-08-26T20:04:58.173Z" }, + { url = "https://files.pythonhosted.org/packages/f9/a3/561c531c0e8bf082c5bef509d00d56f82e0ea7e1e3e3a7fc8fa78742a6e5/numpy-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2da5960c3cf0df7eafefd806d4e612c5e19358de82cb3c343631188991566ccd", size = 13907081, upload-time = "2024-08-26T20:05:19.098Z" }, + { url = "https://files.pythonhosted.org/packages/fa/66/f7177ab331876200ac7563a580140643d1179c8b4b6a6b0fc9838de2a9b8/numpy-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:496f71341824ed9f3d2fd36cf3ac57ae2e0165c143b55c3a035ee219413f3318", size = 19523451, upload-time = "2024-08-26T20:05:47.479Z" }, + { url = "https://files.pythonhosted.org/packages/25/7f/0b209498009ad6453e4efc2c65bcdf0ae08a182b2b7877d7ab38a92dc542/numpy-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a61ec659f68ae254e4d237816e33171497e978140353c0c2038d46e63282d0c8", size = 19927572, upload-time = "2024-08-26T20:06:17.137Z" }, + { url = "https://files.pythonhosted.org/packages/3e/df/2619393b1e1b565cd2d4c4403bdd979621e2c4dea1f8532754b2598ed63b/numpy-2.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d731a1c6116ba289c1e9ee714b08a8ff882944d4ad631fd411106a30f083c326", size = 14400722, upload-time = "2024-08-26T20:06:39.16Z" }, + { url = "https://files.pythonhosted.org/packages/22/ad/77e921b9f256d5da36424ffb711ae79ca3f451ff8489eeca544d0701d74a/numpy-2.0.2-cp310-cp310-win32.whl", hash = "sha256:984d96121c9f9616cd33fbd0618b7f08e0cfc9600a7ee1d6fd9b239186d19d97", size = 6472170, upload-time = "2024-08-26T20:06:50.361Z" }, + { url = "https://files.pythonhosted.org/packages/10/05/3442317535028bc29cf0c0dd4c191a4481e8376e9f0db6bcf29703cadae6/numpy-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:c7b0be4ef08607dd04da4092faee0b86607f111d5ae68036f16cc787e250a131", size = 15905558, upload-time = "2024-08-26T20:07:13.881Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cf/034500fb83041aa0286e0fb16e7c76e5c8b67c0711bb6e9e9737a717d5fe/numpy-2.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:49ca4decb342d66018b01932139c0961a8f9ddc7589611158cb3c27cbcf76448", size = 21169137, upload-time = "2024-08-26T20:07:45.345Z" }, + { url = "https://files.pythonhosted.org/packages/4a/d9/32de45561811a4b87fbdee23b5797394e3d1504b4a7cf40c10199848893e/numpy-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:11a76c372d1d37437857280aa142086476136a8c0f373b2e648ab2c8f18fb195", size = 13703552, upload-time = "2024-08-26T20:08:06.666Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ca/2f384720020c7b244d22508cb7ab23d95f179fcfff33c31a6eeba8d6c512/numpy-2.0.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:807ec44583fd708a21d4a11d94aedf2f4f3c3719035c76a2bbe1fe8e217bdc57", size = 5298957, upload-time = "2024-08-26T20:08:15.83Z" }, + { url = "https://files.pythonhosted.org/packages/0e/78/a3e4f9fb6aa4e6fdca0c5428e8ba039408514388cf62d89651aade838269/numpy-2.0.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8cafab480740e22f8d833acefed5cc87ce276f4ece12fdaa2e8903db2f82897a", size = 6905573, upload-time = "2024-08-26T20:08:27.185Z" }, + { url = "https://files.pythonhosted.org/packages/a0/72/cfc3a1beb2caf4efc9d0b38a15fe34025230da27e1c08cc2eb9bfb1c7231/numpy-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a15f476a45e6e5a3a79d8a14e62161d27ad897381fecfa4a09ed5322f2085669", size = 13914330, upload-time = "2024-08-26T20:08:48.058Z" }, + { url = "https://files.pythonhosted.org/packages/ba/a8/c17acf65a931ce551fee11b72e8de63bf7e8a6f0e21add4c937c83563538/numpy-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13e689d772146140a252c3a28501da66dfecd77490b498b168b501835041f951", size = 19534895, upload-time = "2024-08-26T20:09:16.536Z" }, + { url = "https://files.pythonhosted.org/packages/ba/86/8767f3d54f6ae0165749f84648da9dcc8cd78ab65d415494962c86fac80f/numpy-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9ea91dfb7c3d1c56a0e55657c0afb38cf1eeae4544c208dc465c3c9f3a7c09f9", size = 19937253, upload-time = "2024-08-26T20:09:46.263Z" }, + { url = "https://files.pythonhosted.org/packages/df/87/f76450e6e1c14e5bb1eae6836478b1028e096fd02e85c1c37674606ab752/numpy-2.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c1c9307701fec8f3f7a1e6711f9089c06e6284b3afbbcd259f7791282d660a15", size = 14414074, upload-time = "2024-08-26T20:10:08.483Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ca/0f0f328e1e59f73754f06e1adfb909de43726d4f24c6a3f8805f34f2b0fa/numpy-2.0.2-cp311-cp311-win32.whl", hash = "sha256:a392a68bd329eafac5817e5aefeb39038c48b671afd242710b451e76090e81f4", size = 6470640, upload-time = "2024-08-26T20:10:19.732Z" }, + { url = "https://files.pythonhosted.org/packages/eb/57/3a3f14d3a759dcf9bf6e9eda905794726b758819df4663f217d658a58695/numpy-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:286cd40ce2b7d652a6f22efdfc6d1edf879440e53e76a75955bc0c826c7e64dc", size = 15910230, upload-time = "2024-08-26T20:10:43.413Z" }, + { url = "https://files.pythonhosted.org/packages/45/40/2e117be60ec50d98fa08c2f8c48e09b3edea93cfcabd5a9ff6925d54b1c2/numpy-2.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:df55d490dea7934f330006d0f81e8551ba6010a5bf035a249ef61a94f21c500b", size = 20895803, upload-time = "2024-08-26T20:11:13.916Z" }, + { url = "https://files.pythonhosted.org/packages/46/92/1b8b8dee833f53cef3e0a3f69b2374467789e0bb7399689582314df02651/numpy-2.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8df823f570d9adf0978347d1f926b2a867d5608f434a7cff7f7908c6570dcf5e", size = 13471835, upload-time = "2024-08-26T20:11:34.779Z" }, + { url = "https://files.pythonhosted.org/packages/7f/19/e2793bde475f1edaea6945be141aef6c8b4c669b90c90a300a8954d08f0a/numpy-2.0.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9a92ae5c14811e390f3767053ff54eaee3bf84576d99a2456391401323f4ec2c", size = 5038499, upload-time = "2024-08-26T20:11:43.902Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ff/ddf6dac2ff0dd50a7327bcdba45cb0264d0e96bb44d33324853f781a8f3c/numpy-2.0.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a842d573724391493a97a62ebbb8e731f8a5dcc5d285dfc99141ca15a3302d0c", size = 6633497, upload-time = "2024-08-26T20:11:55.09Z" }, + { url = "https://files.pythonhosted.org/packages/72/21/67f36eac8e2d2cd652a2e69595a54128297cdcb1ff3931cfc87838874bd4/numpy-2.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05e238064fc0610c840d1cf6a13bf63d7e391717d247f1bf0318172e759e692", size = 13621158, upload-time = "2024-08-26T20:12:14.95Z" }, + { url = "https://files.pythonhosted.org/packages/39/68/e9f1126d757653496dbc096cb429014347a36b228f5a991dae2c6b6cfd40/numpy-2.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0123ffdaa88fa4ab64835dcbde75dcdf89c453c922f18dced6e27c90d1d0ec5a", size = 19236173, upload-time = "2024-08-26T20:12:44.049Z" }, + { url = "https://files.pythonhosted.org/packages/d1/e9/1f5333281e4ebf483ba1c888b1d61ba7e78d7e910fdd8e6499667041cc35/numpy-2.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:96a55f64139912d61de9137f11bf39a55ec8faec288c75a54f93dfd39f7eb40c", size = 19634174, upload-time = "2024-08-26T20:13:13.634Z" }, + { url = "https://files.pythonhosted.org/packages/71/af/a469674070c8d8408384e3012e064299f7a2de540738a8e414dcfd639996/numpy-2.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec9852fb39354b5a45a80bdab5ac02dd02b15f44b3804e9f00c556bf24b4bded", size = 14099701, upload-time = "2024-08-26T20:13:34.851Z" }, + { url = "https://files.pythonhosted.org/packages/d0/3d/08ea9f239d0e0e939b6ca52ad403c84a2bce1bde301a8eb4888c1c1543f1/numpy-2.0.2-cp312-cp312-win32.whl", hash = "sha256:671bec6496f83202ed2d3c8fdc486a8fc86942f2e69ff0e986140339a63bcbe5", size = 6174313, upload-time = "2024-08-26T20:13:45.653Z" }, + { url = "https://files.pythonhosted.org/packages/b2/b5/4ac39baebf1fdb2e72585c8352c56d063b6126be9fc95bd2bb5ef5770c20/numpy-2.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:cfd41e13fdc257aa5778496b8caa5e856dc4896d4ccf01841daee1d96465467a", size = 15606179, upload-time = "2024-08-26T20:14:08.786Z" }, + { url = "https://files.pythonhosted.org/packages/43/c1/41c8f6df3162b0c6ffd4437d729115704bd43363de0090c7f913cfbc2d89/numpy-2.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9059e10581ce4093f735ed23f3b9d283b9d517ff46009ddd485f1747eb22653c", size = 21169942, upload-time = "2024-08-26T20:14:40.108Z" }, + { url = "https://files.pythonhosted.org/packages/39/bc/fd298f308dcd232b56a4031fd6ddf11c43f9917fbc937e53762f7b5a3bb1/numpy-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:423e89b23490805d2a5a96fe40ec507407b8ee786d66f7328be214f9679df6dd", size = 13711512, upload-time = "2024-08-26T20:15:00.985Z" }, + { url = "https://files.pythonhosted.org/packages/96/ff/06d1aa3eeb1c614eda245c1ba4fb88c483bee6520d361641331872ac4b82/numpy-2.0.2-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:2b2955fa6f11907cf7a70dab0d0755159bca87755e831e47932367fc8f2f2d0b", size = 5306976, upload-time = "2024-08-26T20:15:10.876Z" }, + { url = "https://files.pythonhosted.org/packages/2d/98/121996dcfb10a6087a05e54453e28e58694a7db62c5a5a29cee14c6e047b/numpy-2.0.2-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:97032a27bd9d8988b9a97a8c4d2c9f2c15a81f61e2f21404d7e8ef00cb5be729", size = 6906494, upload-time = "2024-08-26T20:15:22.055Z" }, + { url = "https://files.pythonhosted.org/packages/15/31/9dffc70da6b9bbf7968f6551967fc21156207366272c2a40b4ed6008dc9b/numpy-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e795a8be3ddbac43274f18588329c72939870a16cae810c2b73461c40718ab1", size = 13912596, upload-time = "2024-08-26T20:15:42.452Z" }, + { url = "https://files.pythonhosted.org/packages/b9/14/78635daab4b07c0930c919d451b8bf8c164774e6a3413aed04a6d95758ce/numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26b258c385842546006213344c50655ff1555a9338e2e5e02a0756dc3e803dd", size = 19526099, upload-time = "2024-08-26T20:16:11.048Z" }, + { url = "https://files.pythonhosted.org/packages/26/4c/0eeca4614003077f68bfe7aac8b7496f04221865b3a5e7cb230c9d055afd/numpy-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fec9451a7789926bcf7c2b8d187292c9f93ea30284802a0ab3f5be8ab36865d", size = 19932823, upload-time = "2024-08-26T20:16:40.171Z" }, + { url = "https://files.pythonhosted.org/packages/f1/46/ea25b98b13dccaebddf1a803f8c748680d972e00507cd9bc6dcdb5aa2ac1/numpy-2.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9189427407d88ff25ecf8f12469d4d39d35bee1db5d39fc5c168c6f088a6956d", size = 14404424, upload-time = "2024-08-26T20:17:02.604Z" }, + { url = "https://files.pythonhosted.org/packages/c8/a6/177dd88d95ecf07e722d21008b1b40e681a929eb9e329684d449c36586b2/numpy-2.0.2-cp39-cp39-win32.whl", hash = "sha256:905d16e0c60200656500c95b6b8dca5d109e23cb24abc701d41c02d74c6b3afa", size = 6476809, upload-time = "2024-08-26T20:17:13.553Z" }, + { url = "https://files.pythonhosted.org/packages/ea/2b/7fc9f4e7ae5b507c1a3a21f0f15ed03e794c1242ea8a242ac158beb56034/numpy-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:a3f4ab0caa7f053f6797fcd4e1e25caee367db3112ef2b6ef82d749530768c73", size = 15911314, upload-time = "2024-08-26T20:17:36.72Z" }, + { url = "https://files.pythonhosted.org/packages/8f/3b/df5a870ac6a3be3a86856ce195ef42eec7ae50d2a202be1f5a4b3b340e14/numpy-2.0.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7f0a0c6f12e07fa94133c8a67404322845220c06a9e80e85999afe727f7438b8", size = 21025288, upload-time = "2024-08-26T20:18:07.732Z" }, + { url = "https://files.pythonhosted.org/packages/2c/97/51af92f18d6f6f2d9ad8b482a99fb74e142d71372da5d834b3a2747a446e/numpy-2.0.2-pp39-pypy39_pp73-macosx_14_0_x86_64.whl", hash = "sha256:312950fdd060354350ed123c0e25a71327d3711584beaef30cdaa93320c392d4", size = 6762793, upload-time = "2024-08-26T20:18:19.125Z" }, + { url = "https://files.pythonhosted.org/packages/12/46/de1fbd0c1b5ccaa7f9a005b66761533e2f6a3e560096682683a223631fe9/numpy-2.0.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26df23238872200f63518dd2aa984cfca675d82469535dc7162dc2ee52d9dd5c", size = 19334885, upload-time = "2024-08-26T20:18:47.237Z" }, + { url = "https://files.pythonhosted.org/packages/cc/dc/d330a6faefd92b446ec0f0dfea4c3207bb1fef3c4771d19cf4543efd2c78/numpy-2.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a46288ec55ebbd58947d31d72be2c63cbf839f0a63b49cb755022310792a3385", size = 15828784, upload-time = "2024-08-26T20:19:11.19Z" }, +] + +[[package]] +name = "numpy" +version = "2.2.6" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.10.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/3e/ed6db5be21ce87955c0cbd3009f2803f59fa08df21b5df06862e2d8e2bdd/numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb", size = 21165245, upload-time = "2025-05-17T21:27:58.555Z" }, + { url = "https://files.pythonhosted.org/packages/22/c2/4b9221495b2a132cc9d2eb862e21d42a009f5a60e45fc44b00118c174bff/numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90", size = 14360048, upload-time = "2025-05-17T21:28:21.406Z" }, + { url = "https://files.pythonhosted.org/packages/fd/77/dc2fcfc66943c6410e2bf598062f5959372735ffda175b39906d54f02349/numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163", size = 5340542, upload-time = "2025-05-17T21:28:30.931Z" }, + { url = "https://files.pythonhosted.org/packages/7a/4f/1cb5fdc353a5f5cc7feb692db9b8ec2c3d6405453f982435efc52561df58/numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf", size = 6878301, upload-time = "2025-05-17T21:28:41.613Z" }, + { url = "https://files.pythonhosted.org/packages/eb/17/96a3acd228cec142fcb8723bd3cc39c2a474f7dcf0a5d16731980bcafa95/numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83", size = 14297320, upload-time = "2025-05-17T21:29:02.78Z" }, + { url = "https://files.pythonhosted.org/packages/b4/63/3de6a34ad7ad6646ac7d2f55ebc6ad439dbbf9c4370017c50cf403fb19b5/numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915", size = 16801050, upload-time = "2025-05-17T21:29:27.675Z" }, + { url = "https://files.pythonhosted.org/packages/07/b6/89d837eddef52b3d0cec5c6ba0456c1bf1b9ef6a6672fc2b7873c3ec4e2e/numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680", size = 15807034, upload-time = "2025-05-17T21:29:51.102Z" }, + { url = "https://files.pythonhosted.org/packages/01/c8/dc6ae86e3c61cfec1f178e5c9f7858584049b6093f843bca541f94120920/numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289", size = 18614185, upload-time = "2025-05-17T21:30:18.703Z" }, + { url = "https://files.pythonhosted.org/packages/5b/c5/0064b1b7e7c89137b471ccec1fd2282fceaae0ab3a9550f2568782d80357/numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d", size = 6527149, upload-time = "2025-05-17T21:30:29.788Z" }, + { url = "https://files.pythonhosted.org/packages/a3/dd/4b822569d6b96c39d1215dbae0582fd99954dcbcf0c1a13c61783feaca3f/numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3", size = 12904620, upload-time = "2025-05-17T21:30:48.994Z" }, + { url = "https://files.pythonhosted.org/packages/da/a8/4f83e2aa666a9fbf56d6118faaaf5f1974d456b1823fda0a176eff722839/numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae", size = 21176963, upload-time = "2025-05-17T21:31:19.36Z" }, + { url = "https://files.pythonhosted.org/packages/b3/2b/64e1affc7972decb74c9e29e5649fac940514910960ba25cd9af4488b66c/numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a", size = 14406743, upload-time = "2025-05-17T21:31:41.087Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9f/0121e375000b5e50ffdd8b25bf78d8e1a5aa4cca3f185d41265198c7b834/numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42", size = 5352616, upload-time = "2025-05-17T21:31:50.072Z" }, + { url = "https://files.pythonhosted.org/packages/31/0d/b48c405c91693635fbe2dcd7bc84a33a602add5f63286e024d3b6741411c/numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491", size = 6889579, upload-time = "2025-05-17T21:32:01.712Z" }, + { url = "https://files.pythonhosted.org/packages/52/b8/7f0554d49b565d0171eab6e99001846882000883998e7b7d9f0d98b1f934/numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a", size = 14312005, upload-time = "2025-05-17T21:32:23.332Z" }, + { url = "https://files.pythonhosted.org/packages/b3/dd/2238b898e51bd6d389b7389ffb20d7f4c10066d80351187ec8e303a5a475/numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf", size = 16821570, upload-time = "2025-05-17T21:32:47.991Z" }, + { url = "https://files.pythonhosted.org/packages/83/6c/44d0325722cf644f191042bf47eedad61c1e6df2432ed65cbe28509d404e/numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1", size = 15818548, upload-time = "2025-05-17T21:33:11.728Z" }, + { url = "https://files.pythonhosted.org/packages/ae/9d/81e8216030ce66be25279098789b665d49ff19eef08bfa8cb96d4957f422/numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab", size = 18620521, upload-time = "2025-05-17T21:33:39.139Z" }, + { url = "https://files.pythonhosted.org/packages/6a/fd/e19617b9530b031db51b0926eed5345ce8ddc669bb3bc0044b23e275ebe8/numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47", size = 6525866, upload-time = "2025-05-17T21:33:50.273Z" }, + { url = "https://files.pythonhosted.org/packages/31/0a/f354fb7176b81747d870f7991dc763e157a934c717b67b58456bc63da3df/numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303", size = 12907455, upload-time = "2025-05-17T21:34:09.135Z" }, + { url = "https://files.pythonhosted.org/packages/82/5d/c00588b6cf18e1da539b45d3598d3557084990dcc4331960c15ee776ee41/numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff", size = 20875348, upload-time = "2025-05-17T21:34:39.648Z" }, + { url = "https://files.pythonhosted.org/packages/66/ee/560deadcdde6c2f90200450d5938f63a34b37e27ebff162810f716f6a230/numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c", size = 14119362, upload-time = "2025-05-17T21:35:01.241Z" }, + { url = "https://files.pythonhosted.org/packages/3c/65/4baa99f1c53b30adf0acd9a5519078871ddde8d2339dc5a7fde80d9d87da/numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3", size = 5084103, upload-time = "2025-05-17T21:35:10.622Z" }, + { url = "https://files.pythonhosted.org/packages/cc/89/e5a34c071a0570cc40c9a54eb472d113eea6d002e9ae12bb3a8407fb912e/numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282", size = 6625382, upload-time = "2025-05-17T21:35:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/f8/35/8c80729f1ff76b3921d5c9487c7ac3de9b2a103b1cd05e905b3090513510/numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87", size = 14018462, upload-time = "2025-05-17T21:35:42.174Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3d/1e1db36cfd41f895d266b103df00ca5b3cbe965184df824dec5c08c6b803/numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249", size = 16527618, upload-time = "2025-05-17T21:36:06.711Z" }, + { url = "https://files.pythonhosted.org/packages/61/c6/03ed30992602c85aa3cd95b9070a514f8b3c33e31124694438d88809ae36/numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49", size = 15505511, upload-time = "2025-05-17T21:36:29.965Z" }, + { url = "https://files.pythonhosted.org/packages/b7/25/5761d832a81df431e260719ec45de696414266613c9ee268394dd5ad8236/numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de", size = 18313783, upload-time = "2025-05-17T21:36:56.883Z" }, + { url = "https://files.pythonhosted.org/packages/57/0a/72d5a3527c5ebffcd47bde9162c39fae1f90138c961e5296491ce778e682/numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4", size = 6246506, upload-time = "2025-05-17T21:37:07.368Z" }, + { url = "https://files.pythonhosted.org/packages/36/fa/8c9210162ca1b88529ab76b41ba02d433fd54fecaf6feb70ef9f124683f1/numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2", size = 12614190, upload-time = "2025-05-17T21:37:26.213Z" }, + { url = "https://files.pythonhosted.org/packages/f9/5c/6657823f4f594f72b5471f1db1ab12e26e890bb2e41897522d134d2a3e81/numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84", size = 20867828, upload-time = "2025-05-17T21:37:56.699Z" }, + { url = "https://files.pythonhosted.org/packages/dc/9e/14520dc3dadf3c803473bd07e9b2bd1b69bc583cb2497b47000fed2fa92f/numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b", size = 14143006, upload-time = "2025-05-17T21:38:18.291Z" }, + { url = "https://files.pythonhosted.org/packages/4f/06/7e96c57d90bebdce9918412087fc22ca9851cceaf5567a45c1f404480e9e/numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d", size = 5076765, upload-time = "2025-05-17T21:38:27.319Z" }, + { url = "https://files.pythonhosted.org/packages/73/ed/63d920c23b4289fdac96ddbdd6132e9427790977d5457cd132f18e76eae0/numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566", size = 6617736, upload-time = "2025-05-17T21:38:38.141Z" }, + { url = "https://files.pythonhosted.org/packages/85/c5/e19c8f99d83fd377ec8c7e0cf627a8049746da54afc24ef0a0cb73d5dfb5/numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f", size = 14010719, upload-time = "2025-05-17T21:38:58.433Z" }, + { url = "https://files.pythonhosted.org/packages/19/49/4df9123aafa7b539317bf6d342cb6d227e49f7a35b99c287a6109b13dd93/numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f", size = 16526072, upload-time = "2025-05-17T21:39:22.638Z" }, + { url = "https://files.pythonhosted.org/packages/b2/6c/04b5f47f4f32f7c2b0e7260442a8cbcf8168b0e1a41ff1495da42f42a14f/numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868", size = 15503213, upload-time = "2025-05-17T21:39:45.865Z" }, + { url = "https://files.pythonhosted.org/packages/17/0a/5cd92e352c1307640d5b6fec1b2ffb06cd0dabe7d7b8227f97933d378422/numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d", size = 18316632, upload-time = "2025-05-17T21:40:13.331Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3b/5cba2b1d88760ef86596ad0f3d484b1cbff7c115ae2429678465057c5155/numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd", size = 6244532, upload-time = "2025-05-17T21:43:46.099Z" }, + { url = "https://files.pythonhosted.org/packages/cb/3b/d58c12eafcb298d4e6d0d40216866ab15f59e55d148a5658bb3132311fcf/numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c", size = 12610885, upload-time = "2025-05-17T21:44:05.145Z" }, + { url = "https://files.pythonhosted.org/packages/6b/9e/4bf918b818e516322db999ac25d00c75788ddfd2d2ade4fa66f1f38097e1/numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6", size = 20963467, upload-time = "2025-05-17T21:40:44Z" }, + { url = "https://files.pythonhosted.org/packages/61/66/d2de6b291507517ff2e438e13ff7b1e2cdbdb7cb40b3ed475377aece69f9/numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda", size = 14225144, upload-time = "2025-05-17T21:41:05.695Z" }, + { url = "https://files.pythonhosted.org/packages/e4/25/480387655407ead912e28ba3a820bc69af9adf13bcbe40b299d454ec011f/numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40", size = 5200217, upload-time = "2025-05-17T21:41:15.903Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4a/6e313b5108f53dcbf3aca0c0f3e9c92f4c10ce57a0a721851f9785872895/numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8", size = 6712014, upload-time = "2025-05-17T21:41:27.321Z" }, + { url = "https://files.pythonhosted.org/packages/b7/30/172c2d5c4be71fdf476e9de553443cf8e25feddbe185e0bd88b096915bcc/numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f", size = 14077935, upload-time = "2025-05-17T21:41:49.738Z" }, + { url = "https://files.pythonhosted.org/packages/12/fb/9e743f8d4e4d3c710902cf87af3512082ae3d43b945d5d16563f26ec251d/numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa", size = 16600122, upload-time = "2025-05-17T21:42:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/12/75/ee20da0e58d3a66f204f38916757e01e33a9737d0b22373b3eb5a27358f9/numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571", size = 15586143, upload-time = "2025-05-17T21:42:37.464Z" }, + { url = "https://files.pythonhosted.org/packages/76/95/bef5b37f29fc5e739947e9ce5179ad402875633308504a52d188302319c8/numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1", size = 18385260, upload-time = "2025-05-17T21:43:05.189Z" }, + { url = "https://files.pythonhosted.org/packages/09/04/f2f83279d287407cf36a7a8053a5abe7be3622a4363337338f2585e4afda/numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff", size = 6377225, upload-time = "2025-05-17T21:43:16.254Z" }, + { url = "https://files.pythonhosted.org/packages/67/0e/35082d13c09c02c011cf21570543d202ad929d961c02a147493cb0c2bdf5/numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06", size = 12771374, upload-time = "2025-05-17T21:43:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/9e/3b/d94a75f4dbf1ef5d321523ecac21ef23a3cd2ac8b78ae2aac40873590229/numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d", size = 21040391, upload-time = "2025-05-17T21:44:35.948Z" }, + { url = "https://files.pythonhosted.org/packages/17/f4/09b2fa1b58f0fb4f7c7963a1649c64c4d315752240377ed74d9cd878f7b5/numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db", size = 6786754, upload-time = "2025-05-17T21:44:47.446Z" }, + { url = "https://files.pythonhosted.org/packages/af/30/feba75f143bdc868a1cc3f44ccfa6c4b9ec522b36458e738cd00f67b573f/numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543", size = 16643476, upload-time = "2025-05-17T21:45:11.871Z" }, + { url = "https://files.pythonhosted.org/packages/37/48/ac2a9584402fb6c0cd5b5d1a91dcf176b15760130dd386bbafdbfe3640bf/numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00", size = 12812666, upload-time = "2025-05-17T21:45:31.426Z" }, +] + [[package]] name = "numpy" version = "2.4.2" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.11'", +] sdist = { url = "https://files.pythonhosted.org/packages/57/fd/0005efbd0af48e55eb3c7208af93f2862d4b1a56cd78e84309a2d959208d/numpy-2.4.2.tar.gz", hash = "sha256:659a6107e31a83c4e33f763942275fd278b21d095094044eb35569e86a21ddae", size = 20723651, upload-time = "2026-01-31T23:13:10.135Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/44/71852273146957899753e69986246d6a176061ea183407e95418c2aa4d9a/numpy-2.4.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e7e88598032542bd49af7c4747541422884219056c268823ef6e5e89851c8825", size = 16955478, upload-time = "2026-01-31T23:10:25.623Z" }, + { url = "https://files.pythonhosted.org/packages/74/41/5d17d4058bd0cd96bcbd4d9ff0fb2e21f52702aab9a72e4a594efa18692f/numpy-2.4.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7edc794af8b36ca37ef5fcb5e0d128c7e0595c7b96a2318d1badb6fcd8ee86b1", size = 14965467, upload-time = "2026-01-31T23:10:28.186Z" }, + { url = "https://files.pythonhosted.org/packages/49/48/fb1ce8136c19452ed15f033f8aee91d5defe515094e330ce368a0647846f/numpy-2.4.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:6e9f61981ace1360e42737e2bae58b27bf28a1b27e781721047d84bd754d32e7", size = 5475172, upload-time = "2026-01-31T23:10:30.848Z" }, + { url = "https://files.pythonhosted.org/packages/40/a9/3feb49f17bbd1300dd2570432961f5c8a4ffeff1db6f02c7273bd020a4c9/numpy-2.4.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:cb7bbb88aa74908950d979eeaa24dbdf1a865e3c7e45ff0121d8f70387b55f73", size = 6805145, upload-time = "2026-01-31T23:10:32.352Z" }, + { url = "https://files.pythonhosted.org/packages/3f/39/fdf35cbd6d6e2fcad42fcf85ac04a85a0d0fbfbf34b30721c98d602fd70a/numpy-2.4.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f069069931240b3fc703f1e23df63443dbd6390614c8c44a87d96cd0ec81eb1", size = 15966084, upload-time = "2026-01-31T23:10:34.502Z" }, + { url = "https://files.pythonhosted.org/packages/1b/46/6fa4ea94f1ddf969b2ee941290cca6f1bfac92b53c76ae5f44afe17ceb69/numpy-2.4.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c02ef4401a506fb60b411467ad501e1429a3487abca4664871d9ae0b46c8ba32", size = 16899477, upload-time = "2026-01-31T23:10:37.075Z" }, + { url = "https://files.pythonhosted.org/packages/09/a1/2a424e162b1a14a5bd860a464ab4e07513916a64ab1683fae262f735ccd2/numpy-2.4.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2653de5c24910e49c2b106499803124dde62a5a1fe0eedeaecf4309a5f639390", size = 17323429, upload-time = "2026-01-31T23:10:39.704Z" }, + { url = "https://files.pythonhosted.org/packages/ce/a2/73014149ff250628df72c58204822ac01d768697913881aacf839ff78680/numpy-2.4.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1ae241bbfc6ae276f94a170b14785e561cb5e7f626b6688cf076af4110887413", size = 18635109, upload-time = "2026-01-31T23:10:41.924Z" }, + { url = "https://files.pythonhosted.org/packages/6c/0c/73e8be2f1accd56df74abc1c5e18527822067dced5ec0861b5bb882c2ce0/numpy-2.4.2-cp311-cp311-win32.whl", hash = "sha256:df1b10187212b198dd45fa943d8985a3c8cf854aed4923796e0e019e113a1bda", size = 6237915, upload-time = "2026-01-31T23:10:45.26Z" }, + { url = "https://files.pythonhosted.org/packages/76/ae/e0265e0163cf127c24c3969d29f1c4c64551a1e375d95a13d32eab25d364/numpy-2.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:b9c618d56a29c9cb1c4da979e9899be7578d2e0b3c24d52079c166324c9e8695", size = 12607972, upload-time = "2026-01-31T23:10:47.021Z" }, + { url = "https://files.pythonhosted.org/packages/29/a5/c43029af9b8014d6ea157f192652c50042e8911f4300f8f6ed3336bf437f/numpy-2.4.2-cp311-cp311-win_arm64.whl", hash = "sha256:47c5a6ed21d9452b10227e5e8a0e1c22979811cad7dcc19d8e3e2fb8fa03f1a3", size = 10485763, upload-time = "2026-01-31T23:10:50.087Z" }, + { url = "https://files.pythonhosted.org/packages/51/6e/6f394c9c77668153e14d4da83bcc247beb5952f6ead7699a1a2992613bea/numpy-2.4.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:21982668592194c609de53ba4933a7471880ccbaadcc52352694a59ecc860b3a", size = 16667963, upload-time = "2026-01-31T23:10:52.147Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f8/55483431f2b2fd015ae6ed4fe62288823ce908437ed49db5a03d15151678/numpy-2.4.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40397bda92382fcec844066efb11f13e1c9a3e2a8e8f318fb72ed8b6db9f60f1", size = 14693571, upload-time = "2026-01-31T23:10:54.789Z" }, + { url = "https://files.pythonhosted.org/packages/2f/20/18026832b1845cdc82248208dd929ca14c9d8f2bac391f67440707fff27c/numpy-2.4.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:b3a24467af63c67829bfaa61eecf18d5432d4f11992688537be59ecd6ad32f5e", size = 5203469, upload-time = "2026-01-31T23:10:57.343Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/2eb97c8a77daaba34eaa3fa7241a14ac5f51c46a6bd5911361b644c4a1e2/numpy-2.4.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:805cc8de9fd6e7a22da5aed858e0ab16be5a4db6c873dde1d7451c541553aa27", size = 6550820, upload-time = "2026-01-31T23:10:59.429Z" }, + { url = "https://files.pythonhosted.org/packages/b1/91/b97fdfd12dc75b02c44e26c6638241cc004d4079a0321a69c62f51470c4c/numpy-2.4.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d82351358ffbcdcd7b686b90742a9b86632d6c1c051016484fa0b326a0a1548", size = 15663067, upload-time = "2026-01-31T23:11:01.291Z" }, + { url = "https://files.pythonhosted.org/packages/f5/c6/a18e59f3f0b8071cc85cbc8d80cd02d68aa9710170b2553a117203d46936/numpy-2.4.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e35d3e0144137d9fdae62912e869136164534d64a169f86438bc9561b6ad49f", size = 16619782, upload-time = "2026-01-31T23:11:03.669Z" }, + { url = "https://files.pythonhosted.org/packages/b7/83/9751502164601a79e18847309f5ceec0b1446d7b6aa12305759b72cf98b2/numpy-2.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:adb6ed2ad29b9e15321d167d152ee909ec73395901b70936f029c3bc6d7f4460", size = 17013128, upload-time = "2026-01-31T23:11:05.913Z" }, + { url = "https://files.pythonhosted.org/packages/61/c4/c4066322256ec740acc1c8923a10047818691d2f8aec254798f3dd90f5f2/numpy-2.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8906e71fd8afcb76580404e2a950caef2685df3d2a57fe82a86ac8d33cc007ba", size = 18345324, upload-time = "2026-01-31T23:11:08.248Z" }, + { url = "https://files.pythonhosted.org/packages/ab/af/6157aa6da728fa4525a755bfad486ae7e3f76d4c1864138003eb84328497/numpy-2.4.2-cp312-cp312-win32.whl", hash = "sha256:ec055f6dae239a6299cace477b479cca2fc125c5675482daf1dd886933a1076f", size = 5960282, upload-time = "2026-01-31T23:11:10.497Z" }, + { url = "https://files.pythonhosted.org/packages/92/0f/7ceaaeaacb40567071e94dbf2c9480c0ae453d5bb4f52bea3892c39dc83c/numpy-2.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:209fae046e62d0ce6435fcfe3b1a10537e858249b3d9b05829e2a05218296a85", size = 12314210, upload-time = "2026-01-31T23:11:12.176Z" }, + { url = "https://files.pythonhosted.org/packages/2f/a3/56c5c604fae6dd40fa2ed3040d005fca97e91bd320d232ac9931d77ba13c/numpy-2.4.2-cp312-cp312-win_arm64.whl", hash = "sha256:fbde1b0c6e81d56f5dccd95dd4a711d9b95df1ae4009a60887e56b27e8d903fa", size = 10220171, upload-time = "2026-01-31T23:11:14.684Z" }, + { url = "https://files.pythonhosted.org/packages/a1/22/815b9fe25d1d7ae7d492152adbc7226d3eff731dffc38fe970589fcaaa38/numpy-2.4.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:25f2059807faea4b077a2b6837391b5d830864b3543627f381821c646f31a63c", size = 16663696, upload-time = "2026-01-31T23:11:17.516Z" }, + { url = "https://files.pythonhosted.org/packages/09/f0/817d03a03f93ba9c6c8993de509277d84e69f9453601915e4a69554102a1/numpy-2.4.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bd3a7a9f5847d2fb8c2c6d1c862fa109c31a9abeca1a3c2bd5a64572955b2979", size = 14688322, upload-time = "2026-01-31T23:11:19.883Z" }, + { url = "https://files.pythonhosted.org/packages/da/b4/f805ab79293c728b9a99438775ce51885fd4f31b76178767cfc718701a39/numpy-2.4.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8e4549f8a3c6d13d55041925e912bfd834285ef1dd64d6bc7d542583355e2e98", size = 5198157, upload-time = "2026-01-31T23:11:22.375Z" }, + { url = "https://files.pythonhosted.org/packages/74/09/826e4289844eccdcd64aac27d13b0fd3f32039915dd5b9ba01baae1f436c/numpy-2.4.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:aea4f66ff44dfddf8c2cffd66ba6538c5ec67d389285292fe428cb2c738c8aef", size = 6546330, upload-time = "2026-01-31T23:11:23.958Z" }, + { url = "https://files.pythonhosted.org/packages/19/fb/cbfdbfa3057a10aea5422c558ac57538e6acc87ec1669e666d32ac198da7/numpy-2.4.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3cd545784805de05aafe1dde61752ea49a359ccba9760c1e5d1c88a93bbf2b7", size = 15660968, upload-time = "2026-01-31T23:11:25.713Z" }, + { url = "https://files.pythonhosted.org/packages/04/dc/46066ce18d01645541f0186877377b9371b8fa8017fa8262002b4ef22612/numpy-2.4.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0d9b7c93578baafcbc5f0b83eaf17b79d345c6f36917ba0c67f45226911d499", size = 16607311, upload-time = "2026-01-31T23:11:28.117Z" }, + { url = "https://files.pythonhosted.org/packages/14/d9/4b5adfc39a43fa6bf918c6d544bc60c05236cc2f6339847fc5b35e6cb5b0/numpy-2.4.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f74f0f7779cc7ae07d1810aab8ac6b1464c3eafb9e283a40da7309d5e6e48fbb", size = 17012850, upload-time = "2026-01-31T23:11:30.888Z" }, + { url = "https://files.pythonhosted.org/packages/b7/20/adb6e6adde6d0130046e6fdfb7675cc62bc2f6b7b02239a09eb58435753d/numpy-2.4.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c7ac672d699bf36275c035e16b65539931347d68b70667d28984c9fb34e07fa7", size = 18334210, upload-time = "2026-01-31T23:11:33.214Z" }, + { url = "https://files.pythonhosted.org/packages/78/0e/0a73b3dff26803a8c02baa76398015ea2a5434d9b8265a7898a6028c1591/numpy-2.4.2-cp313-cp313-win32.whl", hash = "sha256:8e9afaeb0beff068b4d9cd20d322ba0ee1cecfb0b08db145e4ab4dd44a6b5110", size = 5958199, upload-time = "2026-01-31T23:11:35.385Z" }, + { url = "https://files.pythonhosted.org/packages/43/bc/6352f343522fcb2c04dbaf94cb30cca6fd32c1a750c06ad6231b4293708c/numpy-2.4.2-cp313-cp313-win_amd64.whl", hash = "sha256:7df2de1e4fba69a51c06c28f5a3de36731eb9639feb8e1cf7e4a7b0daf4cf622", size = 12310848, upload-time = "2026-01-31T23:11:38.001Z" }, + { url = "https://files.pythonhosted.org/packages/6e/8d/6da186483e308da5da1cc6918ce913dcfe14ffde98e710bfeff2a6158d4e/numpy-2.4.2-cp313-cp313-win_arm64.whl", hash = "sha256:0fece1d1f0a89c16b03442eae5c56dc0be0c7883b5d388e0c03f53019a4bfd71", size = 10221082, upload-time = "2026-01-31T23:11:40.392Z" }, + { url = "https://files.pythonhosted.org/packages/25/a1/9510aa43555b44781968935c7548a8926274f815de42ad3997e9e83680dd/numpy-2.4.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5633c0da313330fd20c484c78cdd3f9b175b55e1a766c4a174230c6b70ad8262", size = 14815866, upload-time = "2026-01-31T23:11:42.495Z" }, + { url = "https://files.pythonhosted.org/packages/36/30/6bbb5e76631a5ae46e7923dd16ca9d3f1c93cfa8d4ed79a129814a9d8db3/numpy-2.4.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:d9f64d786b3b1dd742c946c42d15b07497ed14af1a1f3ce840cce27daa0ce913", size = 5325631, upload-time = "2026-01-31T23:11:44.7Z" }, + { url = "https://files.pythonhosted.org/packages/46/00/3a490938800c1923b567b3a15cd17896e68052e2145d8662aaf3e1ffc58f/numpy-2.4.2-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:b21041e8cb6a1eb5312dd1d2f80a94d91efffb7a06b70597d44f1bd2dfc315ab", size = 6646254, upload-time = "2026-01-31T23:11:46.341Z" }, + { url = "https://files.pythonhosted.org/packages/d3/e9/fac0890149898a9b609caa5af7455a948b544746e4b8fe7c212c8edd71f8/numpy-2.4.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:00ab83c56211a1d7c07c25e3217ea6695e50a3e2f255053686b081dc0b091a82", size = 15720138, upload-time = "2026-01-31T23:11:48.082Z" }, + { url = "https://files.pythonhosted.org/packages/ea/5c/08887c54e68e1e28df53709f1893ce92932cc6f01f7c3d4dc952f61ffd4e/numpy-2.4.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2fb882da679409066b4603579619341c6d6898fc83a8995199d5249f986e8e8f", size = 16655398, upload-time = "2026-01-31T23:11:50.293Z" }, + { url = "https://files.pythonhosted.org/packages/4d/89/253db0fa0e66e9129c745e4ef25631dc37d5f1314dad2b53e907b8538e6d/numpy-2.4.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:66cb9422236317f9d44b67b4d18f44efe6e9c7f8794ac0462978513359461554", size = 17079064, upload-time = "2026-01-31T23:11:52.927Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d5/cbade46ce97c59c6c3da525e8d95b7abe8a42974a1dc5c1d489c10433e88/numpy-2.4.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0f01dcf33e73d80bd8dc0f20a71303abbafa26a19e23f6b68d1aa9990af90257", size = 18379680, upload-time = "2026-01-31T23:11:55.22Z" }, + { url = "https://files.pythonhosted.org/packages/40/62/48f99ae172a4b63d981babe683685030e8a3df4f246c893ea5c6ef99f018/numpy-2.4.2-cp313-cp313t-win32.whl", hash = "sha256:52b913ec40ff7ae845687b0b34d8d93b60cb66dcee06996dd5c99f2fc9328657", size = 6082433, upload-time = "2026-01-31T23:11:58.096Z" }, + { url = "https://files.pythonhosted.org/packages/07/38/e054a61cfe48ad9f1ed0d188e78b7e26859d0b60ef21cd9de4897cdb5326/numpy-2.4.2-cp313-cp313t-win_amd64.whl", hash = "sha256:5eea80d908b2c1f91486eb95b3fb6fab187e569ec9752ab7d9333d2e66bf2d6b", size = 12451181, upload-time = "2026-01-31T23:11:59.782Z" }, + { url = "https://files.pythonhosted.org/packages/6e/a4/a05c3a6418575e185dd84d0b9680b6bb2e2dc3e4202f036b7b4e22d6e9dc/numpy-2.4.2-cp313-cp313t-win_arm64.whl", hash = "sha256:fd49860271d52127d61197bb50b64f58454e9f578cb4b2c001a6de8b1f50b0b1", size = 10290756, upload-time = "2026-01-31T23:12:02.438Z" }, { url = "https://files.pythonhosted.org/packages/18/88/b7df6050bf18fdcfb7046286c6535cabbdd2064a3440fca3f069d319c16e/numpy-2.4.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:444be170853f1f9d528428eceb55f12918e4fda5d8805480f36a002f1415e09b", size = 16663092, upload-time = "2026-01-31T23:12:04.521Z" }, { url = "https://files.pythonhosted.org/packages/25/7a/1fee4329abc705a469a4afe6e69b1ef7e915117747886327104a8493a955/numpy-2.4.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d1240d50adff70c2a88217698ca844723068533f3f5c5fa6ee2e3220e3bdb000", size = 14698770, upload-time = "2026-01-31T23:12:06.96Z" }, { url = "https://files.pythonhosted.org/packages/fb/0b/f9e49ba6c923678ad5bc38181c08ac5e53b7a5754dbca8e581aa1a56b1ff/numpy-2.4.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:7cdde6de52fb6664b00b056341265441192d1291c130e99183ec0d4b110ff8b1", size = 5208562, upload-time = "2026-01-31T23:12:09.632Z" }, @@ -59,4 +380,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a5/55/6e1a61ded7af8df04016d81b5b02daa59f2ea9252ee0397cb9f631efe9e5/numpy-2.4.2-cp314-cp314t-win32.whl", hash = "sha256:8c50dd1fc8826f5b26a5ee4d77ca55d88a895f4e4819c7ecc2a9f5905047a443", size = 6153937, upload-time = "2026-01-31T23:12:47.229Z" }, { url = "https://files.pythonhosted.org/packages/45/aa/fa6118d1ed6d776b0983f3ceac9b1a5558e80df9365b1c3aa6d42bf9eee4/numpy-2.4.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fcf92bee92742edd401ba41135185866f7026c502617f422eb432cfeca4fe236", size = 12631844, upload-time = "2026-01-31T23:12:48.997Z" }, { url = "https://files.pythonhosted.org/packages/32/0a/2ec5deea6dcd158f254a7b372fb09cfba5719419c8d66343bab35237b3fb/numpy-2.4.2-cp314-cp314t-win_arm64.whl", hash = "sha256:1f92f53998a17265194018d1cc321b2e96e900ca52d54c7c77837b71b9465181", size = 10565379, upload-time = "2026-01-31T23:12:51.345Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f8/50e14d36d915ef64d8f8bc4a087fc8264d82c785eda6711f80ab7e620335/numpy-2.4.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:89f7268c009bc492f506abd6f5265defa7cb3f7487dc21d357c3d290add45082", size = 16833179, upload-time = "2026-01-31T23:12:53.5Z" }, + { url = "https://files.pythonhosted.org/packages/17/17/809b5cad63812058a8189e91a1e2d55a5a18fd04611dbad244e8aeae465c/numpy-2.4.2-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:e6dee3bb76aa4009d5a912180bf5b2de012532998d094acee25d9cb8dee3e44a", size = 14889755, upload-time = "2026-01-31T23:12:55.933Z" }, + { url = "https://files.pythonhosted.org/packages/3e/ea/181b9bcf7627fc8371720316c24db888dcb9829b1c0270abf3d288b2e29b/numpy-2.4.2-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:cd2bd2bbed13e213d6b55dc1d035a4f91748a7d3edc9480c13898b0353708920", size = 5399500, upload-time = "2026-01-31T23:12:58.671Z" }, + { url = "https://files.pythonhosted.org/packages/33/9f/413adf3fc955541ff5536b78fcf0754680b3c6d95103230252a2c9408d23/numpy-2.4.2-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:cf28c0c1d4c4bf00f509fa7eb02c58d7caf221b50b467bcb0d9bbf1584d5c821", size = 6714252, upload-time = "2026-01-31T23:13:00.518Z" }, + { url = "https://files.pythonhosted.org/packages/91/da/643aad274e29ccbdf42ecd94dafe524b81c87bcb56b83872d54827f10543/numpy-2.4.2-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e04ae107ac591763a47398bb45b568fc38f02dbc4aa44c063f67a131f99346cb", size = 15797142, upload-time = "2026-01-31T23:13:02.219Z" }, + { url = "https://files.pythonhosted.org/packages/66/27/965b8525e9cb5dc16481b30a1b3c21e50c7ebf6e9dbd48d0c4d0d5089c7e/numpy-2.4.2-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:602f65afdef699cda27ec0b9224ae5dc43e328f4c24c689deaf77133dbee74d0", size = 16727979, upload-time = "2026-01-31T23:13:04.62Z" }, + { url = "https://files.pythonhosted.org/packages/de/e5/b7d20451657664b07986c2f6e3be564433f5dcaf3482d68eaecd79afaf03/numpy-2.4.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be71bf1edb48ebbbf7f6337b5bfd2f895d1902f6335a5830b20141fc126ffba0", size = 12502577, upload-time = "2026-01-31T23:13:07.08Z" }, ] From b84db7f035410034f1a92b62361dd4d427f180ea Mon Sep 17 00:00:00 2001 From: Carlos Guzman <42706936+guzman109@users.noreply.github.com> Date: Wed, 25 Feb 2026 15:50:02 -0600 Subject: [PATCH 05/16] fix: fix cibuildwheel before-build conan path and skip selector --- pyproject.toml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6d89eab..3f4e449 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,17 +44,16 @@ CMAKE_TOOLCHAIN_FILE = {env = "CMAKE_TOOLCHAIN_FILE", default = ""} environment = { ARRAYMORPH_USE_CONAN = "1" } # Only build CPython, skip 32-bit and musl -skip = ["pp*", "*-win32", "*-manylinux_i686", "*-musllinux*"] +skip = ["*-win32", "*-manylinux_i686", "*-musllinux*"] # Before building: # 1. Install h5py so we can locate its bundled HDF5 shared library # 2. Set H5PY_HDF5_DIR so CMake knows which HDF5 binary to link against # 3. Install conan and resolve C++ dependencies before-build = [ - "pip install h5py", - "export H5PY_HDF5_DIR=$(python -c \"import h5py,os;d=os.path.dirname(h5py.__file__);print(os.path.join(d,'.dylibs') if os.path.exists(os.path.join(d,'.dylibs')) else os.path.join(os.path.dirname(d),'h5py.libs'))\")", - "pip install conan", - "cd {project}/arraymorph && conan install . --build=missing -of build", + """pip install h5py conan && \ + export H5PY_HDF5_DIR=$(python -c "import h5py,os;d=os.path.dirname(h5py.__file__);print(os.path.join(d,'.dylibs') if os.path.exists(os.path.join(d,'.dylibs')) else os.path.join(os.path.dirname(d),'h5py.libs'))") && \ + cd {project}/lib && conan install . --build=missing -of build""", ] From 944bbff3448fcce07eaf708246dc652a584b61ef Mon Sep 17 00:00:00 2001 From: Carlos Guzman Date: Wed, 25 Feb 2026 19:58:00 -0600 Subject: [PATCH 06/16] refactor(ci): drop cibuildwheel/manylinux for plain Ubuntu + uv - Build on ubuntu-latest/macos-latest directly - uv for Python setup, installs, build, and publish - scikit-build-core with --no-build-isolation to preserve toolchain paths - Dynamic toolchain discovery (handles cmake_layout output paths) - auditwheel/delocate repair wheels for PyPI manylinux compliance - Slim AWS SDK to S3-only, move Azure options to configure() --- ...uild-lib_array_morph-and-pypi-package.yaml | 151 ++++++++++++++++-- .python-version | 2 +- justfile | 4 +- lib/conanfile.py | 21 ++- pyproject.toml | 43 +---- 5 files changed, 155 insertions(+), 66 deletions(-) diff --git a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml index 6676732..2eaa082 100644 --- a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml +++ b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml @@ -5,7 +5,7 @@ on: branches: [main] release: types: [published] - workflow_dispatch: # Manual trigger — builds and tests only, does not publish to PyPI + workflow_dispatch: jobs: build_wheels: @@ -15,19 +15,12 @@ jobs: fail-fast: false matrix: include: - # Linux x86_64 - os: linux arch: x86_64 runner: ubuntu-latest - # Linux ARM64 - os: linux arch: aarch64 runner: ubuntu-24.04-arm - # macOS Intel - - os: macos - arch: x86_64 - runner: macos-13 - # macOS Apple Silicon - os: macos arch: arm64 runner: macos-latest @@ -39,13 +32,109 @@ jobs: - name: Set up uv uses: astral-sh/setup-uv@v5 - - name: Install cibuildwheel - run: uv tool install cibuildwheel + - name: Set up Python + run: uv python install 3.12 + + # ────────────────────────────────────────────── + # 1. System deps + # ────────────────────────────────────────────── + + - name: Install system deps (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y \ + libhdf5-dev \ + cmake ninja-build \ + pkg-config patchelf + + - name: Install system deps (macOS) + if: runner.os == 'macOS' + run: brew install hdf5 ninja + + - name: Install Python tools + run: | + uv pip install --system \ + conan scikit-build-core h5py \ + build auditwheel delocate + + # ────────────────────────────────────────────── + # 2. Conan: install C++ deps + # ────────────────────────────────────────────── + + - name: Conan install + working-directory: lib + run: | + conan profile detect --force + conan install . --build=missing -of build \ + -c tools.system.package_manager:mode=install + + - name: Find Conan toolchain + run: | + TOOLCHAIN=$(find lib/build -name "conan_toolchain.cmake" | head -1) + if [ -z "$TOOLCHAIN" ]; then + echo "ERROR: conan_toolchain.cmake not found" + find lib/build -type f -name "*.cmake" || true + exit 1 + fi + echo "CMAKE_TOOLCHAIN_FILE=$TOOLCHAIN" >> $GITHUB_ENV + echo "Found toolchain at: $TOOLCHAIN" + + # ────────────────────────────────────────────── + # 3. Discover h5py HDF5 + build wheel + # ────────────────────────────────────────────── - - name: Build wheels - run: cibuildwheel --output-dir wheelhouse - env: - CIBW_ARCHS: ${{ matrix.arch }} + - name: Discover h5py HDF5 location + run: | + H5PY_HDF5_DIR=$(python3 -c " + import h5py, os + d = os.path.dirname(h5py.__file__) + dylibs = os.path.join(d, '.dylibs') + libs = os.path.join(os.path.dirname(d), 'h5py.libs') + print(dylibs if os.path.exists(dylibs) else libs) + ") + echo "H5PY_HDF5_DIR=$H5PY_HDF5_DIR" >> $GITHUB_ENV + echo "Discovered h5py HDF5 at: $H5PY_HDF5_DIR" + ls -la "$H5PY_HDF5_DIR" + + - name: Build wheel + run: | + echo "Toolchain: $CMAKE_TOOLCHAIN_FILE" + echo "H5PY HDF5: $H5PY_HDF5_DIR" + uv build --wheel --no-build-isolation + + # ────────────────────────────────────────────── + # 4. Repair wheel for PyPI (manylinux / delocate) + # ────────────────────────────────────────────── + + - name: Repair wheel (Linux) + if: runner.os == 'Linux' + run: | + export LD_LIBRARY_PATH="${H5PY_HDF5_DIR}:${LD_LIBRARY_PATH}" + auditwheel show dist/*.whl + auditwheel repair dist/*.whl -w wheelhouse/ \ + --exclude libhdf5.so \ + --exclude libhdf5.so.310 \ + --exclude libhdf5_hl.so \ + --exclude libhdf5_hl.so.310 + + - name: Repair wheel (macOS) + if: runner.os == 'macOS' + run: | + export DYLD_LIBRARY_PATH="${H5PY_HDF5_DIR}:${DYLD_LIBRARY_PATH}" + delocate-listdeps dist/*.whl + delocate-wheel -w wheelhouse/ dist/*.whl \ + --exclude libhdf5 \ + --exclude libhdf5_hl + + # ────────────────────────────────────────────── + # 5. Test + upload + # ────────────────────────────────────────────── + + - name: Smoke test + run: | + uv pip install --system wheelhouse/*.whl --force-reinstall + python3 -c "import arraymorph; print('arraymorph imported successfully')" - name: Upload wheel artifact uses: actions/upload-artifact@v4 @@ -54,7 +143,6 @@ jobs: path: wheelhouse/*.whl retention-days: 7 - # Extract the native lib from the wheel and attach to the GitHub release - name: Extract native library from wheel if: github.event_name == 'release' shell: bash @@ -73,16 +161,39 @@ jobs: with: files: ${{ env.LIB_ARTIFACT }} + build_sdist: + name: Build sdist + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up uv + uses: astral-sh/setup-uv@v5 + + - name: Set up Python + run: uv python install 3.12 + + - name: Build sdist + run: uv build --sdist + + - name: Upload sdist artifact + uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/*.tar.gz + retention-days: 7 + publish: name: Publish to PyPI - needs: build_wheels + needs: [build_wheels, build_sdist] runs-on: ubuntu-latest if: github.event_name == 'release' environment: name: pypi url: https://pypi.org/p/arraymorph permissions: - id-token: write # Required for OIDC trusted publishing + id-token: write steps: - name: Set up uv @@ -95,5 +206,11 @@ jobs: merge-multiple: true path: dist/ + - name: Download sdist + uses: actions/download-artifact@v4 + with: + name: sdist + path: dist/ + - name: Publish to PyPI run: uv publish dist/* diff --git a/.python-version b/.python-version index cc1923a..bd28b9c 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.8 +3.9 diff --git a/justfile b/justfile index ccd9e7b..1b1889b 100644 --- a/justfile +++ b/justfile @@ -21,11 +21,11 @@ default: deps: cd lib && conan install . --build=missing -s build_type=Release -# Build Python wheel (runs scikit-build-core + CMake) +# Build Python wheel (scikit-build-core handles CMake) wheel: CMAKE_TOOLCHAIN_FILE={{ CMAKE_TOOLCHAIN_FILE }} \ H5PY_HDF5_DIR={{ H5PY_HDF5_DIR }} \ - uv build + uv build --wheel --no-build-isolation # Install editable into current venv (for development iteration) dev: diff --git a/lib/conanfile.py b/lib/conanfile.py index 0f6c4d4..be860ad 100644 --- a/lib/conanfile.py +++ b/lib/conanfile.py @@ -1,7 +1,5 @@ from conan import ConanFile from conan.tools.cmake import cmake_layout, CMakeToolchain, CMakeDeps -from conan.tools.gnu import PkgConfigDeps -from conan.tools.layout import basic_layout class ArrayMorphRecipe(ConanFile): @@ -12,15 +10,26 @@ class ArrayMorphRecipe(ConanFile): def requirements(self): self.requires("aws-sdk-cpp/1.11.692") self.requires("azure-sdk-for-cpp/1.16.1") - # self.requires("azure-storage-cpp/7.5.0") self.requires("hdf5/1.14.6") self.requires("libcurl/8.17.0") self.requires("openssl/3.6.1") def configure(self): self.options["*"].shared = False + + # AWS SDK: ONLY S3 — disable everything that pulls in + # audio (libalsa), GUI (xorg), and other unnecessary deps self.options["aws-sdk-cpp"].s3 = True self.options["aws-sdk-cpp"].text_to_speech = False + self.options["aws-sdk-cpp"].access_management = False + self.options["aws-sdk-cpp"].identity_management = False + self.options["aws-sdk-cpp"].transfer = False + self.options["aws-sdk-cpp"].queues = False + self.options["aws-sdk-cpp"].messaging = False + + # Azure SDK: only blob storage + self.options["azure-sdk-for-cpp"].with_storage_blobs = True + self.options["azure-sdk-for-cpp"].with_storage_datalake = False def layout(self): cmake_layout(self) @@ -28,7 +37,5 @@ def layout(self): def generate(self): tc = CMakeToolchain(self, generator="Ninja") tc.generate() - self.options["azure-sdk-for-cpp"].with_storage_blobs = True - self.options["azure-sdk-for-cpp"].with_storage_datalake = False - pc = CMakeDeps(self) - pc.generate() + deps = CMakeDeps(self) + deps.generate() diff --git a/pyproject.toml b/pyproject.toml index 3f4e449..7406930 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["scikit-build-core", "cmake>=3.20", "ninja"] +requires = ["scikit-build-core"] build-backend = "scikit_build_core.build" [project] @@ -9,7 +9,7 @@ description = "HDF5 VOL connector for cloud object storage (AWS S3, Azure Blob)" readme = "README.md" license = { text = "MIT" } authors = [{ name = "ICICLE AI Institute" }] -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = ["h5py>=3.11.0"] [project.urls] @@ -21,47 +21,12 @@ Issues = "https://github.com/ICICLE-ai/ArrayMorph/issues" # --- scikit-build-core settings --- [tool.scikit-build] -# CMakeLists.txt is inside arraymorph/ subdirectory cmake.source-dir = "lib" cmake.build-type = "Release" -cmake.verbose = true - -# Install the Python package from src/ +build.verbose = true wheel.packages = ["src/arraymorph"] -# Pass h5py HDF5 location to CMake -# This gets set by the before-build script in cibuildwheel [tool.scikit-build.cmake.define] CMAKE_POSITION_INDEPENDENT_CODE = "ON" CMAKE_TOOLCHAIN_FILE = {env = "CMAKE_TOOLCHAIN_FILE", default = ""} - -# --- cibuildwheel settings --- - -[tool.cibuildwheel] -# ARRAYMORPH_USE_CONAN signals CMake to use the Conan toolchain. -# H5PY_HDF5_DIR is intentionally NOT set here — h5py must be installed -# first (in before-build) before we can discover its bundled HDF5 path. -environment = { ARRAYMORPH_USE_CONAN = "1" } - -# Only build CPython, skip 32-bit and musl -skip = ["*-win32", "*-manylinux_i686", "*-musllinux*"] - -# Before building: -# 1. Install h5py so we can locate its bundled HDF5 shared library -# 2. Set H5PY_HDF5_DIR so CMake knows which HDF5 binary to link against -# 3. Install conan and resolve C++ dependencies -before-build = [ - """pip install h5py conan && \ - export H5PY_HDF5_DIR=$(python -c "import h5py,os;d=os.path.dirname(h5py.__file__);print(os.path.join(d,'.dylibs') if os.path.exists(os.path.join(d,'.dylibs')) else os.path.join(os.path.dirname(d),'h5py.libs'))") && \ - cd {project}/lib && conan install . --build=missing -of build""", -] - - -[tool.cibuildwheel.linux] -# manylinux2014 has the toolchain we need -manylinux-x86_64-image = "manylinux_2_28" -manylinux-aarch64-image = "manylinux_2_28" - -[tool.cibuildwheel.macos] -# Build for both Intel and Apple Silicon -archs = ["x86_64", "arm64"] +H5PY_HDF5_DIR = {env = "H5PY_HDF5_DIR", default = ""} From 769015b1e0d978044eb386b1951467879b8ee913 Mon Sep 17 00:00:00 2001 From: Carlos Guzman Date: Wed, 25 Feb 2026 20:14:10 -0600 Subject: [PATCH 07/16] Swithing to python 3.9 and fixing system installed pip packages in runner. --- .../workflows/build-lib_array_morph-and-pypi-package.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml index 2eaa082..489c113 100644 --- a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml +++ b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml @@ -33,7 +33,7 @@ jobs: uses: astral-sh/setup-uv@v5 - name: Set up Python - run: uv python install 3.12 + run: uv python install 3.9 # ────────────────────────────────────────────── # 1. System deps @@ -44,17 +44,16 @@ jobs: run: | sudo apt-get update sudo apt-get install -y \ - libhdf5-dev \ cmake ninja-build \ pkg-config patchelf - name: Install system deps (macOS) if: runner.os == 'macOS' - run: brew install hdf5 ninja + run: brew install ninja cmake - name: Install Python tools run: | - uv pip install --system \ + uv pip install \ conan scikit-build-core h5py \ build auditwheel delocate From 086c934dc12d78dbec41cf750161ecbfedfb093d Mon Sep 17 00:00:00 2001 From: Carlos Guzman Date: Wed, 25 Feb 2026 20:20:19 -0600 Subject: [PATCH 08/16] Moved to latest uv runner. --- .../workflows/build-lib_array_morph-and-pypi-package.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml index 489c113..149a036 100644 --- a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml +++ b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml @@ -29,8 +29,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Set up uv - uses: astral-sh/setup-uv@v5 + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.10.6" - name: Set up Python run: uv python install 3.9 @@ -53,6 +55,8 @@ jobs: - name: Install Python tools run: | + uv venv --python 3.9 + source .venv/bin/activate uv pip install \ conan scikit-build-core h5py \ build auditwheel delocate From a20b21b1ec325aa9daa7c75c60c2721d1894635a Mon Sep 17 00:00:00 2001 From: Carlos Guzman Date: Wed, 25 Feb 2026 20:21:59 -0600 Subject: [PATCH 09/16] Trying uv tool for conan --- .github/workflows/build-lib_array_morph-and-pypi-package.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml index 149a036..54ebea6 100644 --- a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml +++ b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml @@ -58,7 +58,7 @@ jobs: uv venv --python 3.9 source .venv/bin/activate uv pip install \ - conan scikit-build-core h5py \ + scikit-build-core h5py \ build auditwheel delocate # ────────────────────────────────────────────── @@ -68,6 +68,7 @@ jobs: - name: Conan install working-directory: lib run: | + uv tool install conan conan profile detect --force conan install . --build=missing -of build \ -c tools.system.package_manager:mode=install From 4db0eba45b98b5c13b4fe63e4732d5c1f7872b48 Mon Sep 17 00:00:00 2001 From: Carlos Guzman Date: Wed, 25 Feb 2026 20:30:14 -0600 Subject: [PATCH 10/16] fix(ci): add xorg/alsa system deps for Conan, persist venv PATH across steps - Pre-install X11 and ALSA dev packages (AWS SDK transitive deps) - Enable sudo for Conan system package manager - Use GITHUB_PATH instead of source activate for venv persistence --- ...uild-lib_array_morph-and-pypi-package.yaml | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml index 54ebea6..47084ef 100644 --- a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml +++ b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml @@ -47,7 +47,23 @@ jobs: sudo apt-get update sudo apt-get install -y \ cmake ninja-build \ - pkg-config patchelf + pkg-config patchelf \ + libx11-dev libx11-xcb-dev libfontenc-dev \ + libice-dev libsm-dev libxau-dev libxaw7-dev \ + libxcomposite-dev libxcursor-dev libxdamage-dev \ + libxdmcp-dev libxext-dev libxfixes-dev libxi-dev \ + libxinerama-dev libxkbfile-dev libxmu-dev \ + libxmuu-dev libxpm-dev libxrandr-dev libxrender-dev \ + libxres-dev libxss-dev libxt-dev libxtst-dev \ + libxv-dev libxxf86vm-dev libxcb-glx0-dev \ + libxcb-render0-dev libxcb-render-util0-dev \ + libxcb-xkb-dev libxcb-icccm4-dev libxcb-image0-dev \ + libxcb-keysyms1-dev libxcb-randr0-dev libxcb-shape0-dev \ + libxcb-sync-dev libxcb-xfixes0-dev libxcb-xinerama0-dev \ + libxcb-dri3-dev uuid-dev libxcb-cursor-dev \ + libxcb-dri2-0-dev libxcb-present-dev \ + libxcb-composite0-dev libxcb-ewmh-dev libxcb-res0-dev \ + libasound2-dev - name: Install system deps (macOS) if: runner.os == 'macOS' @@ -56,7 +72,7 @@ jobs: - name: Install Python tools run: | uv venv --python 3.9 - source .venv/bin/activate + echo "${{ github.workspace }}/.venv/bin" >> $GITHUB_PATH uv pip install \ scikit-build-core h5py \ build auditwheel delocate @@ -71,7 +87,8 @@ jobs: uv tool install conan conan profile detect --force conan install . --build=missing -of build \ - -c tools.system.package_manager:mode=install + -c tools.system.package_manager:mode=install \ + -c tools.system.package_manager:sudo=True - name: Find Conan toolchain run: | From b96490c34a4a11841c4d1e37e441b3313193fb60 Mon Sep 17 00:00:00 2001 From: Carlos Guzman Date: Wed, 25 Feb 2026 20:40:15 -0600 Subject: [PATCH 11/16] fix: fix Find Conan toolchain. --- .github/workflows/build-lib_array_morph-and-pypi-package.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml index 47084ef..3a4f691 100644 --- a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml +++ b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml @@ -92,10 +92,10 @@ jobs: - name: Find Conan toolchain run: | - TOOLCHAIN=$(find lib/build -name "conan_toolchain.cmake" | head -1) + TOOLCHAIN=$(find ${{ github.workspace }}/lib/build -name "conan_toolchain.cmake" | head -1) if [ -z "$TOOLCHAIN" ]; then echo "ERROR: conan_toolchain.cmake not found" - find lib/build -type f -name "*.cmake" || true + find ${{ github.workspace }}/lib/build -type f -name "*.cmake" || true exit 1 fi echo "CMAKE_TOOLCHAIN_FILE=$TOOLCHAIN" >> $GITHUB_ENV From 78dcd698bcba4032deb82fed6ce2cd5ca7d942f0 Mon Sep 17 00:00:00 2001 From: Carlos Guzman Date: Wed, 25 Feb 2026 20:48:23 -0600 Subject: [PATCH 12/16] missed system install pip package in smoke test --- .github/workflows/build-lib_array_morph-and-pypi-package.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml index 3a4f691..6884632 100644 --- a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml +++ b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml @@ -154,7 +154,7 @@ jobs: - name: Smoke test run: | - uv pip install --system wheelhouse/*.whl --force-reinstall + uv pip install wheelhouse/*.whl --force-reinstall python3 -c "import arraymorph; print('arraymorph imported successfully')" - name: Upload wheel artifact From c6c7ef23761438bc363c1a08d85419d28ab7d40c Mon Sep 17 00:00:00 2001 From: Carlos Guzman Date: Wed, 25 Feb 2026 21:03:15 -0600 Subject: [PATCH 13/16] Adding dyanmic pyproject versioning. Follow git tag. --- pyproject.toml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7406930..94669ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,10 @@ [build-system] -requires = ["scikit-build-core"] +requires = ["scikit-build-core", "setuptools-scm"] build-backend = "scikit_build_core.build" [project] name = "arraymorph" -version = "0.2.0" +dynamic = ["version"] description = "HDF5 VOL connector for cloud object storage (AWS S3, Azure Blob)" readme = "README.md" license = { text = "MIT" } @@ -25,8 +25,11 @@ cmake.source-dir = "lib" cmake.build-type = "Release" build.verbose = true wheel.packages = ["src/arraymorph"] +metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" [tool.scikit-build.cmake.define] CMAKE_POSITION_INDEPENDENT_CODE = "ON" -CMAKE_TOOLCHAIN_FILE = {env = "CMAKE_TOOLCHAIN_FILE", default = ""} -H5PY_HDF5_DIR = {env = "H5PY_HDF5_DIR", default = ""} +CMAKE_TOOLCHAIN_FILE = { env = "CMAKE_TOOLCHAIN_FILE", default = "" } +H5PY_HDF5_DIR = { env = "H5PY_HDF5_DIR", default = "" } + +[tool.setuptools_scm] From 589386375998e11dc4eabb37a535e8fb51f1e6a7 Mon Sep 17 00:00:00 2001 From: Carlos Guzman Date: Wed, 25 Feb 2026 21:32:01 -0600 Subject: [PATCH 14/16] Adding support for multiple python versions. Changed name to lib_arraymorph. --- ...uild-lib_array_morph-and-pypi-package.yaml | 79 +++++++++++++------ lib/src/CMakeLists.txt | 12 +-- 2 files changed, 59 insertions(+), 32 deletions(-) diff --git a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml index 6884632..6682ad8 100644 --- a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml +++ b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml @@ -9,25 +9,39 @@ on: jobs: build_wheels: - name: Build (${{ matrix.os }} / ${{ matrix.arch }}) + name: Build (${{ matrix.os }} / ${{ matrix.arch }} / py${{ matrix.python }}) runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: include: - - os: linux - arch: x86_64 - runner: ubuntu-latest - - os: linux - arch: aarch64 - runner: ubuntu-24.04-arm - - os: macos - arch: arm64 - runner: macos-latest + # Linux x86_64 + - { os: linux, arch: x86_64, runner: ubuntu-latest, python: "3.9" } + - { os: linux, arch: x86_64, runner: ubuntu-latest, python: "3.10" } + - { os: linux, arch: x86_64, runner: ubuntu-latest, python: "3.11" } + - { os: linux, arch: x86_64, runner: ubuntu-latest, python: "3.12" } + - { os: linux, arch: x86_64, runner: ubuntu-latest, python: "3.13" } + - { os: linux, arch: x86_64, runner: ubuntu-latest, python: "3.14" } + # Linux aarch64 + - { os: linux, arch: aarch64, runner: ubuntu-24.04-arm, python: "3.9" } + - { os: linux, arch: aarch64, runner: ubuntu-24.04-arm, python: "3.10" } + - { os: linux, arch: aarch64, runner: ubuntu-24.04-arm, python: "3.11" } + - { os: linux, arch: aarch64, runner: ubuntu-24.04-arm, python: "3.12" } + - { os: linux, arch: aarch64, runner: ubuntu-24.04-arm, python: "3.13" } + - { os: linux, arch: aarch64, runner: ubuntu-24.04-arm, python: "3.14" } + # macOS arm64 + - { os: macos, arch: arm64, runner: macos-latest, python: "3.9" } + - { os: macos, arch: arm64, runner: macos-latest, python: "3.10" } + - { os: macos, arch: arm64, runner: macos-latest, python: "3.11" } + - { os: macos, arch: arm64, runner: macos-latest, python: "3.12" } + - { os: macos, arch: arm64, runner: macos-latest, python: "3.13" } + - { os: macos, arch: arm64, runner: macos-latest, python: "3.14" } steps: - name: Checkout uses: actions/checkout@v4 + with: + fetch-depth: 0 # setuptools-scm needs full history - name: Install uv uses: astral-sh/setup-uv@v7 @@ -35,7 +49,7 @@ jobs: version: "0.10.6" - name: Set up Python - run: uv python install 3.9 + run: uv python install ${{ matrix.python }} # ────────────────────────────────────────────── # 1. System deps @@ -71,16 +85,23 @@ jobs: - name: Install Python tools run: | - uv venv --python 3.9 + uv venv --python ${{ matrix.python }} echo "${{ github.workspace }}/.venv/bin" >> $GITHUB_PATH uv pip install \ - scikit-build-core h5py \ + scikit-build-core setuptools-scm h5py \ build auditwheel delocate # ────────────────────────────────────────────── - # 2. Conan: install C++ deps + # 2. Conan: install C++ deps (cached per platform) # ────────────────────────────────────────────── + - name: Cache Conan packages + uses: actions/cache@v4 + with: + path: ~/.conan2 + key: conan-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles('lib/conanfile.py') }} + restore-keys: conan-${{ matrix.os }}-${{ matrix.arch }}- + - name: Conan install working-directory: lib run: | @@ -125,7 +146,7 @@ jobs: uv build --wheel --no-build-isolation # ────────────────────────────────────────────── - # 4. Repair wheel for PyPI (manylinux / delocate) + # 4. Repair wheel for PyPI # ────────────────────────────────────────────── - name: Repair wheel (Linux) @@ -160,24 +181,24 @@ jobs: - name: Upload wheel artifact uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.os }}-${{ matrix.arch }} + name: wheels-${{ matrix.os }}-${{ matrix.arch }}-py${{ matrix.python }} path: wheelhouse/*.whl retention-days: 7 - name: Extract native library from wheel - if: github.event_name == 'release' + if: github.event_name == 'release' && matrix.python == '3.12' shell: bash run: | wheel_file=$(ls wheelhouse/*.whl | head -1) mkdir -p extracted_lib - unzip -j "$wheel_file" "arraymorph/lib/lib_array_morph*" -d extracted_lib/ - lib_file=$(ls extracted_lib/lib_array_morph*) + unzip -j "$wheel_file" "arraymorph/lib/lib_arraymorph*" -d extracted_lib/ + lib_file=$(ls extracted_lib/lib_arraymorph*) ext="${lib_file##*.}" - cp "$lib_file" "lib_array_morph-${{ matrix.os }}-${{ matrix.arch }}.$ext" - echo "LIB_ARTIFACT=lib_array_morph-${{ matrix.os }}-${{ matrix.arch }}.$ext" >> $GITHUB_ENV + cp "$lib_file" "lib_arraymorph-${{ matrix.os }}-${{ matrix.arch }}.$ext" + echo "LIB_ARTIFACT=lib_arraymorph-${{ matrix.os }}-${{ matrix.arch }}.$ext" >> $GITHUB_ENV - name: Attach native library to GitHub release - if: github.event_name == 'release' + if: github.event_name == 'release' && matrix.python == '3.12' uses: softprops/action-gh-release@v2 with: files: ${{ env.LIB_ARTIFACT }} @@ -188,9 +209,13 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + fetch-depth: 0 - - name: Set up uv - uses: astral-sh/setup-uv@v5 + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.10.6" - name: Set up Python run: uv python install 3.12 @@ -217,8 +242,10 @@ jobs: id-token: write steps: - - name: Set up uv - uses: astral-sh/setup-uv@v5 + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.10.6" - name: Download all wheels uses: actions/download-artifact@v4 diff --git a/lib/src/CMakeLists.txt b/lib/src/CMakeLists.txt index 32dbbad..4e30db2 100644 --- a/lib/src/CMakeLists.txt +++ b/lib/src/CMakeLists.txt @@ -37,18 +37,18 @@ target_include_directories(group_callbacks PUBLIC ${PROJECT_INCLUDE_DIRS}) target_link_libraries(group_callbacks PRIVATE dataset_callbacks ${ALL_DEPS}) # 3. The Final VOL Connector (shared library) -add_library(array_morph SHARED s3vl/vol_connector.cc) -set_target_properties(array_morph PROPERTIES PREFIX "lib_") -target_include_directories(array_morph PUBLIC ${PROJECT_INCLUDE_DIRS}) -target_link_libraries(array_morph PUBLIC +add_library(arraymorph SHARED s3vl/vol_connector.cc) +set_target_properties(arraymorph PROPERTIES PREFIX "lib_") +target_include_directories(arraymorph PUBLIC ${PROJECT_INCLUDE_DIRS}) +target_link_libraries(arraymorph PUBLIC group_callbacks ${ALL_DEPS} ) # 4. Install into the Python package's lib/ directory # scikit-build-core sets CMAKE_INSTALL_PREFIX to the wheel staging area -# so this puts the .so/.dylib at: arraymorph/lib/lib_array_morph.so -install(TARGETS array_morph +# so this puts the .so/.dylib at: arraymorph/lib/lib_arraymorph.so +install(TARGETS arraymorph LIBRARY DESTINATION arraymorph/lib # .so (Linux) and .dylib (macOS) RUNTIME DESTINATION arraymorph/lib # .dll (Windows) ) From b1286525d6d9a44f9d11af89164691928db1666a Mon Sep 17 00:00:00 2001 From: Carlos Guzman Date: Wed, 25 Feb 2026 21:50:08 -0600 Subject: [PATCH 15/16] UV uses venv for python version. --- .../workflows/build-lib_array_morph-and-pypi-package.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml index 6682ad8..c920d6e 100644 --- a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml +++ b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml @@ -141,10 +141,7 @@ jobs: - name: Build wheel run: | - echo "Toolchain: $CMAKE_TOOLCHAIN_FILE" - echo "H5PY HDF5: $H5PY_HDF5_DIR" - uv build --wheel --no-build-isolation - + uv build --wheel --no-build-isolation --python ${{ github.workspace }}/.venv/bin/python # ────────────────────────────────────────────── # 4. Repair wheel for PyPI # ────────────────────────────────────────────── From 5d98a7d15c947b669857d2527b576d3fe147b570 Mon Sep 17 00:00:00 2001 From: Carlos Guzman Date: Wed, 25 Feb 2026 22:26:07 -0600 Subject: [PATCH 16/16] ci: add Python 3.9-3.14 matrix, Conan cache, TestPyPI gate, setuptools-scm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Build wheels for 18 targets (3 platforms × 6 Python versions) - Cache Conan packages per platform, shared across Python versions - Publish to TestPyPI and smoke test before pushing to real PyPI - Dynamic versioning from git tags via setuptools-scm - Rename lib_array_morph → lib_arraymorph --- ...uild-lib_array_morph-and-pypi-package.yaml | 112 ++++++++++++++++-- 1 file changed, 104 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml index c920d6e..639598e 100644 --- a/.github/workflows/build-lib_array_morph-and-pypi-package.yaml +++ b/.github/workflows/build-lib_array_morph-and-pypi-package.yaml @@ -23,12 +23,42 @@ jobs: - { os: linux, arch: x86_64, runner: ubuntu-latest, python: "3.13" } - { os: linux, arch: x86_64, runner: ubuntu-latest, python: "3.14" } # Linux aarch64 - - { os: linux, arch: aarch64, runner: ubuntu-24.04-arm, python: "3.9" } - - { os: linux, arch: aarch64, runner: ubuntu-24.04-arm, python: "3.10" } - - { os: linux, arch: aarch64, runner: ubuntu-24.04-arm, python: "3.11" } - - { os: linux, arch: aarch64, runner: ubuntu-24.04-arm, python: "3.12" } - - { os: linux, arch: aarch64, runner: ubuntu-24.04-arm, python: "3.13" } - - { os: linux, arch: aarch64, runner: ubuntu-24.04-arm, python: "3.14" } + - { + os: linux, + arch: aarch64, + runner: ubuntu-24.04-arm, + python: "3.9", + } + - { + os: linux, + arch: aarch64, + runner: ubuntu-24.04-arm, + python: "3.10", + } + - { + os: linux, + arch: aarch64, + runner: ubuntu-24.04-arm, + python: "3.11", + } + - { + os: linux, + arch: aarch64, + runner: ubuntu-24.04-arm, + python: "3.12", + } + - { + os: linux, + arch: aarch64, + runner: ubuntu-24.04-arm, + python: "3.13", + } + - { + os: linux, + arch: aarch64, + runner: ubuntu-24.04-arm, + python: "3.14", + } # macOS arm64 - { os: macos, arch: arm64, runner: macos-latest, python: "3.9" } - { os: macos, arch: arm64, runner: macos-latest, python: "3.10" } @@ -41,7 +71,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - fetch-depth: 0 # setuptools-scm needs full history + fetch-depth: 0 # setuptools-scm needs full history - name: Install uv uses: astral-sh/setup-uv@v7 @@ -227,9 +257,75 @@ jobs: path: dist/*.tar.gz retention-days: 7 + publish_test: + name: Publish to TestPyPI + needs: [build_wheels, build_sdist] + runs-on: ubuntu-latest + if: github.event_name == 'release' + permissions: + id-token: write + environment: + name: testpypi + url: https://test.pypi.org/p/arraymorph + + steps: + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.10.6" + + - name: Download all wheels + uses: actions/download-artifact@v4 + with: + pattern: wheels-* + merge-multiple: true + path: dist/ + + - name: Download sdist + uses: actions/download-artifact@v4 + with: + name: sdist + path: dist/ + + - name: Publish to TestPyPI + run: uv publish dist/* --publish-url https://test.pypi.org/legacy/ + + test_testpypi: + name: Test TestPyPI (${{ matrix.os }}) + needs: [publish_test] + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - { os: linux, runner: ubuntu-latest } + - { os: macos, runner: macos-latest } + + steps: + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.10.6" + + - name: Set up Python + run: uv python install 3.12 + + - name: Install from TestPyPI + run: | + uv venv --python 3.12 + source .venv/bin/activate + uv pip install \ + --index-url https://test.pypi.org/simple/ \ + --extra-index-url https://pypi.org/simple/ \ + arraymorph + + - name: Test + run: | + source .venv/bin/activate + python -c "import arraymorph; print('arraymorph imported successfully')" + publish: name: Publish to PyPI - needs: [build_wheels, build_sdist] + needs: [test_testpypi] # ← now waits for TestPyPI to pass runs-on: ubuntu-latest if: github.event_name == 'release' environment: