From fcb40f7401139e9beee8ed7ba0fabc702f5d5cd7 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Tue, 7 Oct 2025 21:34:50 +0200 Subject: [PATCH 1/2] Added configuration to build python wheel --- .github/workflows/python-bindings.yml | 66 ++++++++++++++++----------- .gitignore | 1 + CMakeLists.txt | 12 ++--- bindings/python_bindings.cpp | 2 +- build-requirements.txt | 1 + py_capio_cl/__init__.py | 1 + pyproject.toml | 32 +++++++++++++ tests/python/test_engine.py | 26 +---------- 8 files changed, 83 insertions(+), 58 deletions(-) create mode 100644 build-requirements.txt create mode 100644 py_capio_cl/__init__.py create mode 100644 pyproject.toml diff --git a/.github/workflows/python-bindings.yml b/.github/workflows/python-bindings.yml index 32c32d6..d862025 100644 --- a/.github/workflows/python-bindings.yml +++ b/.github/workflows/python-bindings.yml @@ -1,4 +1,5 @@ name: "Python Bindings Unit Tests" + on: push: branches: @@ -6,52 +7,65 @@ on: pull_request: branches: - main + concurrency: group: build-python-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true + jobs: python-tests: - name: "Test python bindings" + name: "Test Python bindings" strategy: matrix: - on: [ 'ubuntu-24.04', 'macos-15-intel' , 'macos-26' ] + os: [ 'ubuntu-24.04', 'macos-15-intel', 'macos-26' ] python: [ '3.10', '3.11', '3.12', '3.13' ] - runs-on: ${{matrix.on}} + runs-on: ${{ matrix.os }} env: INSTALL_PREFIX: "/usr/local" steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v6 + - name: "Checkout repository" + uses: actions/checkout@v4 + + - name: "Set up Python ${{ matrix.python }}" + uses: actions/setup-python@v6 with: - python-version: ${{matrix.python}} - - name: "Install packages ubuntu" - if: ${{ startsWith(matrix.on, 'ubuntu-') }} - run: sudo apt install -y ninja-build g++ + python-version: ${{ matrix.python }} - - name: "Setup macos runner" - if: ${{ startsWith(matrix.on, 'macos-') }} + # Install platform build dependencies + - name: "Install system packages (Ubuntu)" + if: startsWith(matrix.os, 'ubuntu-') + run: sudo apt-get update && sudo apt-get install -y ninja-build g++ cmake + + - name: "Setup Homebrew (macOS)" + if: startsWith(matrix.os, 'macos-') uses: Homebrew/actions/setup-homebrew@main - - name: "Install macos packages" - if: ${{ startsWith(matrix.on, 'macos-') }} + - name: "Install system packages (macOS)" + if: startsWith(matrix.os, 'macos-') run: brew install ninja gcc cmake - - name: "Run CMake" + # Install Python build dependencies + - name: "Install Python build tools" + run: | + python -m pip install --upgrade pip + python -m pip install -r build-requirements.txt + python -m pip install build pytest + + # Build wheel from pyproject.toml + - name: "Build Python wheel" run: | - cmake -DCMAKE_BUILD_TYPE=Release \ - -G Ninja \ - -B ../build \ - -S ${GITHUB_WORKSPACE} - cmake --build ../build -j $(nproc) - sudo cmake --install ../build --prefix $INSTALL_PREFIX + python -m build + ls -lh dist/ - - name: "Install python test environment" - run: python -m pip install -r ${GITHUB_WORKSPACE}/test-requirement.txt + # Install built wheel for testing + - name: "Install built package" + run: | + pip install dist/*.whl + python -c "import importlib.metadata; print('Installed:', importlib.metadata.version('py-capio-cl'))" - - name: "Run python tests" + # Run unit tests + - name: "Run Python tests" run: | - export CAPIO_CL_PY_BINDING_PATH=$(realpath $INSTALL_PREFIX/lib/python/py_capio_cl*) - echo "Python module path: $CAPIO_CL_PY_BINDING_PATH" - pytest ${GITHUB_WORKSPACE}/tests/python/test_* \ No newline at end of file + pytest -v tests/python/test_* diff --git a/.gitignore b/.gitignore index 5fafb18..ec1a7cb 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ cmake-build-*/** .venv/** __pycache__/ *.pyc +dist/** # Prerequisites *.d diff --git a/CMakeLists.txt b/CMakeLists.txt index c5112a6..39021c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,10 @@ project(CAPIO-CL VERSION 1.0.0 ) +# Used to infer release when building pyhton bindings +file(WRITE "${CMAKE_CURRENT_SOURCE_DIR}/VERSION.txt" "${PROJECT_VERSION}") + + set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED TRUE) set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) @@ -80,7 +84,7 @@ install(TARGETS ${TARGET_NAME} ##################################### if (BUILD_PYTHON_BINDINGS) - set(PYTHON_BIND_NAME py_capio_cl) + set(PYTHON_BIND_NAME _py_capio_cl) pybind11_add_module(${PYTHON_BIND_NAME} bindings/python_bindings.cpp @@ -97,11 +101,7 @@ if (BUILD_PYTHON_BINDINGS) ${CMAKE_CURRENT_SOURCE_DIR} ) - install(TARGETS ${PYTHON_BIND_NAME} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}/python - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}/python - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - ) + install(TARGETS _py_capio_cl DESTINATION py_capio_cl) endif () diff --git a/bindings/python_bindings.cpp b/bindings/python_bindings.cpp index c612bba..9a0f50e 100644 --- a/bindings/python_bindings.cpp +++ b/bindings/python_bindings.cpp @@ -5,7 +5,7 @@ namespace py = pybind11; -PYBIND11_MODULE(py_capio_cl, m) { +PYBIND11_MODULE(_py_capio_cl, m) { m.doc() = "CAPIO-CL: Cross Application Programmable I/O - Coordination Language python bindings."; py::class_( diff --git a/build-requirements.txt b/build-requirements.txt new file mode 100644 index 0000000..c795b05 --- /dev/null +++ b/build-requirements.txt @@ -0,0 +1 @@ +build \ No newline at end of file diff --git a/py_capio_cl/__init__.py b/py_capio_cl/__init__.py new file mode 100644 index 0000000..428aaff --- /dev/null +++ b/py_capio_cl/__init__.py @@ -0,0 +1 @@ +from ._py_capio_cl import * \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a54a914 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,32 @@ +[build-system] +requires = [ + "scikit-build-core[pyproject]", + "pybind11", + "setuptools", + "wheel" +] +build-backend = "scikit_build_core.build" + +[project] +name = "py_capio_cl" +dynamic = ["version"] +description = "High-performance collective I/O bindings for CAPIO (pybind11)" +readme = "README.md" +license = { text = "MIT" } +requires-python = ">=3.10" +authors = [ + { name = "Marco Edoardo Santimaria", email = "marcoedoardo.santimaria@unito.it" } +] +urls = { Homepage = "https://github.com/High-Performance-IO/CAPIO-CL" } +dependencies = [] + + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "CMakeLists.txt" +regex = "(?m)VERSION\\s+(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)" +result = "{major}.{minor}.{patch}" + +[tool.scikit-build] +cmake.version = ">=3.18" +ninja.version = ">=1.10" diff --git a/tests/python/test_engine.py b/tests/python/test_engine.py index dab0f94..48c8375 100644 --- a/tests/python/test_engine.py +++ b/tests/python/test_engine.py @@ -1,28 +1,4 @@ -''' -TEST(testpy_capio_clEngine, testAddFileDefault) { - py_capio_cl::Engine engine; -EXPECT_EQ(engine.size(), 0); -engine.newFile("test.dat"); -EXPECT_EQ(engine.size(), 1); -EXPECT_EQ(engine.getCommitRule("test.dat"), py_capio_cl::COMMITTED_ON_TERMINATION); -EXPECT_EQ(engine.getFireRule("test.dat"), py_capio_cl::MODE_UPDATE); -EXPECT_TRUE(engine.getConsumers("test.dat").empty()); -EXPECT_TRUE(engine.getProducers("test.dat").empty()); -EXPECT_FALSE(engine.isPermanent("test.dat")); -EXPECT_FALSE(engine.isExcluded("test.dat")); -EXPECT_TRUE(engine.isFile("test.dat")); -EXPECT_FALSE(engine.isDirectory("test.dat")); -EXPECT_EQ(engine.getDirectoryFileCount("test.dat"), 0); -EXPECT_FALSE(engine.isStoredInMemory("test.dat")); -} -''' - -import os -import importlib.util - -spec = importlib.util.spec_from_file_location("py_capio_cl", os.getenv("CAPIO_CL_PY_BINDING_PATH")) -py_capio_cl = importlib.util.module_from_spec(spec) -spec.loader.exec_module(py_capio_cl) +import py_capio_cl def test_instantiation(): From dce4b0ddb652c7faaaebf0ada1a892d73a599ce0 Mon Sep 17 00:00:00 2001 From: Marco Edoardo Santimaria Date: Tue, 7 Oct 2025 21:52:09 +0200 Subject: [PATCH 2/2] bugfix --- .github/workflows/python-bindings.yml | 27 +++++-------------- CMakeLists.txt | 4 --- README.md | 9 ++++--- build-requirements.txt | 2 +- pyproject.toml | 18 ++++++++----- test-requirement.txt => test-requirements.txt | 0 6 files changed, 25 insertions(+), 35 deletions(-) rename test-requirement.txt => test-requirements.txt (100%) diff --git a/.github/workflows/python-bindings.yml b/.github/workflows/python-bindings.yml index d862025..15eb4cc 100644 --- a/.github/workflows/python-bindings.yml +++ b/.github/workflows/python-bindings.yml @@ -1,5 +1,4 @@ name: "Python Bindings Unit Tests" - on: push: branches: @@ -7,20 +6,18 @@ on: pull_request: branches: - main - concurrency: group: build-python-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true - jobs: python-tests: name: "Test Python bindings" strategy: matrix: - os: [ 'ubuntu-24.04', 'macos-15-intel', 'macos-26' ] + on: [ 'ubuntu-24.04', 'macos-15-intel', 'macos-26' ] python: [ '3.10', '3.11', '3.12', '3.13' ] - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.on }} env: INSTALL_PREFIX: "/usr/local" @@ -35,37 +32,27 @@ jobs: # Install platform build dependencies - name: "Install system packages (Ubuntu)" - if: startsWith(matrix.os, 'ubuntu-') + if: startsWith(matrix.on, 'ubuntu-') run: sudo apt-get update && sudo apt-get install -y ninja-build g++ cmake - name: "Setup Homebrew (macOS)" - if: startsWith(matrix.os, 'macos-') + if: startsWith(matrix.on, 'macos-') uses: Homebrew/actions/setup-homebrew@main - name: "Install system packages (macOS)" - if: startsWith(matrix.os, 'macos-') + if: startsWith(matrix.on, 'macos-') run: brew install ninja gcc cmake # Install Python build dependencies - - name: "Install Python build tools" + - name: "Build and Install Python wheel" run: | python -m pip install --upgrade pip python -m pip install -r build-requirements.txt - python -m pip install build pytest - - # Build wheel from pyproject.toml - - name: "Build Python wheel" - run: | python -m build - ls -lh dist/ - - # Install built wheel for testing - - name: "Install built package" - run: | pip install dist/*.whl - python -c "import importlib.metadata; print('Installed:', importlib.metadata.version('py-capio-cl'))" # Run unit tests - name: "Run Python tests" run: | + python -m pip install -r test-requirements.txt pytest -v tests/python/test_* diff --git a/CMakeLists.txt b/CMakeLists.txt index 39021c3..8097fdc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,10 +9,6 @@ project(CAPIO-CL VERSION 1.0.0 ) -# Used to infer release when building pyhton bindings -file(WRITE "${CMAKE_CURRENT_SOURCE_DIR}/VERSION.txt" "${PROJECT_VERSION}") - - set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED TRUE) set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) diff --git a/README.md b/README.md index 831070e..dcc7a1f 100644 --- a/README.md +++ b/README.md @@ -93,12 +93,15 @@ These bindings expose the core C++ APIs—such as `Engine`, `Parser`, directly to Python, allowing the CAPIO-CL logic to be used within python projects. ### 🔧 Building the Bindings -To include Python support when building CAPIO-CL: +You can build and install the Python bindings directly from the CAPIO-CL source tree using: ```bash -cmake -DBUILD_PYTHON_BINDINGS=ON .. -cmake --build . --target py_capio_cl +pip install --upgrade pip +pip install -r build-requirements.txt +python -m build +pip install dst/*.whl ``` +Now you will be able to directly import the package **py_capio_cl** in your project! --- diff --git a/build-requirements.txt b/build-requirements.txt index c795b05..09d7689 100644 --- a/build-requirements.txt +++ b/build-requirements.txt @@ -1 +1 @@ -build \ No newline at end of file +build==1.3.0 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index a54a914..711b17b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,24 +1,28 @@ [build-system] requires = [ - "scikit-build-core[pyproject]", - "pybind11", - "setuptools", - "wheel" + "scikit-build-core[pyproject]==0.11.6", + "pybind11==3.0.1", + "setuptools==80.9.0", + "wheel==0.45.1" ] build-backend = "scikit_build_core.build" [project] name = "py_capio_cl" dynamic = ["version"] -description = "High-performance collective I/O bindings for CAPIO (pybind11)" +description = "Python bindings for the CAPIO-CL coordination language" readme = "README.md" license = { text = "MIT" } requires-python = ">=3.10" authors = [ { name = "Marco Edoardo Santimaria", email = "marcoedoardo.santimaria@unito.it" } ] -urls = { Homepage = "https://github.com/High-Performance-IO/CAPIO-CL" } -dependencies = [] + +[project.urls] +Homepage = "https://capio.hpc4ai.it" +Documentation = "https://capio.hpc4ai.it/docs/coord-language/" +Repository = "https://github.com/High-Performance-IO/CAPIO-CL" +Issues = "https://github.com/High-Performance-IO/CAPIO-CL/issues" [tool.scikit-build.metadata.version] diff --git a/test-requirement.txt b/test-requirements.txt similarity index 100% rename from test-requirement.txt rename to test-requirements.txt