Ci/cibuildwheel linux

.github/workflows/conan-lock-test.yaml

@@ -0,0 +1,48 @@
+name: Conan Lock Test
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [main]
+    paths:
+      - "Dockerfile.conan-lock"
+      - "Dockerfile"
+      - "lib/**"
+      - "src/**"
+      - "pyproject.toml"
+      - ".github/workflows/conan-lock-test.yaml"
+  push:
+    branches: [main]
+    paths:
+      - "Dockerfile.conan-lock"
+      - "Dockerfile"
+      - "lib/**"
+      - "src/**"
+      - "pyproject.toml"
+      - ".github/workflows/conan-lock-test.yaml"
+
+jobs:
+  docker_conan_lock:
+    name: Docker Conan lock build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Dockerfile.conan-lock
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./Dockerfile.conan-lock
+          platforms: linux/amd64
+          push: false
+          load: false
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          tags: arraymorph-conan-lock-test:ci

.python-version

@@ -1 +1 @@
-3.9
+3.10

Dockerfile

@@ -0,0 +1,102 @@
+# -----------------------------
+# Stage 1: build
+# -----------------------------
+FROM ubuntu:24.04 AS builder
+
+ENV DEBIAN_FRONTEND=noninteractive
+WORKDIR /work
+
+RUN apt-get update && apt-get install -y \
+  build-essential \
+  cmake \
+  ninja-build \
+  pkg-config \
+  git \
+  curl \
+  ca-certificates \
+  && rm -rf /var/lib/apt/lists/*
+
+# Install uv
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+COPY pyproject.toml README.md /work/
+COPY lib/conanfile.py /work/lib/conanfile.py
+
+# Create environment
+RUN uv python install 3.10
+RUN uv venv --python 3.10 
+
+# Install build deps
+RUN uv pip install --python .venv/bin/python \
+  build \
+  scikit-build-core \
+  conan \
+  'h5py>=3.10,<3.16' \
+  ninja \
+  cmake
+
+# Conan
+RUN --mount=type=cache,target=/root/.conan2 \
+  .venv/bin/conan profile detect --force
+RUN --mount=type=cache,target=/root/.conan2 \
+  .venv/bin/conan install lib --build=missing -s build_type=Release -of /work/lib/build
+
+COPY src /work/src
+COPY lib/CMakeLists.txt /work/lib/CMakeLists.txt
+COPY lib/include /work/lib/include
+COPY lib/src /work/lib/src
+
+# Build wheel
+RUN --mount=type=cache,target=/root/.conan2 \
+  .venv/bin/python -c 'import h5py, os; d=os.path.dirname(h5py.__file__); libdir=os.path.join(d, ".dylibs") if os.path.exists(os.path.join(d, ".dylibs")) else os.path.join(os.path.dirname(d), "h5py.libs"); print(f"h5py {h5py.__version__} -> {libdir}")' && \
+  export H5PY_HDF5_DIR="$(.venv/bin/python -c 'import h5py, os; d=os.path.dirname(h5py.__file__); print(os.path.join(d, ".dylibs") if os.path.exists(os.path.join(d, ".dylibs")) else os.path.join(os.path.dirname(d), "h5py.libs"))')" && \
+  export CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=/work/lib/build/build/Release/generators/conan_toolchain.cmake" && \
+  .venv/bin/python -m build --wheel --no-isolation
+
+# -----------------------------
+# Stage 2: runtime
+# -----------------------------
+FROM ubuntu:24.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y \
+  curl \
+  ca-certificates \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+COPY --from=builder /work/dist /app/dist
+
+RUN uv venv --python 3.10
+RUN uv pip install --python .venv/bin/python dist/*.whl
+
+ENV STORAGE_PLATFORM=S3
+ENV AWS_REGION=us-east-1
+
+COPY ./examples/python/write.py ./test.py
+
+CMD bash -lc '\
+  set -e; \
+  PLUGIN_PATH=$(./.venv/bin/python -c "import arraymorph; print(arraymorph.get_plugin_path())"); \
+  PLUGIN_DIR=$(./.venv/bin/python -c "import arraymorph; print(arraymorph.get_plugin_dir())"); \
+  H5PY_DIR=$(./.venv/bin/python -c "import h5py, os; d=os.path.dirname(h5py.__file__); print(os.path.join(d,\".dylibs\") if os.path.exists(os.path.join(d,\".dylibs\")) else os.path.join(os.path.dirname(d),\"h5py.libs\"))"); \
+  export HDF5_PLUGIN_PATH="$PLUGIN_DIR"; \
+  export HDF5_VOL_CONNECTOR="arraymorph under_vol=0;under_info={}"; \
+  export LD_LIBRARY_PATH="$H5PY_DIR:${LD_LIBRARY_PATH:-}"; \
+  echo "plugin path: $PLUGIN_PATH"; \
+  echo "plugin dir:  $PLUGIN_DIR"; \
+  echo "h5py dir:    $H5PY_DIR"; \
+  echo "--- plugin dir ---"; \
+  ls -la "$PLUGIN_DIR"; \
+  echo "--- h5py dir ---"; \
+  ls -la "$H5PY_DIR"; \
+  echo "--- ldd plugin ---"; \
+  ldd "$PLUGIN_PATH" || true; \
+  echo "--- h5py check ---"; \
+  ./.venv/bin/python -c "import h5py; print(\"h5py\", h5py.__version__); print(\"hdf5\", h5py.version.hdf5_version)"; \
+  echo "--- run test ---"; \
+  ./.venv/bin/python -u test.py'
+

examples/python/write.py

@@ -1,12 +1,14 @@
 import h5py
 import numpy as np
-
-f = h5py.File('demo.h5', 'w')
-dset = f.create_dataset("test", (100, 100), chunks=(10, 10), dtype='i4')
+import sys
+print("before open", file=sys.stderr, flush=True)
+f = h5py.File("demo-mac.h5", "w")
+dset = f.create_dataset("test", (100, 100), chunks=(10, 10), dtype="i4")
 
 # write 100 x 100 2d arrays filled with value 1
-data = np.ones((100, 100), dtype='i4')
-for i in range(100):
-    for j in range(100):
-        data[i][j] = i + j
-dset[:] = data
+print("before open")
+with f:
+    print("after open")
+    dset = f.create_dataset("x", data=[1, 2, 3])
+    print("after create_dataset")
+print("after close")

justfile

@@ -1,47 +1,55 @@
-# ArrayMorph — Top-Level Build Orchestration
-# https://just.systems
+# Required Python packages (install in .venv before running just):
+#
+# pip install \
+#   build \
+#   scikit-build-core \
+#   cibuildwheel \
+#   conan \
+#   h5py==3.15.1
 
-# --- Settings ---
-set dotenv-load := true
-set export := true
+set shell := ["bash", "-eu", "-o", "pipefail", "-c"]
 
-# --- Variables ---
-VCPKG_TOOLCHAIN := env("VCPKG_ROOT", home_directory() / ".vcpkg") / "scripts/buildsystems/vcpkg.cmake"
-HDF5_DIR := `./.venv/bin/python3 -c "import h5py,os;d=os.path.dirname(h5py.__file__);print(os.path.join(d,'.dylibs') if os.path.exists(os.path.join(d,'.dylibs')) else os.path.join(os.path.dirname(d),'h5py.libs'))"`
+BUILD_TYPE := env_var_or_default("BUILD_TYPE", "Release")
 
-# --- Recipes ---
+# Automatically discover the HDF5 directory shipped with h5py.
+# This will FAIL if:
+#   • .venv does not exist
+#   • h5py is not installed
+# which is intentional.
+
+H5PY_HDF5_DIR := `./.venv/bin/python3 -c "import h5py,os;d=os.path.dirname(h5py.__file__);print(os.path.join(d,'.dylibs') if os.path.exists(os.path.join(d,'.dylibs')) else os.path.join(os.path.dirname(d),'h5py.libs'))"`
+CONAN_BUILD := justfile_directory() / "lib" / "build" / BUILD_TYPE
+TOOLCHAIN := CONAN_BUILD / "generators" / "conan_toolchain.cmake"
 
-# List available commands
 default:
     @just --list
 
-# Build Python wheel (scikit-build-core handles CMake)
-wheel:
-    CMAKE_TOOLCHAIN_FILE={{ VCPKG_TOOLCHAIN }} \
-    HDF5_DIR={{ HDF5_DIR }} \
-      uv build --wheel --no-build-isolation
-
-# Install editable into current venv (for development iteration)
-dev:
-    CMAKE_TOOLCHAIN_FILE={{ VCPKG_TOOLCHAIN }} \
-    HDF5_DIR={{ HDF5_DIR }} \
-    uv pip install -e .
+info:
+    @echo "BUILD_TYPE: {{ BUILD_TYPE }}"
+    @echo "H5PY_HDF5_DIR: {{ H5PY_HDF5_DIR }}"
+    @echo "CONAN_BUILD: {{ CONAN_BUILD }}"
+    @echo "TOOLCHAIN: {{ TOOLCHAIN }}"
 
-# Full build from scratch: deps → wheel
-build: wheel
+deps:
+    conan install lib --build=missing -s build_type={{ BUILD_TYPE }}
 
-# Full build + test
-all: build
+configure: deps
+    H5PY_HDF5_DIR={{ H5PY_HDF5_DIR }} \
+    cmake -S lib -B {{ CONAN_BUILD }} \
+      -DCMAKE_BUILD_TYPE={{ BUILD_TYPE }} \
+      -DCMAKE_TOOLCHAIN_FILE={{ TOOLCHAIN }}
 
-# Clean build artifacts
-clean:
-    rm -rf lib/build lib/vcpkg_installed dist *.egg-info .test-venv
+build: configure
+    cmake --build {{ CONAN_BUILD }}
 
-# Full clean rebuild
-rebuild: clean build
+wheel: deps
+    CMAKE_TOOLCHAIN_FILE={{ TOOLCHAIN }} \
+    H5PY_HDF5_DIR={{ H5PY_HDF5_DIR }} \
+    ./.venv/bin/python -m build --wheel --no-isolation
 
-# Show current env var values (for debugging)
-info:
-    @echo "CMAKE_TOOLCHAIN_FILE: {{ VCPKG_TOOLCHAIN }}"
-    @echo "HDF5_DIR:        {{ HDF5_DIR }}"
-    @echo "Plugin lib:            $(find lib/build -name 'lib_array_morph*' 2>/dev/null || echo 'not built')"
+clean:
+    rm -rf \
+        lib/build \
+        dist \
+        wheelhouse \
+        *.egg-info