From d4603a364c5bb3ce1c138f4ea71f50ec301940f8 Mon Sep 17 00:00:00 2001 From: Ted Kisner Date: Fri, 24 Jan 2025 14:01:48 -0800 Subject: [PATCH 1/3] Work on support for numpy-2 and python-3.12 - Remove numpy-1 checks / constraints. - Update wheel workflows and scripts. Add dependabot workflow and combine wheel test / deploy workflows into one. Remove python-3.8 support. - Update README instructions. - Add pyproject.toml and remove unsupported distutils from setup.py - Fix healpix_bare.c compiler warnings - Add missing guard around omp.h inclusion - Remove deprecated datetime functions - Bump vendored boost to version 1.87 and compile our own libflac-1.5.0 - Build our own OpenBLAS for macos wheels --- .github/dependabot.yml | 9 + .github/workflows/deploy.yml | 97 ----- .github/workflows/official-docker-images.yml | 6 +- .github/workflows/pytest.yml | 11 +- .github/workflows/wheels.yml | 107 +++++- CMakeLists.txt | 90 +++-- Dockerfile | 2 +- README.rst | 348 +++++------------- conda_deps.txt | 14 - conda_dev_requirements.txt | 35 ++ docker/so3g-setup.sh | 8 +- pyproject.toml | 51 +++ python/__init__.py | 6 - requirements.txt | 3 +- setup.py | 166 +++------ src/array_ops.cxx | 4 +- src/healpix_bare.c | 14 +- test/test_proj_astro.py | 2 +- wheels/build_requirements.txt | 11 - wheels/install_deps_linux.sh | 78 ++-- wheels/install_deps_osx.sh | 182 +++++---- wheels/repair_wheel_linux.sh | 10 +- wheels/repair_wheel_macos.sh | 13 +- ...{spt3g.patch => spt3g_disable_tests.patch} | 0 wheels/spt3g_sys_time.patch | 12 + wheels/test_local_cibuildwheel.sh | 2 +- wheels/test_local_macos.sh | 50 +-- 27 files changed, 611 insertions(+), 720 deletions(-) create mode 100644 .github/dependabot.yml delete mode 100644 .github/workflows/deploy.yml delete mode 100644 conda_deps.txt create mode 100644 conda_dev_requirements.txt create mode 100644 pyproject.toml delete mode 100644 wheels/build_requirements.txt rename wheels/{spt3g.patch => spt3g_disable_tests.patch} (100%) create mode 100644 wheels/spt3g_sys_time.patch diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..2c431b0b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index 293d4da0..00000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,97 +0,0 @@ -# Deploy tagged releases. - -name: Deploy Release - -on: - push: - tags: - - '*' - -env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TWINE_TOKEN }} - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - build_wheels: - name: Build wheel for cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }} - runs-on: ${{ matrix.os }} - strategy: - # Ensure that a wheel builder finishes even if another fails. Useful for - # debugging multiple problems in parallel. - fail-fast: false - matrix: - include: - # Linux 64 bit manylinux - - os: ubuntu-latest - arch: x86_64 - python: 38 - builder: manylinux - - os: ubuntu-latest - arch: x86_64 - python: 39 - builder: manylinux - - os: ubuntu-latest - arch: x86_64 - python: 310 - builder: manylinux - - os: ubuntu-latest - arch: x86_64 - python: 311 - builder: manylinux - - # MacOS x86_64. The macos-13 runner is the last - # Intel-based runner version. At some point we'll - # need to switch to macos-14 and test cross compiling. - - os: macos-13 - arch: x86_64 - python: 39 - builder: macosx - - os: macos-13 - arch: x86_64 - python: 310 - builder: macosx - - os: macos-13 - arch: x86_64 - python: 311 - builder: macosx - env: - CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }} - CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.builder }}2014 - CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.builder }}2014 - CIBW_TEST_SKIP: "*-macosx_arm64" - CIBW_BUILD_VERBOSITY: 3 - CIBW_ENVIRONMENT_LINUX: "CC=gcc CXX=g++ CFLAGS='-O3 -fPIC' CXXFLAGS='-O3 -fPIC -std=c++14' BOOST_ROOT=/usr/local" - CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=13.0 CC=gcc-14 CXX=g++-14 CFLAGS='-O3 -fPIC' CXXFLAGS='-O3 -fPIC -std=c++14' BOOST_ROOT=/usr/local FLAC_ROOT=/usr/local BLAS_LIBRARIES=/usr/local/lib/libscipy_openblas.dylib" - CIBW_BEFORE_BUILD_LINUX: ./wheels/install_deps_linux.sh - CIBW_BEFORE_BUILD_MACOS: ./wheels/install_deps_osx.sh ${{ matrix.builder }}_${{ matrix.arch }} - CIBW_REPAIR_WHEEL_COMMAND_LINUX: ./wheels/repair_wheel_linux.sh {dest_dir} {wheel} - CIBW_REPAIR_WHEEL_COMMAND_MACOS: ./wheels/repair_wheel_macos.sh {dest_dir} {wheel} {delocate_archs} - CIBW_BEFORE_TEST: export OMP_NUM_THREADS=2 - CIBW_TEST_REQUIRES: pytest - CIBW_TEST_COMMAND: > - python -c 'import so3g.smurf.reader; from spt3g import core' && - python -m pytest {package}/test - steps: - - name: Checkout - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - name: Install Python - with: - python-version: '3.10' - - - name: Install cibuildwheel - run: | - python -m pip install twine cibuildwheel==2.20.0 - - - name: Build wheel - run: | - python -m cibuildwheel --output-dir wheelhouse - - - name: Upload to PyPI - run: | - python -m twine upload wheelhouse/so3g*cp${{ matrix.python }}-${{ matrix.builder }}*.whl diff --git a/.github/workflows/official-docker-images.yml b/.github/workflows/official-docker-images.yml index 795bd7c7..01be0411 100644 --- a/.github/workflows/official-docker-images.yml +++ b/.github/workflows/official-docker-images.yml @@ -12,14 +12,10 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 # Fetch all history for all tags and branches with: fetch-depth: 0 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 # Build - name: Build docker images diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 0864e10d..e4ce3b06 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -2,8 +2,11 @@ name: Run Tests on: push: - branches-ignore: [ master ] + branches: + - master pull_request: + branches: + - master concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -15,11 +18,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 + uses: actions/checkout@v4 - name: Build docker images run: | diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index af4d1dc5..2c7da150 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -1,7 +1,10 @@ -name: Test Binary Wheels +name: Binary Wheels -on: workflow_dispatch +on: + workflow_dispatch: + release: + types: [ published ] concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -18,10 +21,6 @@ jobs: matrix: include: # Linux 64 bit manylinux - - os: ubuntu-latest - arch: x86_64 - python: 38 - builder: manylinux - os: ubuntu-latest arch: x86_64 python: 39 @@ -34,6 +33,14 @@ jobs: arch: x86_64 python: 311 builder: manylinux + - os: ubuntu-latest + arch: x86_64 + python: 312 + builder: manylinux + # - os: ubuntu-latest + # arch: x86_64 + # python: 313 + # builder: manylinux # MacOS x86_64. The macos-13 runner is the last # Intel-based runner version. At some point we'll @@ -42,27 +49,80 @@ jobs: arch: x86_64 python: 39 builder: macosx + deploy: 13.0 - os: macos-13 arch: x86_64 python: 310 builder: macosx + deploy: 13.0 - os: macos-13 arch: x86_64 python: 311 builder: macosx + deploy: 13.0 + - os: macos-13 + arch: x86_64 + python: 312 + builder: macosx + deploy: 13.0 + # - os: macos-13 + # arch: x86_64 + # python: 313 + # builder: macosx + # deploy: 13.0 + + # MacOS arm64 + - os: macos-latest + arch: arm64 + python: 310 + builder: macosx + deploy: 14.0 + - os: macos-latest + arch: arm64 + python: 311 + builder: macosx + deploy: 14.0 + - os: macos-latest + arch: arm64 + python: 312 + builder: macosx + deploy: 14.0 + # - os: macos-latest + # arch: arm64 + # python: 313 + # builder: macosx + # deploy: 14.0 env: CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }} CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.builder }}2014 CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.builder }}2014 CIBW_BUILD_VERBOSITY: 3 - CIBW_ENVIRONMENT_LINUX: "CC=gcc CXX=g++ CFLAGS='-O3 -fPIC' CXXFLAGS='-O3 -fPIC -std=c++14' BOOST_ROOT=/usr/local" - CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=13.0 CC=gcc-14 CXX=g++-14 CFLAGS='-O3 -fPIC' CXXFLAGS='-O3 -fPIC -std=c++14' BOOST_ROOT=/usr/local FLAC_ROOT=/usr/local BLAS_LIBRARIES=/usr/local/lib/libscipy_openblas.dylib" + CIBW_ENVIRONMENT_LINUX: > + CC=gcc + CXX=g++ + CFLAGS='-O3 -fPIC' + CXXFLAGS='-O3 -fPIC -std=c++14' + BOOST_ROOT=/usr/local + FLAC_ROOT=/usr/local + SO3G_BUILD_BLAS_LIBRARIES='-L/usr/local/lib -lopenblas -fopenmp -lm -lgfortran' + CIBW_ENVIRONMENT_MACOS: > + MACOSX_DEPLOYMENT_TARGET=${{ matrix.deploy }} + CC=gcc-14 + CXX=g++-14 + CFLAGS='-O3 -fPIC' + CXXFLAGS='-O3 -fPIC -std=c++14' + CPATH='/usr/local/include' + BOOST_ROOT=/usr/local + FLAC_ROOT=/usr/local + SO3G_BUILD_BLAS_LIBRARIES='/usr/local/lib/libopenblas.dylib' CIBW_BEFORE_BUILD_LINUX: ./wheels/install_deps_linux.sh - CIBW_BEFORE_BUILD_MACOS: ./wheels/install_deps_osx.sh ${{ matrix.builder }}_${{ matrix.arch }} + CIBW_BEFORE_BUILD_MACOS: > + ln -s $(dirname $(readlink -f $(which python3)))/python3-config $(dirname $(which python3))/python3-config && + ./wheels/install_deps_osx.sh CIBW_REPAIR_WHEEL_COMMAND_LINUX: ./wheels/repair_wheel_linux.sh {dest_dir} {wheel} CIBW_REPAIR_WHEEL_COMMAND_MACOS: ./wheels/repair_wheel_macos.sh {dest_dir} {wheel} {delocate_archs} CIBW_BEFORE_TEST: export OMP_NUM_THREADS=2 - CIBW_TEST_REQUIRES: pytest + CIBW_TEST_REQUIRES: pytest pixell CIBW_TEST_COMMAND: > python -c 'import so3g.smurf.reader; from spt3g import core' && python -m pytest {package}/test @@ -73,11 +133,11 @@ jobs: - uses: actions/setup-python@v5 name: Install Python with: - python-version: '3.10' + python-version: '3.12' - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==2.20.0 + python -m pip install cibuildwheel==2.22.0 - name: Build wheel run: | @@ -85,5 +145,24 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: wheels_cp${{ matrix.python }}-${{ matrix.builder }} - path: ./wheelhouse/so3g*cp${{ matrix.python }}-${{ matrix.builder }}*.whl + name: wheels_cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }} + path: ./wheelhouse/so3g*cp${{ matrix.python }}-${{ matrix.builder }}*${{ matrix.arch }}*.whl + + upload_pypi: + needs: build_wheels + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + if: github.event_name == 'release' && github.event.action == 'published' + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + # unpacks all CIBW artifacts into dist/ + pattern: wheels_* + path: dist + merge-multiple: true + + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/CMakeLists.txt b/CMakeLists.txt index b7684edb..0cd83c01 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,15 +1,12 @@ -cmake_minimum_required (VERSION 3.1) +cmake_minimum_required(VERSION 3.17) project (so3g) include(local.cmake OPTIONAL) # cmake policies -- best to keep these in sync with spt3g! -if(POLICY CMP0060) # Suppress cmake stripping full paths from libraries in some cases - cmake_policy(SET CMP0060 NEW) -endif() -cmake_policy(SET CMP0012 NEW) # Allow use of true in boolean expressions -if(POLICY CMP0042) # Enable RPATH on OSX - cmake_policy(SET CMP0042 NEW) +# Don't warn about removal of FindBoost in cmake 3.30+ +if(POLICY CMP0167) + cmake_policy(SET CMP0167 NEW) endif() # Default to Release because we want that -O3. This is what spt3g_software does too. @@ -18,12 +15,20 @@ if(NOT CMAKE_BUILD_TYPE) "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel" FORCE) endif(NOT CMAKE_BUILD_TYPE) +# Require C++ 17 (aligned with spt3g) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +# We are building libraries that will eventually be linked into shared +# modules. All code should be built with PIC. +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + # For this to be found, make sure the spt3g build directory can be # searched; i.e. -DCMAKE_PREFIX_PATH=/path/to/spt3g_software/build find_package(Spt3g REQUIRED) -find_package(PythonInterp 3) -find_package(PythonLibs 3) +find_package(Python COMPONENTS Interpreter Development.Module REQUIRED) find_package(FLAC) find_package(GSL) @@ -36,11 +41,14 @@ else() endif() # Determine the location of site-packages. -execute_process ( COMMAND ${PYTHON_EXECUTABLE} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())" OUTPUT_VARIABLE PYTHON_SITE_PACKAGES OUTPUT_STRIP_TRAILING_WHITESPACE) +execute_process(COMMAND ${Python_EXECUTABLE} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())" OUTPUT_VARIABLE PYTHON_SITE_PACKAGES OUTPUT_STRIP_TRAILING_WHITESPACE) -# Numpy include directory? -execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import numpy; print(numpy.get_include())" - OUTPUT_VARIABLE NUMPY_INCLUDE_DIR OUTPUT_STRIP_TRAILING_WHITESPACE) +# Numpy include directory +execute_process(COMMAND ${Python_EXECUTABLE} -c + "import numpy; print(numpy.get_include())" + OUTPUT_VARIABLE NUMPY_INCLUDE_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE +) include_directories(${CMAKE_SOURCE_DIR}/include ${CMAKE_BINARY_DIR} ) include_directories(${NUMPY_INCLUDE_DIR}) @@ -48,7 +56,6 @@ include_directories(${NUMPY_INCLUDE_DIR}) # # Define the so3g build target. This is a shared library. # - set(CMAKE_LIBRARY_OUTPUT_DIRECTORY so3g) add_library(so3g SHARED src/main.cxx @@ -66,29 +73,28 @@ add_library(so3g SHARED # We could disable the lib prefix on the output library... but let's not. #set_target_properties(so3g PROPERTIES PREFIX "") -# Make a list of .py files for the library. -file(GLOB MY_PYTHONS - "${CMAKE_CURRENT_SOURCE_DIR}/python/*.py") -file(GLOB MY_PYTHONS_HK - "${CMAKE_CURRENT_SOURCE_DIR}/python/hk/*.py") -file(GLOB MY_PYTHONS_PROJ - "${CMAKE_CURRENT_SOURCE_DIR}/python/proj/*.py") -file(GLOB MY_PYTHONS_SMURF - "${CMAKE_CURRENT_SOURCE_DIR}/python/smurf/*.py") +# Disable boost python auto_ptr warnings +target_compile_definitions(so3g PUBLIC BOOST_NO_AUTO_PTR) -# Provide list of libs to link against. -target_link_libraries(so3g spt3g::core) -# Link GSL +# Link to the core spt3g library. This brings in boost dependencies +# as well. +target_link_libraries(so3g PUBLIC spt3g::core) + +# Link to GSL target_include_directories(so3g PRIVATE ${GSL_INCLUDE_DIR}) -target_link_libraries(so3g ${GSL_LIBRARIES}) +target_link_libraries(so3g PUBLIC ${GSL_LIBRARIES}) + +# FLAC- library already comes from spt3g dependencies, but +# we need to have the headers. +target_include_directories(so3g PRIVATE ${FLAC_INCLUDE_DIR}) # You probably want to select openblas, so pass -DBLA_VENDOR=OpenBLAS find_package(BLAS REQUIRED) if(BLAS_FOUND) message("-- BLAS found: ${BLAS_LIBRARIES}") - target_link_libraries(so3g ${BLAS_LIBRARIES}) + target_link_libraries(so3g PUBLIC ${BLAS_LIBRARIES}) - # The BLAS library may or may not include the cblas_ bindings. + # The BLAS library may or may not include the cblas_* bindings. # This variable set is needed by check_function_exists; starting in # cmake v3.18 you can say BLAS::BLAS instead of the lib path... set(CMAKE_REQUIRED_LIBRARIES ${BLAS_LIBRARIES}) @@ -97,27 +103,37 @@ if(BLAS_FOUND) message("-- cblas bindings are included in the BLAS library") else() message("-- cblas bindings not found in BLAS; adding cblas.") - target_link_libraries(so3g cblas) + target_link_libraries(so3g PUBLIC cblas) endif() # On MacOS with clang linking to the Accelerate framework, the cblas - # headers are not always found. Handle this case. + # headers are not always found. Handle this case. Also note that the + # Accelerate framework has documented numerical problems- consider using + # a better BLAS/LAPACK implementation. if(BLAS_Accelerate_LIBRARY) - #string(REGEX REPLACE "^(.*)/System/Library/Frameworks/Accelerate.framework" "\\1" FRAMEWORK_INCLUDE "${BLAS_Accelerate_LIBRARY}") - #target_compile_options(so3g PRIVATE -F ${FRAMEWORK_INCLUDE}) target_include_directories(so3g PRIVATE ${BLAS_Accelerate_LIBRARY}/Versions/A/Frameworks/vecLib.framework/Headers) endif() endif(BLAS_FOUND) # This custom target generates _version.h, in the build tree. That is all. add_custom_target(so3g-version - COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/version_h.py - SO3G_VERSION_STRING ${CMAKE_CURRENT_BINARY_DIR}/_version.h - SOURCES version_h.py - ) + COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/version_h.py + SO3G_VERSION_STRING ${CMAKE_CURRENT_BINARY_DIR}/_version.h + SOURCES version_h.py +) add_dependencies(so3g so3g-version) +# Make a list of .py files for the library. +file(GLOB MY_PYTHONS + "${CMAKE_CURRENT_SOURCE_DIR}/python/*.py") +file(GLOB MY_PYTHONS_HK + "${CMAKE_CURRENT_SOURCE_DIR}/python/hk/*.py") +file(GLOB MY_PYTHONS_PROJ + "${CMAKE_CURRENT_SOURCE_DIR}/python/proj/*.py") +file(GLOB MY_PYTHONS_SMURF + "${CMAKE_CURRENT_SOURCE_DIR}/python/smurf/*.py") + # Define the install rules. if(DEFINED PYTHON_INSTALL_DEST) diff --git a/Dockerfile b/Dockerfile index 49abf5ad..1ec32b50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ RUN apt update && apt install -y \ build-essential \ automake \ gfortran \ - libopenblas-dev \ + libopenblas-openmp-dev \ libbz2-dev \ python-is-python3 diff --git a/README.rst b/README.rst index f4524991..44dd4cf0 100644 --- a/README.rst +++ b/README.rst @@ -22,294 +22,122 @@ so3g Glue functions and new classes for SO work in the spt3g paradigm. -Environment Setup -================= +Installation from Binary Packages +=================================== -Before installing SO software, make sure you know what python -environment you will be using: +If you are just "using" `so3g` and not actively modifying the source, simply install the binary wheels from PyPI:: -- Are you working on Linux or MacOS? + pip install so3g -- Are you using a "python3" executable provided by your OS or one - provided by another source (Anaconda, conda-forge / miniforge, - homebrew, macports, etc)? +Building from Source +====================== -- Are you going to be actively developing so3g or just installing - and using it? +When developing the `so3g` code, you will need to build from source. There are two methods documented here: (1) using a conda environment to provide python and all compiled dependencies and (2) using a virtualenv for python and OS packages for compiled dependencies. In both cases, the compiled dependencies include: -After you have determined the answers to these questions, you can -set up your working environment. +- A C++ compiler supporting the c++17 standard -Using Conda ------------ +- BLAS / LAPACK -If you already have a conda installation (a.k.a. conda "base" or "root" -environment) that is recent, then you can use that to create an -environment. First, verify some info about your installation:: +- Boost (at least version 1.87 for numpy-2 compatibility) - which python3 - python3 --version - which conda +- GSL -Your python version should be at least 3.7.0. Does the location of python3 -match the location of the conda command (are they in the same bin -directory)? If so, then you are ready. If you do not have conda installed -but would like to use it, you might consider installing the "miniforge" -root environment (https://github.com/conda-forge/miniforge) which is -configured to get packages from the conda-forge channel. Conda-forge provides -a large number of software packages built from high quality recipes. +- libFLAC -The next step is to create a dedicated conda environment for your SO work. -This allows us to create and delete these environments as needed without -messing up the root environment:: - - conda create -n simons # <- Only do this once - conda activate simons - -Now install as many dependencies as possible from conda packages. These -are listed in a text file in the top of this git repo:: - - conda install --file conda_deps.txt - -Using a Virtualenv ------------------- - -If you are using a python3 provided by your OS or somewhere else, you -can work inside a "virtualenv". This is like a sandbox where you can -install packages for working on one project and you can always just -wipe the directory and start over if something gets messed up. In general, -you should never pip install packages directly to your OS or package -manager location (e.g. pip installing as root on linux, or pip installing -directly to /usr/local with homebrew). Doing so will create problems -for the future on your system. We will create a virtualenv in our home -directory:: - - python3 -m venv ${HOME}/simons # <- Just do this once - source ${HOME}/simons/bin/activate - -Now install some basic packages and then all of our requirements:: - - python3 -m pip install --upgrade pip setuptools wheel - python3 -m pip install -r requirements.txt - -Other Python Packages ----------------------- - -If you will be using the pointing code in so3g, install pixell and qpoint -with pip (regardless of whether you are using a conda env or virtualenv):: - - pip install pixell - pip install qpoint +Building with Conda Tools +---------------------------- +This method is the most reliable, since we will be using a self-consistent set of dependencies and the same compilers that were used to build those. First, ensure that you have a conda base environment that uses the conda-forge channels. The easiest way to get this is to use the "mini-forge" installer (https://github.com/conda-forge/miniforge). -Installing Pre-Built Wheels -=========================== +Once you have the conda "base" environment installed, create a new environment for Simons Observatory work. We force the python version to 3.12, since the default (3.13) is still missing some of our dependencies:: -If you are just *using* so3g (not modifying or developing it), then you can -install the latest release of the package with:: + conda create -n simons python==3.12 # <- Only do this once + conda activate simons - pip install so3g +Now install all of our dependencies (except for spt3g):: -This command should be used regardless of whether you are working in a -conda env or a virtualenv. + conda install --file conda_dev_requirements.txt -Now any time you activate your virtualenv / conda env, you can use so3g. +Next, choose how to install spt3g. +Bundled SPT3G +~~~~~~~~~~~~~~~~~ -Building From Source -==================== +If you are just testing a quick change, you can use `pip` to install so3g. This will download a copy of spt3g and bundle it into the the installed package. The downside is that **every time** you run pip, it will re-build all of spt3g and so3g under the hood with cmake:: -If you will be developing so3g or want more control over the build, then -you should build from source. You will need to install boost, FLAC, a BLAS/LAPACK -library, and the spt3g_software package. For this discussion, we will assume that you -have git checkouts of spt3g_software and so3g in:: + pip install -vv . - ${HOME}/git/spt3g_software - ${HOME}/git/so3g +Separate SPT3G +~~~~~~~~~~~~~~~~~ -And that you will be installing both spt3g_software and so3g into:: +If you are going to be developing so3g and repeatedly building it, you probably want to install spt3g once. See the `instructions from that package `_ to download and install. When building, you can install into your conda environment like this:: - ${HOME}/so3g + cd spt3g_software + mkdir -p build + cd build + cmake \ + -DCMAKE_INSTALL_PREFIX=${CONDA_PREFIX} \ + -DCMAKE_C_COMPILER=${CC} \ + -DCMAKE_CXX_COMPILER=${CXX} \ + -DPython_ROOT_DIR=${CONDA_PREFIX} \ + .. + make -j 4 install + # Copy the python package into place + cp -r ./spt3g ${CONDA_PREFIX}/lib/python3.12/site-packages/ -Adjust the instructions below if your situation is different. We also need to be -able to load this install prefix into our shell environment. There are several -ways of doing that. The first is to edit your bash shell resource file and add -a shell function like this:: +When building `so3g` against a stand-alone version of `spt3g`, you need to use cmake directly:: - load_so3g () { - # Load the appropriate python environment (edit this!) - source "${HOME}/simons/bin/activate" + cd so3g + mkdir -p build + cd build + cmake \ + -DCMAKE_INSTALL_PREFIX=${CONDA_PREFIX} \ + -DCMAKE_C_COMPILER=${CC} \ + -DCMAKE_CXX_COMPILER=${CXX} \ + -DPython_ROOT_DIR=${CONDA_PREFIX} \ + -DBLAS_LIBRARIES='-L${CONDA_PREFIX}/lib -lopenblas -fopenmp' \ + .. + make -j 4 install - # Location of our installed spt3g / so3g - dir="${HOME}/so3g" - # Prepend our executable search path - export PATH="${dir}/bin:${PATH}" - - # Get the python major / minor version - pyver=$(python3 --version 2>&1 | awk '{print $2}' \ - | sed -e "s#\(.*\)\.\(.*\)\..*#\1.\2#") - - # Put our python module into our search path - export PYTHONPATH="${dir}/lib/python${pyver}/site-packages" - - # Prepend our executable path and shared library search path. - # These lines are for linux: - if [ -z ${LD_LIBRARY_PATH} ]; then - export LD_LIBRARY_PATH="${dir}/lib" - else - export LD_LIBRARY_PATH="${dir}/lib:${LD_LIBRARY_PATH}" - fi - # These lines are for MacOS: - #if [ -z DYLD_LIBRARY_PATH ]; then - # export DYLD_LIBRARY_PATH="${dir}/lib" - #else - # export DYLD_LIBRARY_PATH="${dir}/lib:${DYLD_LIBRARY_PATH}" - #fi - } - -From a new shell, you can now run "load_so3g" to load your python stack -and put your install prefix into your environment. If you use environment -modules, see the README and example in the `modules/`_ directory. - -.. _modules/: ./modules - -Special Note on Conda ---------------------- - -If you are building spt3g / so3g from source, it is **highly** recommended -that you install dependencies from OS packages, use the python3 from your OS or -package manager, and use a virtualenv as described in the first section. -If you use a conda-provided python, then you have two choices: - -1. Build boost from source using that python (so that boost-python works - correctly). - -2. Install the conda package for boost (and other dependencies) and use - conda provided compilers to build spt3g / so3g. - -Both of these choices are beyond the scope of this README. Below we assume -that you are using a virtualenv created with the system (or homebrew / macports) -python3. - -Prerequisites on Linux ----------------------- - -The easiest approach in this case is to use your OS package manager. For -example:: - - apt install \ - libboost-all-dev \ - libopenblas-openmp-dev \ - libflac-dev - -Make sure your python virtualenv is activated. Next, download -and install spt3g_software -(https://github.com/CMB-S4/spt3g_software). Check the major / minor version -of your python (e.g. 3.7, 3.8 or 3.9). We use that information to install -spt3g into the correct site-packages directory. Below we assume an install -prefix of "${HOME}/so3g" and that we are using python3.9:: - - cd ${HOME}/git/spt3g_software - mkdir -p build - cd build - cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER="gcc" \ - -DCMAKE_CXX_COMPILER="g++" \ - -DCMAKE_C_FLAGS="-O3 -g -fPIC" \ - -DCMAKE_CXX_FLAGS="-O3 -g -fPIC -std=c++11" \ - -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ - -DPython_EXECUTABLE:FILEPATH=$(which python3) \ - -DPYTHON_MODULE_DIR="${HOME}/so3g/lib/python3.9/site-packages" \ - -DCMAKE_INSTALL_PREFIX="${HOME}/so3g" \ - .. - make -j 2 install - -Prerequisites on MacOS ----------------------- - -The so3g / spt3g_software does not seem to run on MacOS when built with the -clang++ compiler (unit tests fail with a cereal error). Instead, we will use -homebrew to install our dependencies and the latest gcc compiler tools:: - - brew install \ - flac \ - bzip2 \ - netcdf \ - sqlite3 \ - boost-python3 \ - gcc - -Next, download and install spt3g_software. Ensure that your virtualenv is -activated. Check the major / minor version of your python (e.g. 3.7, 3.8 -or 3.9). We use that information to install spt3g into our virtualenv or -conda environment. Below we assume that our environment is in our home -directory in a folder called "simons" and that we are using python3.9. -We further assume that the homebrew gcc version is called "gcc-11". -Also, this assumes that homebrew is installing things to /usr/local:: - - cd ${HOME}/git/spt3g_software - mkdir -p build - cd build - cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_COMPILER="gcc-11" \ - -DCMAKE_CXX_COMPILER="g++-11" \ - -DCMAKE_C_FLAGS="-O3 -g -fPIC" \ - -DCMAKE_CXX_FLAGS="-O3 -g -fPIC -std=c++11" \ - -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ - -DBOOST_ROOT="/usr/local" \ - -DPython_EXECUTABLE:FILEPATH=$(which python3) \ - -DPYTHON_MODULE_DIR="${HOME}/so3g/lib/python3.9/site-packages" \ - -DCMAKE_INSTALL_PREFIX="${HOME}/so3g" \ - .. - make -j 2 install - -Compilation and Installation +Building with OS Packages ---------------------------- -To compile and install the so3g package (assuming our same install -prefix of $HOME/so3g), we need to point it to the spt3g build directory -that we used previously. For example:: - - cd ${HOME}/git/so3g - mkdir -p build - cd build - cmake \ - -DCMAKE_PREFIX_PATH=${HOME}/git/spt3g_software/build \ - -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ - -DPYTHON_INSTALL_DEST="${HOME}/so3g" \ - -DCMAKE_INSTALL_PREFIX="${HOME}/so3g" \ - .. - make -j 2 install - -The definition of `CMAKE_PREFIX_PATH` must point to the build -directory for `spt3g`, because cmake output there will be used to -generate best compilation and/or linking instructions for Boost and -other dependencies of spt3g/so3g. - -Now you can run your "load_so3g" (or similar) command whenever you want -to load your python stack and also the so3g install prefix. - - -Local configuration through local.cmake ---------------------------------------- - -Optional, site-specific parameters may be set in the file local.cmake. -Lines declaring set(VARIABLE, value) should have the same effect as -passing -DVARIABLE=value to the cmake invocation. - -To change the destination directory for the installation, add lines -like this one:: - - set(CMAKE_INSTALL_PREFIX $ENV{HOME}/.local/) - set(PYTHON_INSTALL_DEST $ENV{HOME}/.local/lib/python3.7/site-packages/) - -To point cmake to the spt3g build directory, add a line like this -one:: - - set(CMAKE_PREFIX_PATH $ENV{HOME}/code/spt3g_software/build) +Another option is to use a virtualenv for python packages and use the compilers and +libraries from your OS to provide so3g dependencies. Install dependencies, for example:: + + apt install \ + libboost-all-dev \ + libopenblas-openmp-dev \ + libflac-dev \ + libgsl-dev \ + libnetcdf-dev + +Then activate your virtualenv. Next you should install to someplace in your library +search path. Note that the commands below will not work unless you change the install +prefix to a user-writable directory (or make install with sudo). You should decide where +you want to install and make sure that the location is in your PATH and +LD_LIBRARY_PATH:: + + cd spt3g_software + mkdir -p build + cd build + cmake \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + .. + make -j 4 install + # + +And similarly for so3g:: + + cd so3g + mkdir -p build + cd build + cmake \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBLAS_LIBRARIES='-lopenblas -fopenmp' \ + .. + make -j 4 install Testing diff --git a/conda_deps.txt b/conda_deps.txt deleted file mode 100644 index 43822872..00000000 --- a/conda_deps.txt +++ /dev/null @@ -1,14 +0,0 @@ -# Requirements to install with conda, if using -# that. -astropy -matplotlib -numpy -scipy -ephem -pytz -pyaml -sqlalchemy -pysqlite3 -tqdm -pytest -cmake diff --git a/conda_dev_requirements.txt b/conda_dev_requirements.txt new file mode 100644 index 00000000..d332cf99 --- /dev/null +++ b/conda_dev_requirements.txt @@ -0,0 +1,35 @@ +# This is the set of conda packages needed when creating +# a development environment for building spt3g / so3g +# using conda compilers and packages to meet dependencies. +# +# For example, assuming you have a conda-forge base environment +# you can do: +# +# conda create -n dev python==3.12 +# conda activate dev +# conda install --file conda_dev_requirements.txt +# python3 -m pip install -vv . +# +# Build tools +compilers +cmake +# Compiled dependencies +libopenblas=*=*openmp* +libblas=*=*openblas +openblas=*=*openmp* +boost +libflac +gsl +netcdf4 +# Python dependencies +numpy +scipy +astropy +matplotlib +ephem +pytz +pyaml +sqlalchemy +tqdm +# Eventually we should make a conda package for these: +# qpoint diff --git a/docker/so3g-setup.sh b/docker/so3g-setup.sh index 9e352e7b..f8594493 100644 --- a/docker/so3g-setup.sh +++ b/docker/so3g-setup.sh @@ -2,6 +2,10 @@ mkdir -p build cd build -cmake .. -make +cmake \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DPython_EXECUTABLE=$(which python3) \ + .. +make -j 2 make install diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..c9f1c65c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,51 @@ +[build-system] +requires = [ + "cmake>=3.17", + "setuptools", + "wheel", + "numpy", + "scipy", + # Astropy depends on numpy 1.x with python-3.9. Place + # a build-time dependency here so that we build with a + # compatible version of numpy. Remove this after dropping + # python-3.9 support. + "astropy", +] +build-backend = "setuptools.build_meta" + +[project] +name = "so3g" +readme = "README.rst" +description = "Tools for Simons Observatory work with spt3g_software" +urls = {source = "https://github.com/simonsobs/so3g"} +license = {file = "LICENSE"} +requires-python = ">=3.9" +dependencies = [ + "numpy", + "scipy", + "astropy", + "matplotlib", + "ephem", + "pytz", + "pyaml", + "sqlalchemy", + "tqdm", + "qpoint", +] +dynamic=["version"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Topic :: Scientific/Engineering :: Astronomy", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] + +[tool.pytest.ini_options] +addopts = [ + "--import-mode=importlib", +] diff --git a/python/__init__.py b/python/__init__.py index d5aee3fc..20baae72 100644 --- a/python/__init__.py +++ b/python/__init__.py @@ -1,12 +1,6 @@ import os import numpy as np -# Verify that we are using numpy 1.x -npversion = np.version.version -npversplit = npversion.split(".") -if npversplit[0] != "1": - msg = f"so3g requires numpy version 1.x but found version {npversion}" - raise RuntimeError(msg) if os.getenv('DOCS_BUILD') == '1': from ._libso3g_docstring_shells import * diff --git a/requirements.txt b/requirements.txt index b825a6b7..b890bf3f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,11 @@ astropy matplotlib -numpy<2 +numpy scipy ephem pytz pyaml sqlalchemy -pysqlite3-wheels tqdm cmake qpoint diff --git a/setup.py b/setup.py index d614cc7d..8bb4a75a 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,8 @@ from setuptools import setup, Extension from setuptools.command.build_ext import build_ext -from distutils.command.clean import clean + +import numpy as np # Absolute path to the directory with this file topdir = Path(__file__).resolve().parent @@ -40,10 +41,8 @@ def get_spt3g_version(): return ver upstream_spt3g_version = get_spt3g_version() -print(f"Using upstream spt3g_software version {upstream_spt3g_version}") # The name of the spt3g source and package dirs -spt3g_pkg_dir = os.path.join(topdir, "python", "spt3g_internal") spt3g_src_dir = os.path.join(topdir, "spt3g_software") @@ -66,8 +65,6 @@ def get_version(): def get_spt3g(): - # if os.path.isdir(spt3g_pkg_dir): - # return # We use git to get the repo, since spt3g uses git to get its version # information. if not os.path.isdir(spt3g_src_dir): @@ -102,23 +99,21 @@ def get_spt3g(): def extract_cmake_env(varprefix): cmake_opts = list() - cpat = re.compile(r"{}_(.*)".format(varprefix)) + cpat = re.compile(f"{varprefix}_(.*)") for k, v in os.environ.items(): mat = cpat.match(k) if mat is not None: - cmake_opts.append("-D{}={}".format(mat.group(1), v)) + cmake_opts.append(f"-D{mat.group(1)}={v}") return cmake_opts def build_common(src_dir, build_dir, install_dir, cmake_extra, debug, pkg, version): cmake_args = list() - #cmake_args = ["-DPYTHON_EXECUTABLE=" + sys.executable] - #cmake_args += ["-DBOOST_PYTHON_MAX_ARITY=20"] cfg = "Debug" if debug else "Release" cmake_args += ["-DCMAKE_BUILD_TYPE=" + cfg] cmake_args += ["-DCMAKE_VERBOSE_MAKEFILE=ON"] - cmake_args += ["-DCMAKE_INSTALL_PREFIX={}".format(install_dir)] - cmake_args.extend(extract_cmake_env("{}_BUILD".format(pkg))) + cmake_args += [f"-DCMAKE_INSTALL_PREFIX={install_dir}"] + cmake_args.extend(extract_cmake_env(f"{pkg}_BUILD")) cmake_args.extend(cmake_extra) build_args = ["--config", cfg] @@ -130,9 +125,14 @@ def build_common(src_dir, build_dir, install_dir, cmake_extra, debug, pkg, versi cxxcomp = env.get("CXX", None) cflags = env.get("CFLAGS", None) cxxflags = env.get("CXXFLAGS", "") - cxxflags = "{} -DVERSION_INFO='{}'".format(cxxflags, version) + cxxflags = f"{cxxflags} -DVERSION_INFO='{version}'" if sys.platform.lower() == "darwin": cmake_args += ["-DCMAKE_SHARED_LINKER_FLAGS='-undefined dynamic_lookup'"] + + # Add numpy includes + numpy_inc = np.get_include() + cxxflags += f" -I{numpy_inc}" + env["CXXFLAGS"] = cxxflags if ccomp is not None: @@ -148,11 +148,15 @@ def build_common(src_dir, build_dir, install_dir, cmake_extra, debug, pkg, versi # CMakeLists.txt is in the source dir cmake_list_dir = os.path.abspath(src_dir) - print("-" * 10, "Running {} CMake".format(pkg), "-" * 40) + print("-" * 10, f"Running {pkg} CMake", "-" * 40) + print(f"cmake {cmake_list_dir} {' '.join(cmake_args)}") sp.check_call(["cmake", cmake_list_dir] + cmake_args, cwd=build_dir, env=env) - print("-" * 10, "Building {}".format(pkg), "-" * 40) - cmake_cmd = ["cmake", "--build", "."] + build_args + ["--", "-j2"] + make_j = 2 + if "CPU_COUNT" in os.environ: + make_j = int(os.environ["CPU_COUNT"]) + print("-" * 10, f"Building {pkg}", "-" * 40) + cmake_cmd = ["cmake", "--build", "."] + build_args + ["--", f"-j{make_j}"] sp.check_call(cmake_cmd, cwd=build_dir) cmake_cmd = ["cmake", "--install", "."] + build_args sp.check_call(cmake_cmd, cwd=build_dir) @@ -178,36 +182,6 @@ def build_so3g(src_dir, build_dir, install_dir, cmake_extra, debug): get_spt3g() -class RealClean(clean): - """Really clean up. - - Delete all temporary build directories when running `python setup.py clean`. - """ - - def run(self): - super().run() - clean_files = [ - "./build", - "./dist", - "./__pycache__", - "./*.egg-info", - spt3g_pkg_dir, - spt3g_src_dir, - "./include/_version.h", - ] - for cf in clean_files: - if not os.path.exists(cf): - continue - # Make paths absolute and relative to this path - apaths = glob.glob(os.path.abspath(cf)) - for path in apaths: - if os.path.isdir(path): - shutil.rmtree(path) - elif os.path.isfile(path): - os.remove(path) - return - - class CMakeExtension(Extension): """ This overrides the built-in extension class and essentially does nothing, @@ -223,6 +197,10 @@ class CMakeBuild(build_ext): Builds the full package using CMake. """ + def initialize_options(self): + super().initialize_options() + self.cmake_build_done = False + def run(self): """ Perform build_cmake before doing the 'normal' stuff @@ -237,6 +215,8 @@ def run(self): # super().run() def build_cmake(self): + if self.cmake_build_done: + return try: out = sp.check_output(["cmake", "--version"]) except OSError: @@ -254,21 +234,20 @@ def build_cmake(self): # CMake build directory for spt3g temp_spt3g = os.path.join(temp_build, "spt3g") + # The python module in the spt3g build directory. This contains + # the compiled libraries and symlinks to the python source. + spt3g_python_dir = os.path.join(temp_spt3g, "spt3g") + # Use CMake to install to the distutils build location install_so3g = os.path.dirname( Path(self.get_ext_fullpath("so3g.libso3g")).resolve().parents[0] ) - # Use CMake to install spt3g python code into a subdirectory of so3g, but - # install the headers and other files to a separate location. + # Fake install directory passed to spt3g cmake. install_spt3g_fake = os.path.join(temp_build, "spt3g_install") - install_spt3g_py = install_so3g - # By default, the spt3g build system attempts to link to libpython, which - # should never be done when building wheels. This link resolution should - # only be done at runtime on the target system after installation. We - # have patched spt3g to not look for the python "Development" target, so - # here we specify the associated CMake variables directly. + # The cmake python discovery can be fragile. Here we override some + # artifacts explicitly. py_exe = sys.executable py_maj = sys.version_info[0] py_min = sys.version_info[1] @@ -283,15 +262,13 @@ def build_cmake(self): dlist3g = [ f"-DPython_EXECUTABLE={py_exe}", f"-DPython_INCLUDE_DIRS={py_incl}", - f"-DPython_LIBRARIES=''", - f"-DPython_RUNTIME_LIBRARY_DIRS=''", - f"-DPython_LIBRARY_DIRS=''", + "-DPython_LIBRARIES=''", + "-DPython_RUNTIME_LIBRARY_DIRS=''", + "-DPython_LIBRARY_DIRS=''", f"-DPython_VERSION_MAJOR={py_maj}", f"-DPython_VERSION_MINOR={py_min}", "-DBoost_ARCHITECTURE=-x64", f"-DBoost_PYTHON_TYPE=python{py_maj}{py_min}", - "-DBoost_DEBUG=ON", - f"-DPYTHON_MODULE_DIR={install_spt3g_py}", ] if "BOOST_ROOT" in os.environ: dlist3g.append(f"-DBOOST_ROOT={os.environ['BOOST_ROOT']}") @@ -306,7 +283,7 @@ def build_cmake(self): [ f"-DFLAC_LIBRARIES={flcroot}/lib/libFLAC.{flcext}", f"-DFLAC_INCLUDE_DIR={flcroot}/include", - f"-DFLAC_FOUND=1", + "-DFLAC_FOUND=1", ] ) @@ -318,28 +295,28 @@ def build_cmake(self): self.debug, ) - # Move spt3g python directory into place. Remove any stale copy of the - # directory. - sh_ext = os.path.splitext(sysconfig.get_config_var("EXT_SUFFIX"))[1] - - install_spt3g_internal = os.path.join(install_so3g, "so3g", "spt3g_internal") - if os.path.isdir(install_spt3g_internal): - print(f"rmtree {install_spt3g_internal}") - shutil.rmtree(install_spt3g_internal) - print(f"mv/rename {os.path.join(install_spt3g_py, 'spt3g')}, {install_spt3g_internal}") - os.rename(os.path.join(install_spt3g_py, "spt3g"), install_spt3g_internal) - build_so3g( topdir, temp_so3g, install_so3g, [ - "-DPYTHON_INSTALL_DEST={}".format(install_so3g), + f"-DPYTHON_INSTALL_DEST={install_so3g}", f"-DCMAKE_PREFIX_PATH={install_spt3g_fake}", ], self.debug, ) + # Move spt3g python directory into place. Remove any stale copy of the + # directory. + install_spt3g_internal = os.path.join(install_so3g, "so3g", "spt3g_internal") + if os.path.isdir(install_spt3g_internal): + print(f"rm stale: {install_spt3g_internal}") + shutil.rmtree(install_spt3g_internal) + print(f"copy {spt3g_python_dir}, {install_spt3g_internal}") + shutil.copytree(spt3g_python_dir, install_spt3g_internal, symlinks=False) + + self.cmake_build_done = True + ext_modules = [ CMakeExtension("so3g.libso3g"), @@ -350,41 +327,13 @@ def build_cmake(self): CMakeExtension("so3g.spt3g_internal.libspt3g-maps"), ] - # Install the python scripts from spt3g -scripts = glob.glob(os.path.join(spt3g_src_dir, "*", "bin", "*")) - - -def readme(): - with open("README.rst") as f: - return f.read() - +raw_scripts = glob.glob(os.path.join(spt3g_src_dir, "*", "bin", "*")) +scripts = [x.removeprefix(f"{topdir}/") for x in raw_scripts] conf = dict() conf["name"] = "so3g" -conf["description"] = "Tools for Simons Observatory work with spt3g_software" -conf["long_description"] = readme() -conf["long_description_content_type"] = "text/x-rst" -conf["author"] = "Simons Observatory Collaboration" -conf["author_email"] = "so_software@simonsobservatory.org" -conf["license"] = "MIT" -conf["url"] = "https://github.com/simonsobs/so3g" conf["version"] = get_version() -conf["python_requires"] = ">=3.7.0" -conf["setup_requires"] = (["wheel", "cmake"],) -conf["install_requires"] = [ - "numpy<2", - "astropy", - "matplotlib", - "scipy", - "ephem", - "pytz", - "pyaml", - "sqlalchemy", - "pysqlite3-wheels", - "tqdm", - "qpoint", -] # Since the so3g python package is in a directory called "python", we can't use the # normal find_packages() function to recursively set these up. Instead we specify them @@ -396,25 +345,14 @@ def readme(): } for sub in ["hk", "proj", "smurf"]: - psub = "so3g.{}".format(sub) + psub = f"so3g.{sub}" pdir = os.path.join("python", sub) conf["packages"].append(psub) conf["package_dir"][psub] = pdir conf["ext_modules"] = ext_modules conf["scripts"] = scripts -conf["cmdclass"] = {"build_ext": CMakeBuild, "clean": RealClean} +conf["cmdclass"] = {"build_ext": CMakeBuild} conf["zip_safe"] = False -conf["classifiers"] = [ - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: BSD License", - "Operating System :: POSIX", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Topic :: Scientific/Engineering :: Astronomy", -] setup(**conf) diff --git a/src/array_ops.cxx b/src/array_ops.cxx index ba1a88b6..21489acc 100644 --- a/src/array_ops.cxx +++ b/src/array_ops.cxx @@ -16,7 +16,9 @@ extern "C" { } #include -#include +#ifdef _OPENMP +# include +#endif // ifdef _OPENMP #include #include diff --git a/src/healpix_bare.c b/src/healpix_bare.c index 747a05a2..58721c74 100644 --- a/src/healpix_bare.c +++ b/src/healpix_bare.c @@ -65,8 +65,8 @@ static t_hpc loc2hpc (tloc loc) double jm = (1.0-tp)*tmp; /* decreasing edge line index */ if (jp>1.) jp = 1.; /* for points too close to the boundary */ if (jm>1.) jm = 1.; - return (loc.z >= 0) ? (t_hpc){1.-jm, 1.-jp, ntt} - : (t_hpc){jp, jm, ntt+8}; + return (loc.z >= 0) ? (t_hpc){1.-jm, 1.-jp, (int32_t)ntt} + : (t_hpc){jp, jm, (int32_t)ntt+8}; } static tloc hpc2loc (t_hpc hpc) @@ -173,7 +173,7 @@ static int64_t hpd2nest (int64_t nside, t_hpd hpd) static t_hpd nest2hpd (int64_t nside, int64_t pix) { int64_t npface_=nside*nside, p2=pix&(npface_-1); - return (t_hpd){compress_bits(p2), compress_bits(p2>>1), pix/npface_}; + return (t_hpd){compress_bits(p2), compress_bits(p2>>1), (int32_t)(pix/npface_)}; } static int64_t hpd2ring (int64_t nside_, t_hpd hpd) @@ -215,7 +215,7 @@ static t_hpd ring2hpd (int64_t nside_, int64_t pix) int64_t irt = iring - (jrll[face]*nside_) + 1; int64_t ipt = 2*iphi- jpll[face]*iring -1; if (ipt>=2*nside_) ipt-=8*nside_; - return (t_hpd) {(ipt-irt)>>1, (-(ipt+irt))>>1, face}; + return (t_hpd) {(ipt-irt)>>1, (-(ipt+irt))>>1, (int32_t)face}; } else if (pix<(npix_-ncap_)) /* Equatorial region */ { @@ -231,7 +231,7 @@ static t_hpd ring2hpd (int64_t nside_, int64_t pix) int64_t irt = iring - (jrll[face]*nside_) + 1; int64_t ipt = 2*iphi- jpll[face]*nside_ - kshift -1; if (ipt>=2*nside_) ipt-=8*nside_; - return (t_hpd) {(ipt-irt)>>1, (-(ipt+irt))>>1, face}; + return (t_hpd) {(ipt-irt)>>1, (-(ipt+irt))>>1, (int32_t)face}; } else /* South Polar cap */ { @@ -242,7 +242,7 @@ static t_hpd ring2hpd (int64_t nside_, int64_t pix) int64_t irt = 4*nside_ - iring - (jrll[face]*nside_) + 1; int64_t ipt = 2*iphi- jpll[face]*iring -1; if (ipt>=2*nside_) ipt-=8*nside_; - return (t_hpd) {(ipt-irt)>>1, (-(ipt+irt))>>1, face}; + return (t_hpd) {(ipt-irt)>>1, (-(ipt+irt))>>1, (int32_t)face}; } } @@ -263,7 +263,7 @@ int64_t ring2nest(int64_t nside, int64_t ipring) static t_hpd loc2hpd (int64_t nside_, tloc loc) { t_hpc tmp = loc2hpc(loc); - return (t_hpd){(tmp.x*nside_), (tmp.y*nside_), tmp.f}; + return (t_hpd){((int64_t)(tmp.x*nside_)), ((int64_t)(tmp.y*nside_)), tmp.f}; } static tloc hpd2loc (int64_t nside_, t_hpd hpd) diff --git a/test/test_proj_astro.py b/test/test_proj_astro.py index 3d488f30..63b744fd 100644 --- a/test/test_proj_astro.py +++ b/test/test_proj_astro.py @@ -237,7 +237,7 @@ def get_pyephem_radec(az, el, t, site, weather=None): esite.long = site.lon * DEG if weather is not None: weather.apply(esite) - d = datetime.datetime.utcfromtimestamp(t) + d = datetime.datetime.fromtimestamp(t, tz=datetime.timezone.utc) Xt = d.year, d.month, d.day, d.hour, d.minute, d.second+d.microsecond*1e-6 esite.date = ephem.date(Xt) return esite.radec_of(az, el) diff --git a/wheels/build_requirements.txt b/wheels/build_requirements.txt deleted file mode 100644 index 136da4ed..00000000 --- a/wheels/build_requirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -# This is the set of packages to install in the build environment -# AFTER installing our pinned version of numpy. -astropy -matplotlib -scipy -ephem -pytz -sqlalchemy -pysqlite3-wheels -tqdm -pytest diff --git a/wheels/install_deps_linux.sh b/wheels/install_deps_linux.sh index d02ed47e..7c1d3647 100755 --- a/wheels/install_deps_linux.sh +++ b/wheels/install_deps_linux.sh @@ -13,7 +13,7 @@ echo "Wheel script directory = ${scriptdir}" # Install library dependencies yum update -y -yum install -y flac-devel bzip2-devel zlib-devel sqlite-devel netcdf-devel +yum install -y bzip2-devel zlib-devel sqlite-devel # Build options @@ -29,37 +29,25 @@ MAKEJ=2 PREFIX=/usr/local +# Link lib64 directory to lib +echo "Symlink /usr/local/lib64 to /usr/local/lib" +rm -rf /usr/local/lib64 +ln -s /usr/local/lib /usr/local/lib64 + # Update pip -pip install --upgrade pip +python3 -m pip install --upgrade pip # Install a couple of base packages that are always required -pip install -v cmake wheel setuptools +python3 -m pip install -v cmake wheel setuptools -# In order to maximize ABI compatibility with numpy, build with the newest numpy -# version containing the oldest ABI version compatible with the python we are using. pyver=$(python3 --version 2>&1 | awk '{print $2}' | sed -e "s#\(.*\)\.\(.*\)\..*#\1.\2#") -if [ ${pyver} == "3.7" ]; then - numpy_ver="1.20" -fi -if [ ${pyver} == "3.8" ]; then - numpy_ver="1.20" -fi -if [ ${pyver} == "3.9" ]; then - numpy_ver="1.20" -fi -if [ ${pyver} == "3.10" ]; then - numpy_ver="1.22" -fi -if [ ${pyver} == "3.11" ]; then - numpy_ver="1.24" -fi # Install build requirements. -CC="${CC}" CFLAGS="${CFLAGS}" pip install -v -r "${scriptdir}/build_requirements.txt" "numpy<${numpy_ver}" +CC="${CC}" CFLAGS="${CFLAGS}" python3 -m pip install -v -r "${scriptdir}/../requirements.txt" # Install Openblas -openblas_version=0.3.28 +openblas_version=0.3.29 openblas_dir=OpenBLAS-${openblas_version} openblas_pkg=${openblas_dir}.tar.gz @@ -83,14 +71,14 @@ tar xzf ${openblas_pkg} \ # Install boost -boost_version=1_86_0 +boost_version=1_87_0 boost_dir=boost_${boost_version} boost_pkg=${boost_dir}.tar.bz2 echo "Fetching boost..." if [ ! -e ${boost_pkg} ]; then - curl -SL "https://archives.boost.io/release/1.86.0/source/${boost_pkg}" -o "${boost_pkg}" + curl -SL "https://archives.boost.io/release/1.87.0/source/${boost_pkg}" -o "${boost_pkg}" fi echo "Building boost..." @@ -102,7 +90,7 @@ tar xjf ${boost_pkg} \ && pushd ${boost_dir} \ && echo "using gcc : : ${CXX} ;" > tools/build/user-config.jam \ && echo "option jobs : ${MAKEJ} ;" >> tools/build/user-config.jam \ - && BOOST_BUILD_USER_CONFIG=tools/build/user-config.jam \ + && BOOST_BUILD_PATH=tools/build \ ./bootstrap.sh \ --with-python=python3 \ --prefix=${PREFIX} \ @@ -110,6 +98,46 @@ tar xjf ${boost_pkg} \ ${pyincl} cxxflags="${CXXFLAGS}" variant=release threading=multi link=shared runtime-link=shared install \ && popd >/dev/null 2>&1 +# Install libFLAC + +flac_version=1.5.0 +flac_dir=flac-${flac_version} +flac_pkg=${flac_dir}.tar.gz + +echo "Fetching libFLAC..." + +if [ ! -e ${flac_pkg} ]; then + curl -SL "https://github.com/xiph/flac/archive/refs/tags/${flac_version}.tar.gz" -o "${flac_pkg}" +fi + +echo "Building libFLAC..." + +rm -rf ${flac_dir} +tar xzf ${flac_pkg} \ + && pushd ${flac_dir} >/dev/null 2>&1 \ + && mkdir -p build \ + && pushd build >/dev/null 2>&1 \ + && cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER="${CC}" \ + -DCMAKE_C_FLAGS="${CFLAGS}" \ + -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ + -DBUILD_DOCS=OFF \ + -DWITH_OGG=OFF \ + -DBUILD_CXXLIBS=OFF \ + -DBUILD_PROGRAMS=OFF \ + -DBUILD_UTILS=OFF \ + -DBUILD_TESTING=OFF \ + -DBUILD_EXAMPLES=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DINSTALL_MANPAGES=OFF \ + -DENABLE_MULTITHREADING=ON \ + -DCMAKE_INSTALL_PREFIX="${PREFIX}" \ + .. \ + && make -j ${MAKEJ} install \ + && popd >/dev/null 2>&1 \ + && popd >/dev/null 2>&1 + # Build GSL gsl_version=2.8 diff --git a/wheels/install_deps_osx.sh b/wheels/install_deps_osx.sh index 06507c1f..4a12fad3 100755 --- a/wheels/install_deps_osx.sh +++ b/wheels/install_deps_osx.sh @@ -6,10 +6,6 @@ set -e -# Note: we are not cross-compiling here. This is for selecting the -# openblas tarball to fetch. -arch=$1 - # Location of this script pushd $(dirname $0) >/dev/null 2>&1 scriptdir=$(pwd) @@ -22,90 +18,157 @@ echo "Wheel script directory = ${scriptdir}" # runtime registration works. use_gcc=yes # use_gcc=no +gcc_version=14 if [ "x${use_gcc}" = "xyes" ]; then - CC=gcc-14 - CXX=g++-14 + CC=gcc-${gcc_version} + CXX=g++-${gcc_version} + FC=gfortran-${gcc_version} CFLAGS="-O3 -fPIC" CXXFLAGS="-O3 -fPIC -std=c++14" + FCFLAGS="-O3 -fPIC -pthread" + OMPFLAGS="-fopenmp" else + export MACOSX_DEPLOYMENT_TARGET=$(python3 -c "import sysconfig as s; print(s.get_config_vars()['MACOSX_DEPLOYMENT_TARGET'])") + echo "Using MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET}" CC=clang CXX=clang++ + #FC="" CFLAGS="-O3 -fPIC" - CXXFLAGS="-O3 -fPIC -std=c++11 -stdlib=libc++" + CXXFLAGS="-O3 -fPIC -std=c++14 -stdlib=libc++" + #FCFLAGS="" + #OMPFLAGS="" fi MAKEJ=2 PREFIX=/usr/local +# Workaround permissions on macos-14 github runner +# https://github.com/actions/runner-images/issues/9272 +sudo chown -R runner:admin /usr/local + # Install library dependencies with homebrew -brew install netcdf -brew install sqlite3 -brew install flac +brew install sqlite3 netcdf -# Optionally install gcc +# Optionally install gcc. if [ "x${use_gcc}" = "xyes" ]; then - brew install gcc@14 + brew install gcc@${gcc_version} fi # Update pip -pip install --upgrade pip +python3 -m pip install --upgrade pip # Install a couple of base packages that are always required -pip install -v cmake wheel setuptools +python3 -m pip install cmake wheel setuptools -# In order to maximize ABI compatibility with numpy, build with the newest numpy -# version containing the oldest ABI version compatible with the python we are using. pyver=$(python3 --version 2>&1 | awk '{print $2}' | sed -e "s#\(.*\)\.\(.*\)\..*#\1.\2#") -if [ ${pyver} == "3.7" ]; then - numpy_ver="1.20" -fi -if [ ${pyver} == "3.8" ]; then - numpy_ver="1.20" -fi -if [ ${pyver} == "3.9" ]; then - numpy_ver="1.20" + +# Install build requirements. +CC="${CC}" CFLAGS="${CFLAGS}" python3 -m pip install -v -r "${scriptdir}/../requirements.txt" scipy_openblas32 + +# Install Openblas + +openblas_version=0.3.29 +openblas_dir=OpenBLAS-${openblas_version} +openblas_pkg=${openblas_dir}.tar.gz + +if [ ! -e ${openblas_pkg} ]; then + echo "Fetching OpenBLAS..." + curl -SL https://github.com/xianyi/OpenBLAS/archive/v${openblas_version}.tar.gz -o ${openblas_pkg} fi -if [ ${pyver} == "3.10" ]; then - numpy_ver="1.22" + +echo "Building OpenBLAS..." + +omp="USE_OPENMP=1" +if [ "x${OMPFLAGS}" = "x" ]; then + omp="USE_OPENMP=0" fi -if [ ${pyver} == "3.11" ]; then - numpy_ver="1.24" + +rm -rf ${openblas_dir} +tar xzf ${openblas_pkg} \ + && pushd ${openblas_dir} >/dev/null 2>&1 \ + && make ${omp} NO_STATIC=1 \ + MAKE_NB_JOBS=${MAKEJ} \ + CC="${CC}" FC="${FC}" DYNAMIC_ARCH=1 TARGET=GENERIC \ + COMMON_OPT="${CFLAGS}" FCOMMON_OPT="${FCFLAGS}" \ + EXTRALIB="${OMPFLAGS}" libs netlib shared \ + && make NO_STATIC=1 DYNAMIC_ARCH=1 TARGET=GENERIC PREFIX="${PREFIX}" install \ + && popd >/dev/null 2>&1 + +# Install libFLAC + +flac_version=1.5.0 +flac_dir=flac-${flac_version} +flac_pkg=${flac_dir}.tar.gz + +echo "Fetching libFLAC..." + +if [ ! -e ${flac_pkg} ]; then + curl -SL "https://github.com/xiph/flac/archive/refs/tags/${flac_version}.tar.gz" -o "${flac_pkg}" fi -# Install build requirements. -CC="${CC}" CFLAGS="${CFLAGS}" pip install -v -r "${scriptdir}/build_requirements.txt" "numpy<${numpy_ver}" "scipy_openblas32" +echo "Building libFLAC..." + +rm -rf ${flac_dir} +tar xzf ${flac_pkg} \ + && pushd ${flac_dir} >/dev/null 2>&1 \ + && mkdir -p build \ + && pushd build >/dev/null 2>&1 \ + && cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER="${CC}" \ + -DCMAKE_C_FLAGS="${CFLAGS}" \ + -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ + -DBUILD_DOCS=OFF \ + -DWITH_OGG=OFF \ + -DBUILD_CXXLIBS=OFF \ + -DBUILD_PROGRAMS=OFF \ + -DBUILD_UTILS=OFF \ + -DBUILD_TESTING=OFF \ + -DBUILD_EXAMPLES=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DINSTALL_MANPAGES=OFF \ + -DENABLE_MULTITHREADING=ON \ + -DCMAKE_INSTALL_PREFIX="${PREFIX}" \ + .. \ + && make -j ${MAKEJ} install \ + && popd >/dev/null 2>&1 \ + && popd >/dev/null 2>&1 + +# Build GSL -# We use the scipy openblas wheel to get the openblas to use. +gsl_version=2.8 +gsl_dir=gsl-${gsl_version} +gsl_pkg=${gsl_dir}.tar.gz -# First ensure that pkg-config is set to search somewhere -if [ -z "${PKG_CONFIG_PATH}" ]; then - export PKG_CONFIG_PATH="/usr/local/lib/pkgconfig" +if [ ! -e ${gsl_pkg} ]; then + echo "Fetching GSL..." + curl -SL https://ftp.gnu.org/gnu/gsl/gsl-${gsl_version}.tar.gz -o ${gsl_pkg} fi -python3 -c "import scipy_openblas32; print(scipy_openblas32.get_pkg_config())" > ${PKG_CONFIG_PATH}/scipy-openblas.pc +echo "Building GSL..." -# To help delocate find the libraries, we copy them into /usr/local -python3 </dev/null 2>&1 \ + && mkdir -p build \ + && pushd build >/dev/null 2>&1 \ + && CC="${CC}" CFLAGS="-O3 -fPIC" ../configure --prefix="${PREFIX}" \ + && make -j ${MAKEJ} \ + && make install \ + && popd >/dev/null 2>&1 # Install boost -boost_version=1_86_0 +boost_version=1_87_0 boost_dir=boost_${boost_version} boost_pkg=${boost_dir}.tar.bz2 echo "Fetching boost..." if [ ! -e ${boost_pkg} ]; then - curl -SL "https://archives.boost.io/release/1.86.0/source/${boost_pkg}" -o "${boost_pkg}" + curl -SL "https://archives.boost.io/release/1.87.0/source/${boost_pkg}" -o "${boost_pkg}" fi echo "Building boost..." @@ -114,8 +177,10 @@ pyincl=$(for d in $(python3-config --includes | sed -e 's/-I//g'); do echo "incl use_line="using darwin : : ${CXX} ;" extra_link="linkflags=\"-stdlib=libc++\"" +#toolset="clang" if [ "x${use_gcc}" = "xyes" ]; then use_line="using gcc : : ${CXX} ;" + #toolset="gcc" extra_link="" fi @@ -124,7 +189,7 @@ tar xjf ${boost_pkg} \ && pushd ${boost_dir} \ && echo ${use_line} > tools/build/user-config.jam \ && echo "option jobs : ${MAKEJ} ;" >> tools/build/user-config.jam \ - && BOOST_BUILD_USER_CONFIG=tools/build/user-config.jam \ + && BOOST_BUILD_PATH=tools/build \ ./bootstrap.sh \ --with-python=python3 \ --prefix=${PREFIX} \ @@ -134,29 +199,6 @@ tar xjf ${boost_pkg} \ variant=release threading=multi link=shared runtime-link=shared install \ && popd >/dev/null 2>&1 -# Build GSL - -gsl_version=2.8 -gsl_dir=gsl-${gsl_version} -gsl_pkg=${gsl_dir}.tar.gz - -if [ ! -e ${gsl_pkg} ]; then - echo "Fetching GSL..." - curl -SL https://ftp.gnu.org/gnu/gsl/gsl-${gsl_version}.tar.gz -o ${gsl_pkg} -fi - -echo "Building GSL..." - -rm -rf ${gsl_dir} -tar xzf ${gsl_pkg} \ - && pushd ${gsl_dir} >/dev/null 2>&1 \ - && mkdir -p build \ - && pushd build >/dev/null 2>&1 \ - && CC="${CC}" CFLAGS="-O3 -fPIC" ../configure --prefix="${PREFIX}" \ - && make -j ${MAKEJ} \ - && make install \ - && popd >/dev/null 2>&1 - # Astropy caching... echo "Attempting to trigger astropy IERS download..." diff --git a/wheels/repair_wheel_linux.sh b/wheels/repair_wheel_linux.sh index 98eaafaf..0b9d42f7 100755 --- a/wheels/repair_wheel_linux.sh +++ b/wheels/repair_wheel_linux.sh @@ -10,9 +10,13 @@ set -e dest_dir=$1 wheel=$2 -spt3g_build=$(ls -d /project/build/lib.linux*/so3g/spt3g_internal) -so3g_build=$(ls -d /project/build/lib.linux*/so3g) +# Location of this script +pushd $(dirname $0) >/dev/null 2>&1 +scriptdir=$(pwd) +popd >/dev/null 2>&1 -export LD_LIBRARY_PATH=${spt3g_build}:${so3g_build}:${LD_LIBRARY_PATH} +# On Linux, we need to add this to LD_LIBRARY_PATH +spt3g_install=$(ls -d ${scriptdir}/../build/lib.*/so3g/spt3g_internal) +export LD_LIBRARY_PATH="/usr/local/lib":"${spt3g_install}":${LD_LIBRARY_PATH} auditwheel repair -w ${dest_dir} ${wheel} diff --git a/wheels/repair_wheel_macos.sh b/wheels/repair_wheel_macos.sh index b8fe4c74..a54191b7 100755 --- a/wheels/repair_wheel_macos.sh +++ b/wheels/repair_wheel_macos.sh @@ -11,9 +11,14 @@ dest_dir=$1 wheel=$2 delocate_archs=$3 -spt3g_libdir=$(ls -d /Users/runner/work/so3g/so3g/build/temp.macosx*/spt3g_install/lib) +# Location of this script +pushd $(dirname $0) >/dev/null 2>&1 +scriptdir=$(pwd) +popd >/dev/null 2>&1 -export DYLD_LIBRARY_PATH=/usr/local/lib:${spt3g_libdir}:${DYLD_LIBRARY_PATH} +spt3g_install=$(ls -d ${scriptdir}/../build/temp.*/spt3g/spt3g) +export DYLD_LIBRARY_PATH="/usr/local/lib":"${spt3g_install}":${DYLD_LIBRARY_PATH} + +delocate-listdeps --all ${wheel} \ +&& delocate-wheel -v --require-archs ${delocate_archs} -w ${dest_dir} ${wheel} -delocate-listdeps ${wheel} \ -&& delocate-wheel --require-archs ${delocate_archs} -w ${dest_dir} ${wheel} diff --git a/wheels/spt3g.patch b/wheels/spt3g_disable_tests.patch similarity index 100% rename from wheels/spt3g.patch rename to wheels/spt3g_disable_tests.patch diff --git a/wheels/spt3g_sys_time.patch b/wheels/spt3g_sys_time.patch new file mode 100644 index 00000000..ff4510e0 --- /dev/null +++ b/wheels/spt3g_sys_time.patch @@ -0,0 +1,12 @@ +diff -urN spt3g_software_orig/core/src/G3TimeStamp.cxx spt3g_software/core/src/G3TimeStamp.cxx +--- spt3g_software_orig/core/src/G3TimeStamp.cxx 2025-01-28 20:47:41.412059665 -0800 ++++ spt3g_software/core/src/G3TimeStamp.cxx 2025-01-28 21:09:58.481129822 -0800 +@@ -2,7 +2,7 @@ + #include + + #include +-#include ++#include + #include + #include + diff --git a/wheels/test_local_cibuildwheel.sh b/wheels/test_local_cibuildwheel.sh index 28932f89..1c39fc4c 100755 --- a/wheels/test_local_cibuildwheel.sh +++ b/wheels/test_local_cibuildwheel.sh @@ -5,7 +5,7 @@ export CIBW_DEBUG_KEEP_CONTAINER=TRUE -export CIBW_BUILD="cp311-manylinux_x86_64" +export CIBW_BUILD="cp312-manylinux_x86_64" export CIBW_MANYLINUX_X86_64_IMAGE="manylinux2014" export CIBW_BUILD_VERBOSITY=3 export CIBW_ENVIRONMENT_LINUX="CC=gcc CXX=g++ CFLAGS='-O3 -fPIC' CXXFLAGS='-O3 -fPIC -std=c++14'" diff --git a/wheels/test_local_macos.sh b/wheels/test_local_macos.sh index 88b1f226..195a7403 100755 --- a/wheels/test_local_macos.sh +++ b/wheels/test_local_macos.sh @@ -41,23 +41,25 @@ echo "Using homebrew installation in ${brew_root}" # Export compiler information use_gcc=yes -export CC=gcc-12 -export CXX=g++-12 -export FC=gfortran-12 +export CC=gcc-14 +export CXX=g++-14 +export FC=gfortran-14 # export CC=clang # export CXX=clang++ # export FC= export CFLAGS="-O3 -fPIC" export FCFLAGS="-O3 -fPIC" # Use the second when building with clang -CXXFLAGS="-O3 -fPIC -std=c++11" -#CXXFLAGS="-O3 -fPIC -std=c++11 -stdlib=libc++" +CXXFLAGS="-O3 -fPIC -std=c++14" +#CXXFLAGS="-O3 -fPIC -std=c++14 -stdlib=libc++" # Install most dependencies with homebrew, including python-3.9 eval ${brew_com} install flac eval ${brew_com} install bzip2 eval ${brew_com} install netcdf eval ${brew_com} install sqlite3 +eval ${brew_com} install openblas +eval ${brew_com} install gsl eval ${brew_com} install boost-python3 if [ "x${use_gcc}" = "xyes" ]; then @@ -101,40 +103,7 @@ python3 -m pip install delocate export BOOST_ROOT="${brew_root}" export LD_LIBRARY_PATH="${brew_root}/lib" export DYLD_LIBRARY_PATH="${brew_root}/lib" -export CPATH="${brew_root}/include" - -# Install the qpoint package -have_qpoint=$(python -c " -try: - import qpoint - print('yes') -except ImportError: - print('no') -") -if [ ${have_qpoint} = "yes" ]; then - echo "qpoint package already installed" -else - qpoint_version=828126de9f195f88bfaf1996527f633382457461 - qpoint_dir="qpoint_temp" - echo "Installing qpoint version ${qpoint_version}" - - echo "Fetching qpoint..." - if [ ! -d ${qpoint_dir} ]; then - git clone https://github.com/arahlin/qpoint.git ${qpoint_dir} - fi - - echo "Building qpoint..." - pushd ${qpoint_dir} \ - && git checkout master \ - && if [ "x$(git branch -l | grep so3g)" != x ]; then \ - git branch -D so3g; fi \ - && git fetch \ - && git checkout -b so3g ${qpoint_version} \ - && python3 setup.py clean \ - && python3 setup.py build \ - && python3 setup.py install \ - && popd > /dev/null -fi +export CPATH="${brew_root}/include:${brew_root}/opt/openblas/include" # Tell setup.py to look in the homebrew prefix for libraries when # building spt3g and so3g. @@ -143,6 +112,9 @@ export SPT3G_BUILD_CMAKE_LIBRARY_PATH="${brew_root}/lib" export SO3G_BUILD_CMAKE_INCLUDE_PATH="${brew_root}/include" export SO3G_BUILD_CMAKE_LIBRARY_PATH="${brew_root}/lib" +export SO3G_BUILD_BLA_VENDOR="OpenBLAS" +export SO3G_BUILD_BLAS_LIBRARIES="${brew_root}/opt/openblas/lib/libopenblas.dylib" + # Now build a wheel python3 setup.py clean python3 -m pip wheel ${topdir} --wheel-dir=build/temp_wheels --no-deps -vvv From 91428485d5b7d80995b0535bf2e8c0999b30f992 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Thu, 20 Feb 2025 19:58:29 -0800 Subject: [PATCH 2/3] Fix README typo --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 44dd4cf0..4a011b44 100644 --- a/README.rst +++ b/README.rst @@ -126,7 +126,8 @@ LD_LIBRARY_PATH:: -DCMAKE_INSTALL_PREFIX=/usr/local \ .. make -j 4 install - # + # Copy the python package into place + cp -r ./spt3g ${CONDA_PREFIX}/lib/python3.12/site-packages/ And similarly for so3g:: From d693c63e87f372c1c7d5b7ff1d9e93d732cd88ff Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Fri, 21 Feb 2025 08:33:20 -0800 Subject: [PATCH 3/3] Add note on the provenance and local modifications to the healpix_bare.c source --- src/healpix_bare.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/healpix_bare.c b/src/healpix_bare.c index 58721c74..2df37944 100644 --- a/src/healpix_bare.c +++ b/src/healpix_bare.c @@ -1,3 +1,11 @@ +// The original source in this file was downloaded from the "healpix_bare" +// package on sourceforge under the 3-clause BSD license (see copyright below). +// +// This code has been modified slightly from the original. In several cases, +// variables with type int64_t were being implicitly truncated to int32_t. The +// code has been changed to add explicit casts to int32_t in these cases to avoid +// compiler warnings / errors. + /* ----------------------------------------------------------------------------- * * Copyright (C) 1997-2019 Krzysztof M. Gorski, Eric Hivon, Martin Reinecke,