From 1c3f6b2c326df3e8e6bf19ec7d35b09588f64255 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Fri, 21 Feb 2025 11:27:39 -0800 Subject: [PATCH 01/10] This work moves the codebase to depend on the latest upstream spt3g. The upstream code has modernized the way its compiled extensions are built and imported. Changes include: - Build python extensions with cmake rather than building libraries and manually dlopening them. - Clean up imports throughout the codebase. Absolute imports of "so3g" are removed and replaced by relative imports from the internal submodules or from the compiled extension. Also clean up linter warnings. - Add a new github workflow that runs tests by building within a conda environment to get all dependencies and runs tests on a selection of python versions and linux/mac. - When building wheels, only build the 3 components of boost that we need. - After consultation with @mhasself and @BrianJKoopman, remove all docker support files that are no longer used. Encode the upstream version of spt3g directly in setup.py (rather than the Dockerfile). --- .github/workflows/official-docker-images.yml | 59 ------------- .github/workflows/pytest.yml | 38 -------- .github/workflows/test.yml | 91 ++++++++++++++++++++ .github/workflows/wheels.yml | 4 +- CMakeLists.txt | 20 +++-- Dockerfile | 30 ------- docker-compose.yml | 6 -- docker/qpoint-setup.sh | 6 -- docker/so3g-setup.sh | 11 --- python/__init__.py | 9 +- python/hk/cli.py | 15 ++-- python/hk/getdata.py | 64 +++++++------- python/hk/scanner.py | 20 +++-- python/hk/session.py | 11 +-- python/hk/translator.py | 11 ++- python/hk/tree.py | 10 +-- python/load_pybindings.py | 44 ---------- python/proj/__init__.py | 3 - python/proj/coords.py | 20 +++-- python/proj/mapthreads.py | 23 +++-- python/proj/quat.py | 36 ++++---- python/proj/ranges.py | 12 +-- python/proj/util.py | 2 - python/proj/wcs.py | 26 +++--- python/smurf/reader.py | 13 ++- python/smurf/smurf_archive.py | 25 +++--- setup.py | 26 +----- src/main.cxx | 4 +- wheels/install_deps_linux.sh | 3 +- wheels/install_deps_osx.sh | 5 +- wheels/spt3g_disable_tests.patch | 68 --------------- wheels/spt3g_sys_time.patch | 12 --- wheels/test_local_cibuildwheel.sh | 2 +- wheels/test_local_macos.sh | 2 +- 34 files changed, 268 insertions(+), 463 deletions(-) delete mode 100644 .github/workflows/official-docker-images.yml delete mode 100644 .github/workflows/pytest.yml create mode 100644 .github/workflows/test.yml delete mode 100644 Dockerfile delete mode 100644 docker-compose.yml delete mode 100644 docker/qpoint-setup.sh delete mode 100644 docker/so3g-setup.sh delete mode 100644 python/load_pybindings.py delete mode 100644 wheels/spt3g_disable_tests.patch delete mode 100644 wheels/spt3g_sys_time.patch diff --git a/.github/workflows/official-docker-images.yml b/.github/workflows/official-docker-images.yml deleted file mode 100644 index b2f58113..00000000 --- a/.github/workflows/official-docker-images.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Build Official Docker Images - -on: - push: - branches: - - master - tags: - - '*' - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - # Fetch all history for all tags and branches - with: - fetch-depth: 0 - - # Build - - name: Build docker images - run: | - docker compose build - - # Test (already been run by pytest workflow, but they don't take long...) - - name: Test with pytest within a docker container - run: | - docker run -v $PWD:/coverage --rm so3g sh -c "COVERAGE_FILE=/coverage/.coverage.docker python3 -m pytest --cov /usr/lib/python3/dist-packages/so3g/ test/" - - - name: Report test coverage - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - pip install coveralls - coverage combine - coverage report - coveralls --service=github - - # Dockerize - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.REGISTRY_USER }} - password: ${{ secrets.REGISTRY_PASSWORD }} - - - name: Build and push official docker image - env: - DOCKERHUB_ORG: "simonsobs" - run: | - export DOCKER_TAG=`git describe --tags --always` - - # Tag all images for upload to the registry - docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker tag {}:latest ${DOCKERHUB_ORG}/{}:latest - docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker tag {}:latest ${DOCKERHUB_ORG}/{}:${DOCKER_TAG} - - # Upload to docker registry - docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker push ${DOCKERHUB_ORG}/{}:latest - docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} docker push ${DOCKERHUB_ORG}/{}:${DOCKER_TAG} - docker compose config | grep 'image: ' | awk -F ': ' '{ print $2 }' | xargs -I {} echo ${DOCKERHUB_ORG}/{}:${DOCKER_TAG} pushed diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml deleted file mode 100644 index e4ce3b06..00000000 --- a/.github/workflows/pytest.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Run Tests - -on: - push: - branches: - - master - pull_request: - branches: - - master - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Build docker images - run: | - docker compose build - - - name: Test with pytest within a docker container - run: | - docker run -v $PWD:/coverage --rm so3g sh -c "COVERAGE_FILE=/coverage/.coverage.docker python3 -m pytest --cov /usr/lib/python3/dist-packages/so3g/ test/" - - - name: Report test coverage - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - pip install coveralls - coverage combine - coverage report - coveralls --service=github diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..a35dd441 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,91 @@ +# In general, we try to run on: +# - The oldest supported python +# - The latest stable python that is the common default on most systems and conda +# - (During transitions) The newly released bleeding edge python + +name: Run Tests + +on: + workflow_dispatch: + push: + branches: + - master + pull_request: + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Python-${{ matrix.python }} on ${{ matrix.arch }} + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash -l {0} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + python: "3.9" + arch: Linux-x86_64 + - os: ubuntu-latest + python: "3.11" + arch: Linux-x86_64 + - os: ubuntu-latest + python: "3.12" + arch: Linux-x86_64 + - os: macos-latest + python: "3.10" + arch: MacOSX-x86_64 + - os: macos-latest + python: "3.12" + arch: MacOSX-x86_64 + - os: macos-latest + python: "3.10" + arch: MacOSX-arm64 + - os: macos-latest + python: "3.12" + arch: MacOSX-arm64 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Conda Base + run: | + sudo rm -rf /usr/share/miniconda \ + && sudo rm -rf /usr/local/miniconda \ + && curl -SL -o miniforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-${{ matrix.arch }}.sh \ + && bash miniforge.sh -b -f -p ~/conda \ + && source ~/conda/etc/profile.d/conda.sh \ + && conda activate base \ + && conda update -n base --yes conda + - name: Check Conda Config + run: | + source ~/conda/etc/profile.d/conda.sh \ + && conda activate base \ + && conda info \ + && conda list \ + && conda config --show-sources \ + && conda config --show + - name: Install Dependencies + run: | + source ~/conda/etc/profile.d/conda.sh \ + && conda create --yes -n test python=${{ matrix.python }} \ + && conda activate test \ + && conda install --yes --file conda_dev_requirements.txt \ + && pip install -r test-requirements.txt + - name: Install + run: | + source ~/conda/etc/profile.d/conda.sh \ + && conda activate test \ + && python3 -m pip install . + - name: Run Tests + run: | + source ~/conda/etc/profile.d/conda.sh \ + && conda activate test \ + && export OMP_NUM_THREADS=2 \ + && pytest ./test diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 2c7da150..1a9edd44 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -101,7 +101,7 @@ jobs: CC=gcc CXX=g++ CFLAGS='-O3 -fPIC' - CXXFLAGS='-O3 -fPIC -std=c++14' + CXXFLAGS='-O3 -fPIC -std=c++17' BOOST_ROOT=/usr/local FLAC_ROOT=/usr/local SO3G_BUILD_BLAS_LIBRARIES='-L/usr/local/lib -lopenblas -fopenmp -lm -lgfortran' @@ -110,7 +110,7 @@ jobs: CC=gcc-14 CXX=g++-14 CFLAGS='-O3 -fPIC' - CXXFLAGS='-O3 -fPIC -std=c++14' + CXXFLAGS='-O3 -fPIC -std=c++17' CPATH='/usr/local/include' BOOST_ROOT=/usr/local FLAC_ROOT=/usr/local diff --git a/CMakeLists.txt b/CMakeLists.txt index 0cd83c01..fe70cdda 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,11 +54,11 @@ include_directories(${CMAKE_SOURCE_DIR}/include ${CMAKE_BINARY_DIR} ) include_directories(${NUMPY_INCLUDE_DIR}) # -# Define the so3g build target. This is a shared library. +# Define the libso3g build target. This is an internal library that holds all the +# linking dependencies. # set(CMAKE_LIBRARY_OUTPUT_DIRECTORY so3g) -add_library(so3g SHARED - src/main.cxx +add_library(so3g OBJECT src/hkagg.cxx src/Intervals.cxx src/Butterworth.cxx @@ -70,9 +70,6 @@ add_library(so3g SHARED src/array_ops.cxx ) -# We could disable the lib prefix on the output library... but let's not. -#set_target_properties(so3g PROPERTIES PREFIX "") - # Disable boost python auto_ptr warnings target_compile_definitions(so3g PUBLIC BOOST_NO_AUTO_PTR) @@ -124,6 +121,15 @@ add_custom_target(so3g-version add_dependencies(so3g so3g-version) +# Add the Python interface module +Python_add_library(_libso3g MODULE WITH_SOABI src/main.cxx) +set_target_properties(_libso3g PROPERTIES PREFIX "") +target_link_libraries(_libso3g PUBLIC so3g) +if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + # Assume Linux-style ld linker + target_link_options(_libso3g PUBLIC "LINKER:--no-as-needed") +endif() + # Make a list of .py files for the library. file(GLOB MY_PYTHONS "${CMAKE_CURRENT_SOURCE_DIR}/python/*.py") @@ -144,7 +150,7 @@ else() set(INSTALL_DEST ${PYTHON_SITE_PACKAGES}/so3g) endif() -install(TARGETS so3g +install(TARGETS _libso3g DESTINATION ${INSTALL_DEST}) install(FILES ${MY_PYTHONS} diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 1ec32b50..00000000 --- a/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -# so3g -# A containerized so3g installation. - -# Build on spt3g base image -FROM simonsobs/spt3g:0.3-289-g4bd3275 - -# Set locale -ENV LANG C.UTF-8 - -# Build tools needed for pixell; blas needed for so3g. -RUN apt update && apt install -y \ - build-essential \ - automake \ - gfortran \ - libopenblas-openmp-dev \ - libbz2-dev \ - python-is-python3 - -# Set the working directory -WORKDIR /app_lib/so3g - -# Copy the current directory contents into the container -ADD . /app_lib/so3g - -# Install any needed packages specified in requirements.txt -RUN pip3 install -r requirements.txt -RUN pip3 install -r test-requirements.txt - -# Build so3g -RUN /bin/bash /app_lib/so3g/docker/so3g-setup.sh diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 59cfb631..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: '3.2' -services: - - so3g: - image: "so3g" - build: . diff --git a/docker/qpoint-setup.sh b/docker/qpoint-setup.sh deleted file mode 100644 index a23dfd8e..00000000 --- a/docker/qpoint-setup.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -QP_VER=1.11.2 -git clone https://github.com/arahlin/qpoint.git --branch $QP_VER -cd qpoint -python3 setup.py install diff --git a/docker/so3g-setup.sh b/docker/so3g-setup.sh deleted file mode 100644 index f8594493..00000000 --- a/docker/so3g-setup.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -mkdir -p build -cd build -cmake \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DPython_EXECUTABLE=$(which python3) \ - .. -make -j 2 -make install diff --git a/python/__init__.py b/python/__init__.py index 20baae72..0101bc11 100644 --- a/python/__init__.py +++ b/python/__init__.py @@ -1,5 +1,4 @@ import os -import numpy as np if os.getenv('DOCS_BUILD') == '1': @@ -9,10 +8,10 @@ from . import spt3g from spt3g import core as spt3g_core - # Our library is called libso3g.{suffix}, but will load into module - # namespace so3g. - from .load_pybindings import load_pybindings - load_pybindings([__path__[0] + '/libso3g'], name='so3g') + # Load all symbols from our compiled extension. This is needed for backwards + # compatibility with external packages that expect those symbols to exist based + # on how the previous loader worked. + from ._libso3g import * # Version is computed by versioneer. __version__ = version() diff --git a/python/hk/cli.py b/python/hk/cli.py index 2cba3291..8257800c 100644 --- a/python/hk/cli.py +++ b/python/hk/cli.py @@ -1,11 +1,11 @@ -import so3g -from spt3g import core import numpy as np import os import sys import csv import argparse +from spt3g import core + _UNITS = { 'bytes': 1, @@ -49,7 +49,7 @@ def get_parser(): # Main "mode" subprocessors. # "list-files" - p = cmdsubp.add_parser( + _ = cmdsubp.add_parser( 'list-files', parents=[data_args, output_args], help="Report per-file stats.", @@ -62,7 +62,7 @@ def get_parser(): """) # "list-provs" - p = cmdsubp.add_parser( + _ = cmdsubp.add_parser( 'list-provs', parents=[data_args, output_args], help="List all data providers (feeds).", @@ -77,7 +77,7 @@ def get_parser(): """) # "list-fields" - p = cmdsubp.add_parser( + _ = cmdsubp.add_parser( 'list-fields', parents=[data_args, output_args], help="List all data field names.", @@ -242,7 +242,6 @@ def main(args=None): if args.mode == 'list-files': rows = [] - file_list = get_file_list for filename in get_file_list(args): file_size = os.path.getsize(filename) clean_exit = True @@ -250,7 +249,7 @@ def main(args=None): while True: try: f = r.Process(None) - except: + except Exception: clean_exit = False break end = r.tell() @@ -318,7 +317,7 @@ def main(args=None): n = len(block.times) for k in keys: field = addr + '.' + k - if not field in counts: + if field not in counts: counts[field] = 0 counts[field] += n header = ['field_name', 'samples'] diff --git a/python/hk/getdata.py b/python/hk/getdata.py index a7de3c34..8ac7fa5b 100644 --- a/python/hk/getdata.py +++ b/python/hk/getdata.py @@ -18,9 +18,10 @@ import numpy as np import datetime as dt - -import so3g from spt3g import core +from .. import _libso3g as libso3g + +from .translator import HKTranslator hk_logger = logging.getLogger(__name__) @@ -92,7 +93,7 @@ def __init__(self, field_groups=None): self.field_groups = list(field_groups) # A translator is used to update frames, on the fly, to the # modern schema assumed here. - self.translator = so3g.hk.HKTranslator() + self.translator = HKTranslator() def _get_groups(self, fields=None, start=None, end=None, short_match=False): @@ -136,7 +137,7 @@ def _get_groups(self, fields=None, start=None, end=None, in any group. """ - span = so3g.IntervalsDouble() + span = libso3g.IntervalsDouble() if start is None: start = span.domain[0] if end is None: @@ -170,7 +171,7 @@ def _get_groups(self, fields=None, start=None, end=None, continue elif full_field not in fields: continue - if not key_field in field_map: + if key_field not in field_map: field_map[key_field] = [fg] else: field_map[key_field].append(fg) @@ -292,7 +293,7 @@ def check_overlap(time_range): for time_range, n, filename, byte_offset, block_index in all_frame_refs: if not check_overlap(time_range): continue - if not filename in files: + if filename not in files: files[filename] = {} if byte_offset not in files[filename]: files[filename][byte_offset] = [] @@ -484,7 +485,7 @@ def __init__(self, pre_proc_dir=None, pre_proc_mode=None): self.field_groups = [] self.frame_info = [] self.counter = -1 - self.translator = so3g.hk.HKTranslator() + self.translator = HKTranslator() self.pre_proc_dir = pre_proc_dir self.pre_proc_mode = pre_proc_mode @@ -516,7 +517,7 @@ def Process(self, f, index_info=None): vers = f.get('hkagg_version', 0) assert(vers == 2) - if f['hkagg_type'] == so3g.HKFrameType.session: + if f['hkagg_type'] == libso3g.HKFrameType.session: session_id = f['session_id'] if self.session_id is not None: if self.session_id != session_id: @@ -526,7 +527,7 @@ def Process(self, f, index_info=None): (session_id, f['start_time']), unit='HKScanner') self.session_id = session_id - elif f['hkagg_type'] == so3g.HKFrameType.status: + elif f['hkagg_type'] == libso3g.HKFrameType.status: # If a provider has disappeared, flush its information into a # FieldGroup. prov_cands = [_HKProvider.from_g3(p) for p in f['providers']] @@ -539,10 +540,9 @@ def Process(self, f, index_info=None): for prov_id in to_flush: self.flush([prov_id]) - elif f['hkagg_type'] == so3g.HKFrameType.data: + elif f['hkagg_type'] == libso3g.HKFrameType.data: # Data frame -- merge info for this provider. prov = self.providers[f['prov_id']] - representatives = prov.blocks.keys() for bidx, (bname, b) in enumerate(zip(f['block_names'], f['blocks'])): assert(isinstance(b, core.G3TimesampleMap)) @@ -560,7 +560,7 @@ def Process(self, f, index_info=None): 'count': len(b.times)} ii.update(index_info) prov.blocks[bname]['index_info'].append(ii) - + else: core.log_warn('Weird hkagg_type: %i' % f['hkagg_type'], unit='HKScanner') @@ -665,7 +665,7 @@ def process_file_with_cache(self, filename): with open(path, 'wb') as pkfl: pickle.dump(hksc, pkfl) if self.pre_proc_mode is not None: - os.chmod( path, self.pre_proc_mode ) + os.chmod( path, self.pre_proc_mode ) self.field_groups += hksc.field_groups self.counter += hksc.counter @@ -711,17 +711,17 @@ class _FieldGroup: def __init__(self, prefix, fields, start, end, index_info): self.prefix = prefix self.fields = list(fields) - self.cover = so3g.IntervalsDouble().add_interval(start, end) + self.cover = libso3g.IntervalsDouble().add_interval(start, end) self.index_info = index_info def __repr__(self): try: return '_FieldGroup("%s", %i fields, %i segments)' % ( self.prefix, len(self.fields), len(self.index_info)) - except: + except Exception: return '_FieldGroup()' -def to_timestamp(some_time, str_format=None): +def to_timestamp(some_time, str_format=None): """Convert the argument to a unix timestamp. Args: @@ -736,25 +736,25 @@ def to_timestamp(some_time, str_format=None): float: Unix timestamp corresponding to some_time. """ - - if type(some_time) == dt.datetime: + + if isinstance(some_time, dt.datetime): return some_time.astimezone(dt.timezone.utc).timestamp() - if type(some_time) == int or type(some_time) == float: + if isinstance(some_time, (int, float)): return some_time - if type(some_time) == str: + if isinstance(some_time, str): if str_format is not None: return to_timestamp(pytz.utc.localize( dt.datetime.strptime(some_time, str_format ))) str_options = ['%Y-%m-%d', '%Y-%m-%d %H:%M', '%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M:%S.%f'] for option in str_options: try: return to_timestamp(pytz.utc.localize( dt.datetime.strptime(some_time, option ))) - except: + except Exception: continue raise ValueError('Could not process string into date object, options are: {}'.format(str_options)) - + raise ValueError('Type of date / time indication is invalid, accepts datetime, int, float, and string') -def load_range(start, stop, fields=None, alias=None, +def load_range(start, stop, fields=None, alias=None, data_dir=None, config=None, pre_proc_dir=None, pre_proc_mode=None, folder_patterns=None, strict=True): """Args: @@ -827,7 +827,7 @@ def load_range(start, stop, fields=None, alias=None, hk_logger.warning('''load_range has a config file - data_dir, fields, and alias are ignored''') with open(config, 'r') as f: setup = yaml.load(f, Loader=yaml.FullLoader) - + if 'data_dir' not in setup.keys(): raise ValueError('load_range config file requires data_dir entry') data_dir = setup['data_dir'] @@ -838,14 +838,14 @@ def load_range(start, stop, fields=None, alias=None, for k in setup['field_list']: fields.append( setup['field_list'][k]) alias.append( k ) - + if data_dir is None and 'OCS_DATA_DIR' not in os.environ.keys(): raise ValueError('if $OCS_DATA_DIR is not defined a data directory must be passed to getdata') if data_dir is None: data_dir = os.environ['OCS_DATA_DIR'] hk_logger.debug('Loading data from {}'.format(data_dir)) - + start_ctime = to_timestamp(start) - 3600 stop_ctime = to_timestamp(stop) + 3600 @@ -877,20 +877,20 @@ def load_range(start, stop, fields=None, alias=None, continue try: t = int(file[:-3]) - except: + except Exception: hk_logger.debug('{} does not have the right format, skipping'.format(file)) continue if t >= start_ctime-3600 and t <=stop_ctime+3600: hk_logger.debug('Processing {}'.format(base+'/'+file)) hksc.process_file_with_cache( base+'/'+file) - + cat = hksc.finalize() start_ctime = to_timestamp(start) stop_ctime = to_timestamp(stop) - + all_fields,_ = cat.get_fields() - + if fields is None: fields = all_fields if alias is not None: @@ -898,7 +898,7 @@ def load_range(start, stop, fields=None, alias=None, hk_logger.error('if provided, alias needs to be the length of fields') else: alias = fields - + # Single pass load. keepers = [] for name, field in zip(alias, fields): @@ -969,7 +969,7 @@ def load_range(start, stop, fields=None, alias=None, # This is the easy way, which just gives you one timeline per # requested field. x1, y1 = cat.simple(field_name) - + assert np.all(np.array(x0) == x1) and np.all(np.array(y0) == y1) import pylab as pl diff --git a/python/hk/scanner.py b/python/hk/scanner.py index 9fabbfc1..f3d300b5 100644 --- a/python/hk/scanner.py +++ b/python/hk/scanner.py @@ -1,12 +1,14 @@ -import so3g -from spt3g import core import numpy as np -from so3g import hk +from spt3g import core + +from .. import _libso3g as libso3g +from . import hk + class HKScanner: """Module that scans and reports on HK archive contents and compliance. - + Attributes: stats (dict): A nested dictionary of statistics that are updated as frames are processed by the module. Elements: @@ -62,7 +64,7 @@ def __call__(self, f): vers = f.get('hkagg_version', 0) self.stats['versions'][vers] = self.stats['versions'].get(vers, 0) + 1 - if f['hkagg_type'] == so3g.HKFrameType.session: + if f['hkagg_type'] == libso3g.HKFrameType.session: session_id = f['session_id'] if self.session_id is not None: if self.session_id != session_id: @@ -73,13 +75,13 @@ def __call__(self, f): self.session_id = session_id self.stats['n_session'] += 1 - elif f['hkagg_type'] == so3g.HKFrameType.status: + elif f['hkagg_type'] == libso3g.HKFrameType.status: # Have any providers disappeared? now_prov_id = [p['prov_id'].value for p in f['providers']] for p, info in self.providers.items(): if p not in now_prov_id: info['active'] = False - + # New providers? for p in now_prov_id: info = self.providers.get(p) @@ -102,7 +104,7 @@ def __call__(self, f): 'block_streams_map': {}, # Map from field name to block name. } - elif f['hkagg_type'] == so3g.HKFrameType.data: + elif f['hkagg_type'] == libso3g.HKFrameType.data: info = self.providers[f['prov_id']] vers = f.get('hkagg_version', 0) @@ -178,7 +180,7 @@ def __call__(self, f): 'data timestamp vectors (%s) .' % (t_this, t_check), unit='HKScanner') self.stats['concerns']['n_warning'] += 1 - + else: core.log_warn('Weird hkagg_type: %i' % f['hkagg_type'], unit='HKScanner') diff --git a/python/hk/session.py b/python/hk/session.py index f5b79986..43820305 100644 --- a/python/hk/session.py +++ b/python/hk/session.py @@ -1,9 +1,10 @@ -import so3g -from spt3g import core import time import os import binascii +from spt3g import core +from .. import _libso3g as libso3g + class HKSessionHelper: def __init__(self, session_id=None, start_time=None, hkagg_version=None, @@ -93,7 +94,7 @@ def session_frame(self): """ f = core.G3Frame() f.type = core.G3FrameType.Housekeeping - f['hkagg_type'] = so3g.HKFrameType.session + f['hkagg_type'] = libso3g.HKFrameType.session f['hkagg_version'] = self.hkagg_version f['session_id'] = self.session_id f['start_time'] = self.start_time @@ -110,7 +111,7 @@ def status_frame(self, timestamp=None): timestamp = time.time() f = core.G3Frame() f.type = core.G3FrameType.Housekeeping - f['hkagg_type'] = so3g.HKFrameType.status + f['hkagg_type'] = libso3g.HKFrameType.status f['hkagg_version'] = self.hkagg_version f['session_id'] = self.session_id f['timestamp'] = timestamp @@ -136,7 +137,7 @@ def data_frame(self, prov_id, timestamp=None): f = core.G3Frame() f.type = core.G3FrameType.Housekeeping f['hkagg_version'] = self.hkagg_version - f['hkagg_type'] = so3g.HKFrameType.data + f['hkagg_type'] = libso3g.HKFrameType.data f['session_id'] = self.session_id f['prov_id'] = prov_id f['timestamp'] = timestamp diff --git a/python/hk/translator.py b/python/hk/translator.py index d6c10741..8f962ca3 100644 --- a/python/hk/translator.py +++ b/python/hk/translator.py @@ -1,10 +1,10 @@ """Backwards compatibility for older SO HK schemas. """ - -import so3g -import so3g.hk from spt3g import core +from .. import _libso3g as libso3g + +from .util import get_g3_time class HKTranslator: @@ -89,7 +89,7 @@ def Process(self, f): f['hkagg_version'] = self.target_version # No difference in Session/Status for v0, v1, v2. - if f.get('hkagg_type') != so3g.HKFrameType.data: + if f.get('hkagg_type') != libso3g.HKFrameType.data: return [f] if self.target_version == 0: @@ -103,7 +103,7 @@ def Process(self, f): # Now process the data blocks. for block in orig_blocks: new_block = core.G3TimesampleMap() - new_block.times = so3g.hk.util.get_g3_time(block.t) + new_block.times = get_g3_time(block.t) for k in block.data.keys(): v = block.data[k] new_block[k] = core.G3VectorDouble(v) @@ -122,7 +122,6 @@ def Process(self, f): field_names = list(sorted(block.keys())) block_names.append('block_for_%s' % field_names[0]) assert(len(block_names[-1]) < 256) # What have you done. - orig_block_names = [] f['block_names'] = core.G3VectorString(block_names) return [f] diff --git a/python/hk/tree.py b/python/hk/tree.py index 643519bb..70c10efe 100644 --- a/python/hk/tree.py +++ b/python/hk/tree.py @@ -2,13 +2,13 @@ of attributes. """ - -from so3g.hk import getdata import time import os import yaml import logging +from .getdata import to_timestamp, HKArchiveScanner + logger = logging.getLogger(__name__) @@ -137,11 +137,11 @@ def __init__(self, start=None, stop=None, config=None, if start is None: start = now - 86400 else: - start = getdata.to_timestamp(start) + start = to_timestamp(start) if stop is None: stop = start + 86400 else: - stop = getdata.to_timestamp(stop) + stop = to_timestamp(stop) if aliases is None: aliases = {} @@ -170,7 +170,7 @@ def __init__(self, start=None, stop=None, config=None, # Walk the files -- same approach as load_ranges logger.debug('Scanning %s (pre_proc=%s)' % (data_dir, pre_proc_dir)) - hksc = getdata.HKArchiveScanner(pre_proc_dir=pre_proc_dir) + hksc = HKArchiveScanner(pre_proc_dir=pre_proc_dir) for folder in range(int(start / 1e5), int(stop / 1e5) + 1): base = os.path.join(data_dir, str(folder)) logger.debug(f' ... checking {base}') diff --git a/python/load_pybindings.py b/python/load_pybindings.py deleted file mode 100644 index 9a8ea4d4..00000000 --- a/python/load_pybindings.py +++ /dev/null @@ -1,44 +0,0 @@ -# -# Based on spt3g.core.load_bindings. -# -import platform, sys, os - -# Starting in spt3g 0.3-240-ga9d32d5, dload may be used. -from spt3g import dload - - -if platform.system().startswith('freebsd') or platform.system().startswith('FreeBSD'): - # C++ modules are extremely fragile when loaded with RTLD_LOCAL, - # which is what Python uses on FreeBSD by default, and maybe other - # systems. Convince it to use RTLD_GLOBAL. - - # See thread by Abrahams et al: - # http://mail.python.org/pipermail/python-dev/2002-May/024074.html - sys.setdlopenflags(0x102) - -def load_pybindings(paths, name=None, lib_suffix=None): - """ - Load all non-private items from the libraries in the list "paths". - Provide the full path to each library, but without extension. The - .so or .dylib will be appended depending on the system - architecture. The namespace into which the items are imported - will be determined from the first path, unless name= is explicitly - provided. - """ - if lib_suffix is None: - if platform.system().startswith('Darwin'): - # OSX compatibility requires .dylib suffix - lib_suffix = ".dylib" - else: - lib_suffix = ".so" - for path in paths: - if name is None: - name = os.path.split(path)[1] - # Save copy of current module def - mod = sys.modules[name] - m = dload.load_dynamic(name, name, path + lib_suffix) - sys.modules[name] = mod # Don't override Python mod with C++ - - for (k,v) in m.__dict__.items(): - if not k.startswith("_"): - mod.__dict__[k] = v diff --git a/python/proj/__init__.py b/python/proj/__init__.py index d0a9790c..5561ba53 100644 --- a/python/proj/__init__.py +++ b/python/proj/__init__.py @@ -1,6 +1,3 @@ -import so3g -from spt3g import core - from . import quat from . import util from . import mapthreads diff --git a/python/proj/coords.py b/python/proj/coords.py index 54b5d7d5..990e03a3 100644 --- a/python/proj/coords.py +++ b/python/proj/coords.py @@ -1,10 +1,9 @@ -import so3g +import numpy as np + +from .. import _libso3g as libso3g from . import quat from .weather import weather_factory -from collections import OrderedDict - -import numpy as np DEG = np.pi / 180. @@ -224,7 +223,7 @@ def coords(self, fplane=None, output=None): be [n_det,n_samp,{lon,lat,cos2psi,sin2psi}] """ # Get a projector, in CAR. - p = so3g.ProjEng_CAR_TQU_NonTiled((1, 1, 1., 1., 1., 1.)) + p = libso3g.ProjEng_CAR_TQU_NonTiled((1, 1, 1., 1., 1., 1.)) # Pre-process the offsets collapse = (fplane is None) if collapse: @@ -277,7 +276,8 @@ def __init__(self, quats=None, resps=None, dets=None): # quats will be an quat coeff array-2 and resps will be a numpy # array with the right shape, so we don't need to check # for this when we use FocalPlane later - if quats is None: quats = [] + if quats is None: + quats = [] # Asarray needed because G3VectorQuat doesn't handle list of lists, # which we want to be able to accept self.quats = quat.G3VectorQuat(np.asarray(quats)) @@ -370,7 +370,8 @@ def from_xieta(cls, *args, **kwargs): xi, eta, gamma, T, P, Q, U, hwp, dets = cls._xieta_compat(*args, **kwargs) gamma = gamma + np.arctan2(U,Q)/2 P = P * (Q**2+U**2)**0.5 - if hwp: gamma = -gamma + if hwp: + gamma = -gamma # Broadcast everything to 1d xi, eta, gamma, T, P, _ = np.broadcast_arrays(xi, eta, gamma, T, P, [0]) quats = quat.rotation_xieta(xi, eta, gamma) @@ -395,7 +396,8 @@ def __getitem__(self, sel): in which case an ``spt3g.core.quat`` is returned for that detector. This is provided for backwards compatibility.""" # FIXME: old sotodlib compat - remove later - if isinstance(sel, str): return self.quats[self._dets.index(sel)] + if isinstance(sel, str): + return self.quats[self._dets.index(sel)] # We go via .coeffs() here to get around G3VectorQuat's lack # of boolean mask support return FocalPlane(quats=self.coeffs()[sel], resps=self.resps[sel]) @@ -424,7 +426,7 @@ def __setitem__(self, name, q): # expects to be able to build up a focalplane by assigning # quats one at a time if name in self._detmap: - i = self._detmap[i] + i = self._detmap[name] self.quats[i] = q else: self._dets.append(name) diff --git a/python/proj/mapthreads.py b/python/proj/mapthreads.py index 92e71f2e..c6570783 100644 --- a/python/proj/mapthreads.py +++ b/python/proj/mapthreads.py @@ -5,10 +5,15 @@ parallelization. """ - -import so3g import numpy as np +from .. import _libso3g as libso3g + +from . import quat, DEG +from .coords import Assembly, FocalPlane +from .wcs import Projectionist + + def get_num_threads(n_threads=None): """Utility function for computing n_threads. If n_threads is not None, it is returned directly. But if it is None, then the OpenMP @@ -16,7 +21,7 @@ def get_num_threads(n_threads=None): """ if n_threads is None: - return so3g.useful_info()['omp_num_threads'] + return libso3g.useful_info()['omp_num_threads'] return n_threads def get_threads_domdir(sight, fplane, shape, wcs, tile_shape=None, @@ -77,11 +82,11 @@ def get_threads_domdir(sight, fplane, shape, wcs, tile_shape=None, active_tiles = [0] # The full assembly, for later. - asm_full = so3g.proj.Assembly.attach(sight, fplane) + asm_full = Assembly.attach(sight, fplane) # Get a Projectionist -- note it can be used with full or # representative assembly. - pmat = so3g.proj.wcs.Projectionist.for_tiled( + pmat = Projectionist.for_tiled( shape, wcs, tile_shape=tile_shape, active_tiles=active_tiles ) if active_tiles is None: @@ -93,9 +98,9 @@ def get_threads_domdir(sight, fplane, shape, wcs, tile_shape=None, # For the scan direction map, use the "representative" subset # detectors, with polarization direction aligned parallel to # elevation. - xi, eta, gamma = so3g.proj.quat.decompose_xieta(fplane_rep.quats) - fplane_xl = so3g.proj.FocalPlane.from_xieta(xi, eta, gamma*0+90*so3g.proj.DEG) - asm_rep = so3g.proj.Assembly.attach(sight, fplane_xl) + xi, eta, gamma = quat.decompose_xieta(fplane_rep.quats) + fplane_xl = FocalPlane.from_xieta(xi, eta, gamma*0+90*DEG) + asm_rep = Assembly.attach(sight, fplane_xl) sig = np.ones((fplane_xl.ndet, len(asm_rep.Q)), dtype='float32') scan_maps = pmat.to_map(sig, asm_rep, comps='TQU') @@ -107,7 +112,7 @@ def get_threads_domdir(sight, fplane, shape, wcs, tile_shape=None, phi = np.arctan2(U, Q) / 2 if plot_prefix: - text = 'Qf=%.2f Uf=%.2f phi=%.1f deg' % (Q/T, U/T, phi / so3g.proj.DEG) + text = 'Qf=%.2f Uf=%.2f phi=%.1f deg' % (Q/T, U/T, phi / DEG) for label, _m in tile_iter(scan_maps): for i in range(3): pl.imshow(_m[i], origin='lower') diff --git a/python/proj/quat.py b/python/proj/quat.py index c092d732..3bbd124e 100644 --- a/python/proj/quat.py +++ b/python/proj/quat.py @@ -1,10 +1,7 @@ import numpy as np -try: - from spt3g.core import quat, G3VectorQuat -except ImportError: - # Pre-Oct 2019 versions. - from spt3g.coordinateutils import quat, G3VectorQuat +from spt3g.core import Quat, G3VectorQuat + """We are using the spt3g quaternion containers, i.e. cu3g.G3VectorQuat and cu3g.quat. One way these are nice is that @@ -12,28 +9,25 @@ The component ordering is (1,i,j,k). We are restricted to 0d or 1d, and that's fine.""" -DEG = np.pi/180 - - def euler(axis, angle): """ The quaternion representing of an Euler rotation. - + For example, if axis=2 the computed quaternion(s) will have components: q = (cos(angle/2), 0, 0, sin(angle/2)) - + Parameters ---------- axis : {0, 1, 2} The index of the cartesian axis of the rotation (x, y, z). angle : float or 1-d float array Angle of rotation, in radians. - + Returns ------- - quat or G3VectorQuat, depending on ndim(angle). + Quat or G3VectorQuat, depending on ndim(angle). """ # Either angle or axis or both can be vectors. angle = np.asarray(angle) @@ -43,13 +37,13 @@ def euler(axis, angle): q[..., 0] = c q[..., axis+1] = s if len(shape) == 1: - return quat(*q) + return Quat(*q) return G3VectorQuat(q) def rotation_iso(theta, phi, psi=None): """Returns the quaternion that composes the Euler rotations: - + Qz(phi) Qy(theta) Qz(psi) Note arguments are in radians. @@ -62,9 +56,9 @@ def rotation_iso(theta, phi, psi=None): def rotation_lonlat(lon, lat, psi=0.): """Returns the quaternion that composes the Euler rotations: - + Qz(lon) Qy(pi/2 - lat) Qz(psi) - + Note arguments are in radians. """ return rotation_iso(np.pi/2 - lat, lon, psi) @@ -92,21 +86,21 @@ def rotation_xieta(xi, eta, gamma=0): def decompose_iso(q): """Decomposes the rotation encoded by q into the product of Euler rotations: - + q = Qz(phi) Qy(theta) Qz(psi) - + Parameters ---------- - q : quat or G3VectorQuat + q : Quat or G3VectorQuat The quaternion(s) to be decomposed. - + Returns ------- (theta, phi, psi) : tuple of floats or of 1-d arrays The rotation angles, in radians. """ - if isinstance(q, quat): + if isinstance(q, Quat): a,b,c,d = q.a, q.b, q.c, q.d else: a,b,c,d = np.transpose(q) diff --git a/python/proj/ranges.py b/python/proj/ranges.py index a5fde7f2..5bbdeee2 100644 --- a/python/proj/ranges.py +++ b/python/proj/ranges.py @@ -1,11 +1,13 @@ -import so3g import numpy as np +from .. import _libso3g as libso3g + + """Objects will self report as being of type "RangesInt32" rather than Ranges. But let's try to use so3g.proj.Ranges when testing types and making new ones and stuff.""" -Ranges = so3g.RangesInt32 +Ranges = libso3g.RangesInt32 class RangesMatrix(): @@ -43,7 +45,7 @@ def copy(self): def zeros_like(self): return RangesMatrix([x.zeros_like() for x in self.ranges], child_shape=self.shape[1:]) - + def ones_like(self): return RangesMatrix([x.ones_like() for x in self.ranges], child_shape=self.shape[1:]) @@ -52,7 +54,7 @@ def buffer(self, buff): [x.buffer(buff) for x in self.ranges] ## just to make this work like Ranges.buffer() return self - + def buffered(self, buff): out = self.copy() [x.buffer(buff) for x in out.ranges] @@ -127,7 +129,7 @@ def __add__(self, x): elif self.shape[0] == x.shape[0]: return self.__class__([r + d for r, d in zip(self.ranges, x)]) return self.__class__([r + x for r in self.ranges]) - + def __mul__(self, x): if isinstance(x, Ranges): return self.__class__([d * x for d in self.ranges]) diff --git a/python/proj/util.py b/python/proj/util.py index b9f8f2dc..656db295 100644 --- a/python/proj/util.py +++ b/python/proj/util.py @@ -1,5 +1,3 @@ -import numpy as np - def ces(el, az0, throw, v_scan, t): """Generate a CES scan pattern. diff --git a/python/proj/wcs.py b/python/proj/wcs.py index 8c2005d3..ce7ded8f 100644 --- a/python/proj/wcs.py +++ b/python/proj/wcs.py @@ -1,9 +1,10 @@ -import so3g -from . import quat - import numpy as np -from .ranges import Ranges, RangesMatrix +from .. import _libso3g as libso3g + +from . import quat + +from .ranges import RangesMatrix from . import mapthreads # For coordinate systems we use the following abbreviations: @@ -111,7 +112,8 @@ def get_ProjEng(self, comps='TQU', proj_name=None, get=True, configured geometry. """ - if proj_name is None: proj_name = self.proj_name + if proj_name is None: + proj_name = self.proj_name tile_suffix = 'Tiled' if self.tiling else 'NonTiled' # Interpolation mode @@ -128,7 +130,7 @@ def get_ProjEng(self, comps='TQU', proj_name=None, get=True, if not get: return projeng_name try: - projeng_cls = getattr(so3g, projeng_name) + projeng_cls = getattr(libso3g, projeng_name) except AttributeError: raise ValueError(f'There is no projector implemented for ' f'pixelization "{proj_name}", components ' @@ -445,7 +447,7 @@ def get_active_tiles(self, assembly, assign=False): tiles = np.nonzero(hits)[0] hits = hits[tiles] if assign is True: - assign = so3g.useful_info()['omp_num_threads'] + assign = libso3g.useful_info()['omp_num_threads'] if assign > 0: group_n = np.array([0 for g in range(assign)]) group_tiles = [[] for _ in group_n] @@ -454,7 +456,7 @@ def get_active_tiles(self, assembly, assign=False): idx = group_n.argmin() group_n[idx] += _n group_tiles[idx].append(_t) - imax = np.argmax(group_n) + # imax = np.argmax(group_n) # max_ratio = group_n[imax] / np.mean(np.concatenate([group_n[:imax], group_n[imax+1:]])) # if len(group_n)>1 and max_ratio > 1.1: # print(f"Warning: Threads poorly balanced. Max/mean hits = {max_ratio}") @@ -543,8 +545,8 @@ def get_q(wcs): quat.euler(1, (delta0 - 90)*quat.DEG) * quat.euler(2, -alpha0 * quat.DEG)) else: - raise ValueError(f'Unimplemented NSC reference (phi0,theta0)=' - '({wcs.wcs.phi0:.2f},{wcs.wcs.theta0:.2f})') + raise ValueError('Unimplemented NSC reference (phi0,theta0)=' + f'({wcs.wcs.phi0:.2f},{wcs.wcs.theta0:.2f})') return Q def __init__(self): @@ -720,7 +722,7 @@ def __init__(self): self._q_fp_to_celestial = None self.active_tiles = None self.proj_name = None - self.q_celestial_to_native = quat.quat(1,0,0,0) + self.q_celestial_to_native = quat.Quat(1,0,0,0) self.interpol = 'nearest' self.tiling = None @@ -771,7 +773,7 @@ def compute_nside_tile(self, assembly, nActivePerThread=5, nThreads=None): nActive = len(self.get_active_tiles(assembly)['active_tiles']) fsky = nActive / (12 * nside_tile0**2) if nThreads is None: - nThreads = so3g.useful_info()['omp_num_threads'] + nThreads = libso3g.useful_info()['omp_num_threads'] # nside_tile is smallest power of 2 satisfying nTile >= nActivePerThread * nthread / fsky self.nside_tile = int(2**np.ceil(0.5 * np.log2(nActivePerThread * nThreads / (12 * fsky)))) self.nside_tile = min(self.nside_tile, self.nside) diff --git a/python/smurf/reader.py b/python/smurf/reader.py index 1565bbe0..0f0a48ac 100644 --- a/python/smurf/reader.py +++ b/python/smurf/reader.py @@ -1,12 +1,11 @@ -import so3g -from spt3g import core import numpy as np import pickle -import datetime, time -import sys, os import warnings import argparse +from spt3g import core + + def g3_to_array(g3file, verbose=False): """ Takes a G3 file output from the SMuRF archiver and reads to a numpy array. @@ -22,7 +21,7 @@ def g3_to_array(g3file, verbose=False): data : array of arrays, where each internal array is a SMuRF channel """ frames = [fr for fr in core.G3File(g3file)] - + data=[] frametimes = [] @@ -35,13 +34,13 @@ def g3_to_array(g3file, verbose=False): warnings.warn('Wrong frame type') strtimes = np.hstack(frametimes) - + times = [] for strt in strtimes: t=core.G3Time(strt).time/core.G3Units.s times.append(t) times = np.asarray(times) - + channums = [] i=0 diff --git a/python/smurf/smurf_archive.py b/python/smurf/smurf_archive.py index 6605f58f..969aeaeb 100644 --- a/python/smurf/smurf_archive.py +++ b/python/smurf/smurf_archive.py @@ -1,18 +1,19 @@ +import ast +from collections import namedtuple +import datetime as dt +from enum import Enum +import os + import sqlalchemy as db from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker, relationship, backref +from sqlalchemy.orm import sessionmaker, relationship -from spt3g import core -import so3g -import datetime as dt -import os -from tqdm import tqdm import numpy as np +from tqdm import tqdm import yaml -import ast -from collections import namedtuple -from enum import Enum + +from spt3g import core Base = declarative_base() @@ -164,7 +165,7 @@ def add_file(self, path, session): db_file = Files(path=path) session.add(db_file) - reader = so3g.G3Reader(path) + reader = core.G3Reader(path) total_channels = 0 file_start, file_stop = None, None @@ -326,7 +327,7 @@ def load_data(self, start, end, show_pb=True, load_biases=True): for frame_info in tqdm(frames, total=num_frames, disable=(not show_pb)): file = frame_info.file.path if file != cur_file: - reader = so3g.G3Reader(file) + reader = core.G3Reader(file) cur_file = file reader.seek(frame_info.offset) @@ -400,7 +401,7 @@ def load_status(self, time, show_pb=False): for frame_info in tqdm(status_frames.all(), disable=(not show_pb)): file = frame_info.file.path if file != cur_file: - reader = so3g.G3Reader(file) + reader = core.G3Reader(file) cur_file = file reader.seek(frame_info.offset) frame = reader.Process(None)[0] diff --git a/setup.py b/setup.py index 8bb4a75a..6c18271c 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,6 @@ import os import sys -import sysconfig import re import subprocess as sp import glob @@ -19,26 +18,9 @@ # Absolute path to the directory with this file topdir = Path(__file__).resolve().parent -# The version of spt3g we will be installing. Get this from the -# Dockerfile for consistency. +# The version of spt3g we will be installing. def get_spt3g_version(): - dockerfile = os.path.join(topdir, "Dockerfile") - ver = None - linepat = re.compile(r".*simonsobs/spt3g:(.*)\s*") - verpat = re.compile(r".*-g(.*)") - with open(dockerfile, "r") as f: - for line in f: - mat = linepat.match(line) - if mat is not None: - fullver = mat.group(1) - vermat = verpat.match(fullver) - if vermat is None: - # This must be an actual tag - ver = fullver - else: - # Extract the short hash - ver = vermat.group(1) - return ver + return "66f373d1b4bf9076fe0af8236eed022c3d006664" upstream_spt3g_version = get_spt3g_version() @@ -56,7 +38,7 @@ def get_version(): ver_info = ver_function() ver = ver_info["version"] sys.path.pop(0) - except: + except Exception: raise RuntimeError("Cannot call get_versions() from version_h.py!") return ver @@ -128,7 +110,7 @@ def build_common(src_dir, build_dir, install_dir, cmake_extra, debug, pkg, versi cxxflags = f"{cxxflags} -DVERSION_INFO='{version}'" if sys.platform.lower() == "darwin": cmake_args += ["-DCMAKE_SHARED_LINKER_FLAGS='-undefined dynamic_lookup'"] - + # Add numpy includes numpy_inc = np.get_include() cxxflags += f" -I{numpy_inc}" diff --git a/src/main.cxx b/src/main.cxx index 97b761ca..6e299953 100644 --- a/src/main.cxx +++ b/src/main.cxx @@ -39,8 +39,6 @@ bp::object useful_info() { } - - PYBINDINGS("so3g") { bp::def("version", version); bp::def("useful_info", useful_info); @@ -51,7 +49,7 @@ static void* _so3g_import_array() { return NULL; } -BOOST_PYTHON_MODULE(so3g) { +BOOST_PYTHON_MODULE(_libso3g) { _so3g_import_array(); G3ModuleRegistrator::CallRegistrarsFor("so3g"); } diff --git a/wheels/install_deps_linux.sh b/wheels/install_deps_linux.sh index 7c1d3647..6bf983cd 100755 --- a/wheels/install_deps_linux.sh +++ b/wheels/install_deps_linux.sh @@ -22,7 +22,7 @@ CXX=g++ FC=gfortran CFLAGS="-O3 -fPIC -pthread" -CXXFLAGS="-O3 -fPIC -pthread -std=c++14" +CXXFLAGS="-O3 -fPIC -pthread -std=c++17" FCFLAGS="-O3 -fPIC -pthread" MAKEJ=2 @@ -94,6 +94,7 @@ tar xjf ${boost_pkg} \ ./bootstrap.sh \ --with-python=python3 \ --prefix=${PREFIX} \ + --with-libraries="iostreams,python,regex" \ && ./b2 --layout=tagged --user-config=./tools/build/user-config.jam \ ${pyincl} cxxflags="${CXXFLAGS}" variant=release threading=multi link=shared runtime-link=shared install \ && popd >/dev/null 2>&1 diff --git a/wheels/install_deps_osx.sh b/wheels/install_deps_osx.sh index 4a12fad3..112e2dd6 100755 --- a/wheels/install_deps_osx.sh +++ b/wheels/install_deps_osx.sh @@ -25,7 +25,7 @@ if [ "x${use_gcc}" = "xyes" ]; then CXX=g++-${gcc_version} FC=gfortran-${gcc_version} CFLAGS="-O3 -fPIC" - CXXFLAGS="-O3 -fPIC -std=c++14" + CXXFLAGS="-O3 -fPIC -std=c++17" FCFLAGS="-O3 -fPIC -pthread" OMPFLAGS="-fopenmp" else @@ -35,7 +35,7 @@ else CXX=clang++ #FC="" CFLAGS="-O3 -fPIC" - CXXFLAGS="-O3 -fPIC -std=c++14 -stdlib=libc++" + CXXFLAGS="-O3 -fPIC -std=c++17 -stdlib=libc++" #FCFLAGS="" #OMPFLAGS="" fi @@ -193,6 +193,7 @@ tar xjf ${boost_pkg} \ ./bootstrap.sh \ --with-python=python3 \ --prefix=${PREFIX} \ + --with-libraries="iostreams,python,regex" \ && ./b2 --layout=tagged --user-config=./tools/build/user-config.jam \ ${pyincl} -sNO_LZMA=1 -sNO_ZSTD=1 \ cxxflags="${CXXFLAGS}" ${extra_link} \ diff --git a/wheels/spt3g_disable_tests.patch b/wheels/spt3g_disable_tests.patch deleted file mode 100644 index 18c42aaf..00000000 --- a/wheels/spt3g_disable_tests.patch +++ /dev/null @@ -1,68 +0,0 @@ -diff -urN spt3g_software_orig/cmake/Spt3gBoostPython.cmake spt3g_software_export/cmake/Spt3gBoostPython.cmake ---- spt3g_software_orig/cmake/Spt3gBoostPython.cmake 2024-08-22 10:25:14.077183587 -0700 -+++ spt3g_software_export/cmake/Spt3gBoostPython.cmake 2024-12-11 12:24:37.355444860 -0800 -@@ -1,7 +1,7 @@ - # Locate Python - - if(${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.12) -- find_package(Python COMPONENTS Interpreter Development) -+ find_package(Python COMPONENTS Interpreter) - else() - find_package(PythonInterp) - find_package(PythonLibs ${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}) -diff -urN spt3g_software_orig/CMakeLists.txt spt3g_software_export/CMakeLists.txt ---- spt3g_software_orig/CMakeLists.txt 2024-08-22 10:24:59.301256298 -0700 -+++ spt3g_software_export/CMakeLists.txt 2024-12-11 12:24:37.364444816 -0800 -@@ -42,7 +42,7 @@ - - # Raise errors on every warning by default - # (use target-specific options to disable particular warnings) --set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror") -+#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror") - - # Interface library for flags and library dependencies - add_library(spt3g INTERFACE) -diff -urN spt3g_software_orig/core/CMakeLists.txt spt3g_software_export/core/CMakeLists.txt ---- spt3g_software_orig/core/CMakeLists.txt 2024-08-06 11:34:45.598647939 -0700 -+++ spt3g_software_export/core/CMakeLists.txt 2024-12-11 12:24:37.364444816 -0800 -@@ -105,8 +105,8 @@ - add_spt3g_test(quaternions) - add_spt3g_test(timesample) - --add_spt3g_test_program(test -- SOURCE_FILES -- ${CMAKE_CURRENT_SOURCE_DIR}/tests/G3TimestreamTest.cxx -- ${CMAKE_CURRENT_SOURCE_DIR}/tests/G3TimestreamMapTest.cxx -- USE_PROJECTS core) -+#add_spt3g_test_program(test -+# SOURCE_FILES -+# ${CMAKE_CURRENT_SOURCE_DIR}/tests/G3TimestreamTest.cxx -+# ${CMAKE_CURRENT_SOURCE_DIR}/tests/G3TimestreamMapTest.cxx -+# USE_PROJECTS core) -diff -urN spt3g_software_orig/core/src/dataio.cxx spt3g_software_export/core/src/dataio.cxx ---- spt3g_software_orig/core/src/dataio.cxx 2024-08-06 11:34:45.606647906 -0700 -+++ spt3g_software_export/core/src/dataio.cxx 2024-12-11 12:24:45.732404214 -0800 -@@ -146,8 +146,14 @@ - stream.push(fs); - } else { - // Simple file case -+ const char * bufcheck = getenv("SO3G_FILESYSTEM_BUFFER"); -+ // Use 20MB default -+ size_t so3g_buffer_size = 20971520; -+ if (bufcheck != nullptr) { -+ so3g_buffer_size = (size_t)atol(bufcheck); -+ } - stream.push(boost::iostreams::file_source(path, -- std::ios::binary)); -+ std::ios::binary), so3g_buffer_size); - } - - return fd; -diff -urN spt3g_software_orig/examples/CMakeLists.txt spt3g_software_export/examples/CMakeLists.txt ---- spt3g_software_orig/examples/CMakeLists.txt 2024-08-06 11:34:45.610647890 -0700 -+++ spt3g_software_export/examples/CMakeLists.txt 2024-12-11 12:24:37.365444811 -0800 -@@ -1,2 +1,2 @@ --add_executable(cppexample cppexample.cxx) --target_link_libraries(cppexample core) -+#add_executable(cppexample cppexample.cxx) -+#target_link_libraries(cppexample core) diff --git a/wheels/spt3g_sys_time.patch b/wheels/spt3g_sys_time.patch deleted file mode 100644 index ff4510e0..00000000 --- a/wheels/spt3g_sys_time.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff -urN spt3g_software_orig/core/src/G3TimeStamp.cxx spt3g_software/core/src/G3TimeStamp.cxx ---- spt3g_software_orig/core/src/G3TimeStamp.cxx 2025-01-28 20:47:41.412059665 -0800 -+++ spt3g_software/core/src/G3TimeStamp.cxx 2025-01-28 21:09:58.481129822 -0800 -@@ -2,7 +2,7 @@ - #include - - #include --#include -+#include - #include - #include - diff --git a/wheels/test_local_cibuildwheel.sh b/wheels/test_local_cibuildwheel.sh index 1c39fc4c..2b879535 100755 --- a/wheels/test_local_cibuildwheel.sh +++ b/wheels/test_local_cibuildwheel.sh @@ -8,7 +8,7 @@ export CIBW_DEBUG_KEEP_CONTAINER=TRUE export CIBW_BUILD="cp312-manylinux_x86_64" export CIBW_MANYLINUX_X86_64_IMAGE="manylinux2014" export CIBW_BUILD_VERBOSITY=3 -export CIBW_ENVIRONMENT_LINUX="CC=gcc CXX=g++ CFLAGS='-O3 -fPIC' CXXFLAGS='-O3 -fPIC -std=c++14'" +export CIBW_ENVIRONMENT_LINUX="CC=gcc CXX=g++ CFLAGS='-O3 -fPIC' CXXFLAGS='-O3 -fPIC -std=c++17'" export CIBW_BEFORE_BUILD_LINUX="./wheels/install_deps_linux.sh" export CIBW_REPAIR_WHEEL_COMMAND_LINUX="./wheels/repair_wheel_linux.sh {dest_dir} {wheel}" export CIBW_BEFORE_TEST="export OMP_NUM_THREADS=2" diff --git a/wheels/test_local_macos.sh b/wheels/test_local_macos.sh index 195a7403..90abf51e 100755 --- a/wheels/test_local_macos.sh +++ b/wheels/test_local_macos.sh @@ -50,7 +50,7 @@ export FC=gfortran-14 export CFLAGS="-O3 -fPIC" export FCFLAGS="-O3 -fPIC" # Use the second when building with clang -CXXFLAGS="-O3 -fPIC -std=c++14" +CXXFLAGS="-O3 -fPIC -std=c++17" #CXXFLAGS="-O3 -fPIC -std=c++14 -stdlib=libc++" # Install most dependencies with homebrew, including python-3.9 From 501ec868f4eca6cb433da82be4e9043844efba8c Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Fri, 21 Feb 2025 17:30:00 -0800 Subject: [PATCH 02/10] Fix circular import. Build with 2 cores on github CI. --- .github/workflows/test.yml | 2 +- python/hk/scanner.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a35dd441..9244fdd9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -82,7 +82,7 @@ jobs: run: | source ~/conda/etc/profile.d/conda.sh \ && conda activate test \ - && python3 -m pip install . + && CPU_COUNT=2 python3 -m pip install -vv . - name: Run Tests run: | source ~/conda/etc/profile.d/conda.sh \ diff --git a/python/hk/scanner.py b/python/hk/scanner.py index f3d300b5..b7061c17 100644 --- a/python/hk/scanner.py +++ b/python/hk/scanner.py @@ -3,7 +3,7 @@ from spt3g import core from .. import _libso3g as libso3g -from . import hk +from .translator import HKTranslator class HKScanner: @@ -205,6 +205,6 @@ def __call__(self, f): p = core.G3Pipeline() p.Add(core.G3Reader(f)) if args.translate: - p.Add(hk.HKTranslator(target_version=args.target_version)) + p.Add(HKTranslator(target_version=args.target_version)) p.Add(HKScanner()) p.Run() From 06a2cb4c9a1a5cd242266764588b7562ba1b5eaf Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Fri, 21 Feb 2025 22:34:30 -0800 Subject: [PATCH 03/10] Fix imports and change unit tests to use G3Reader --- python/proj/__init__.py | 2 +- python/proj/mapthreads.py | 4 +++- python/proj/wcs.py | 17 ++++++++++------- test/test_g3super.py | 8 ++++---- test/test_indexed.py | 24 ++++++++++++------------ 5 files changed, 30 insertions(+), 25 deletions(-) diff --git a/python/proj/__init__.py b/python/proj/__init__.py index 5561ba53..f43301d7 100644 --- a/python/proj/__init__.py +++ b/python/proj/__init__.py @@ -2,7 +2,7 @@ from . import util from . import mapthreads -from .wcs import Projectionist, ProjectionistHealpix, Ranges, RangesMatrix +from .wcs import Projectionist, ProjectionistHealpix, RangesMatrix from .coords import CelestialSightLine, EarthlySite, Assembly, FocalPlane, SITES from .weather import Weather, weather_factory from .ranges import Ranges, RangesMatrix diff --git a/python/proj/mapthreads.py b/python/proj/mapthreads.py index c6570783..d7459246 100644 --- a/python/proj/mapthreads.py +++ b/python/proj/mapthreads.py @@ -9,10 +9,12 @@ from .. import _libso3g as libso3g -from . import quat, DEG +from . import quat from .coords import Assembly, FocalPlane from .wcs import Projectionist +DEG = np.pi/180. + def get_num_threads(n_threads=None): """Utility function for computing n_threads. If n_threads is not diff --git a/python/proj/wcs.py b/python/proj/wcs.py index ce7ded8f..c51c14c3 100644 --- a/python/proj/wcs.py +++ b/python/proj/wcs.py @@ -7,6 +7,9 @@ from .ranges import RangesMatrix from . import mapthreads +DEG = np.pi / 180. + + # For coordinate systems we use the following abbreviations: # # - DC: Detector coordinates @@ -536,14 +539,14 @@ def get_q(wcs): # This is typical for cylindrical projections. assert((delta0 >= 0 and wcs.wcs.lonpole == 180.0) or (delta0 <= 0 and wcs.wcs.lonpole == 0.0)) - Q = (quat.euler(1, delta0 * quat.DEG) * - quat.euler(2, -alpha0 * quat.DEG)) + Q = (quat.euler(1, delta0 * DEG) * + quat.euler(2, -alpha0 * DEG)) elif (wcs.wcs.phi0 == 0. and wcs.wcs.theta0 == 90.): # This is typical for zenithal projections. assert(wcs.wcs.lonpole == 180.0) Q = (quat.euler(2, np.pi) * - quat.euler(1, (delta0 - 90)*quat.DEG) * - quat.euler(2, -alpha0 * quat.DEG)) + quat.euler(1, (delta0 - 90)*DEG) * + quat.euler(2, -alpha0 * DEG)) else: raise ValueError('Unimplemented NSC reference (phi0,theta0)=' f'({wcs.wcs.phi0:.2f},{wcs.wcs.theta0:.2f})') @@ -597,7 +600,7 @@ def for_geom(cls, shape, wcs, interpol=None): self.q_celestial_to_native = self.get_q(wcs) # Store the grid info. - self.cdelt = np.array(wcs.wcs.cdelt) * quat.DEG + self.cdelt = np.array(wcs.wcs.cdelt) * DEG self.crpix = np.array(wcs.wcs.crpix) # Pixel interpolation mode @@ -636,8 +639,8 @@ def for_source_at(cls, alpha0, delta0, gamma0=0., assert(gamma0 == 0.) self.q_celestial_to_native = ( quat.euler(2, np.pi) - * quat.euler(1, (delta0 - 90)*quat.DEG) - * quat.euler(2, -alpha0 * quat.DEG)) + * quat.euler(1, (delta0 - 90)*DEG) + * quat.euler(2, -alpha0 * DEG)) return self @classmethod diff --git a/test/test_g3super.py b/test/test_g3super.py index e997c378..72a7a2d3 100644 --- a/test/test_g3super.py +++ b/test/test_g3super.py @@ -191,7 +191,7 @@ def test_11_fallback(self): ts = self._get_ts(1, nsamp, sigma=0, dtype='int32') ts.encode() self._readback_compare(ts) - + # Random time vector. n = 200 ts = self._get_ts(1, n, sigma=0, dtype='int32') @@ -241,7 +241,7 @@ def test_30_cpp_interface(self): self.assertTrue(np.all(ts.data[1:] == 0.)) self._readback_compare(ts) del ts - + def test_40_encoding_serialized(self): test_file = 'test_g3super.g3' offsets = { @@ -314,12 +314,12 @@ def test_50_compression(self): del w # Readback - r = so3g.G3IndexedReader(test_file) + r = core.G3Reader(test_file) last = 0 for dtype in ALL_DTYPES: for i in range(2): r.Process(None)[0] - here = r.Tell() + here = r.tell() sizes[dtype].append(here - last) last = here diff --git a/test/test_indexed.py b/test/test_indexed.py index 791653fa..eeb1a82f 100644 --- a/test/test_indexed.py +++ b/test/test_indexed.py @@ -67,7 +67,7 @@ def write_example_file(filename='hk_out.g3'): class TestG3IndexedReader(unittest.TestCase): - """TestCase for testing the so3g.G3IndexedReader, which has seek + """TestCase for testing the spt3g.core.G3Reader, which has seek capabilities to jump to known frames within a .g3 file. """ @@ -80,37 +80,37 @@ def tearDown(self): os.remove(self._file) def test_seek(self): - """Test the Seek/Tell functionality of the G3IndexedReader. We read the + """Test the seek/tell functionality of the G3Reader. We read the first four frames, recording the position of the only Wiring frame in the file - with Tell(). Then we Seek to that location and start reading again, expecting - the first frame after Seek() to be the wiring frame. + with tell(). Then we Seek to that location and start reading again, expecting + the first frame after seek() to be the wiring frame. """ - print("Testing Seek/Tell in G3IndexedReader") - r = so3g.G3IndexedReader(self._file) + print("Testing Seek/Tell in G3Reader") + r = core.G3Reader(self._file) # Limit the number of Process calls, if we hit the end of the file, # then Seek won't work... for i in range(4): - pos = r.Tell() + pos = r.tell() f = r.Process(None)[0] print(" " + str(f.type)) if f.type == core.G3FrameType.Wiring: w_pos = pos print(' Saved wiring frame position: {}'.format(w_pos)) - r.Seek(w_pos) + r.seek(w_pos) # Now that we've seeked, our next frame should be Wiring assert r.Process(None)[0].type == core.G3FrameType.Wiring # Confirm exception is raised if seek at eof. - r = so3g.G3IndexedReader(self._file) + r = core.G3Reader(self._file) while len(r.Process(None)): pass - pos = r.Tell() + pos = r.tell() # Ok to seek to EOF if at EOF. - r.Seek(pos) + r.seek(pos) # No back seeking once there, though. with self.assertRaises(RuntimeError): - r.Seek(0) + r.seek(0) From 7e89b62e364d91148db0b96ff8da52a5d98afb57 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Sat, 22 Feb 2025 00:26:47 -0800 Subject: [PATCH 04/10] Try to overcome the dreaded unregistered polymorphic type error on clang builds --- .github/workflows/test.yml | 4 ++-- include/G3SuperTimestream.h | 9 ++++++++- src/G3SuperTimestream.cxx | 10 +++++++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9244fdd9..86e5db83 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,11 +78,11 @@ jobs: && conda activate test \ && conda install --yes --file conda_dev_requirements.txt \ && pip install -r test-requirements.txt - - name: Install + - name: Install so3g run: | source ~/conda/etc/profile.d/conda.sh \ && conda activate test \ - && CPU_COUNT=2 python3 -m pip install -vv . + && CPU_COUNT=2 python3 -m pip install -v . - name: Run Tests run: | source ~/conda/etc/profile.d/conda.sh \ diff --git a/include/G3SuperTimestream.h b/include/G3SuperTimestream.h index 654c3083..26ec0768 100644 --- a/include/G3SuperTimestream.h +++ b/include/G3SuperTimestream.h @@ -106,7 +106,14 @@ namespace cereal { } G3_POINTERS(G3SuperTimestream); -G3_SERIALIZABLE(G3SuperTimestream, 0); + +CEREAL_CLASS_VERSION(G3SuperTimestream, 0); +CEREAL_REGISTER_TYPE_WITH_NAME(G3SuperTimestream, "G3SuperTimestream"); +CEREAL_REGISTER_POLYMORPHIC_RELATION(G3FrameObject, G3SuperTimestream); + +CEREAL_FORCE_DYNAMIC_INIT(libso3g); + +// G3_SERIALIZABLE(G3SuperTimestream, 0); class g3supertimestream_exception : std::exception { diff --git a/src/G3SuperTimestream.cxx b/src/G3SuperTimestream.cxx index f69228bd..661f77c7 100644 --- a/src/G3SuperTimestream.cxx +++ b/src/G3SuperTimestream.cxx @@ -1176,7 +1176,15 @@ G3SuperTimestreamPtr test_cxx_interface(int nsamps, int first, int second) } -G3_SPLIT_SERIALIZABLE_CODE(G3SuperTimestream); +template void G3SuperTimestream::save( + cereal::PortableBinaryOutputArchive &, unsigned +) const; + +template void G3SuperTimestream::load(cereal::PortableBinaryInputArchive &, unsigned); + +CEREAL_REGISTER_DYNAMIC_INIT(libso3g); + +// G3_SPLIT_SERIALIZABLE_CODE(G3SuperTimestream); static void translate_ValueError(g3supertimestream_exception const& e) { From 974a50cfcff0f242a3e2c5c59e69fcb638990d99 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Sun, 23 Feb 2025 20:00:00 -0800 Subject: [PATCH 05/10] Try to fix dynamic library paths when repairing wheels. --- .github/workflows/test.yml | 27 +++++++++++++++------------ wheels/repair_wheel_linux.sh | 2 +- wheels/repair_wheel_macos.sh | 2 +- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 86e5db83..9efc2b6f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,18 +38,21 @@ jobs: - os: ubuntu-latest python: "3.12" arch: Linux-x86_64 - - os: macos-latest - python: "3.10" - arch: MacOSX-x86_64 - - os: macos-latest - python: "3.12" - arch: MacOSX-x86_64 - - os: macos-latest - python: "3.10" - arch: MacOSX-arm64 - - os: macos-latest - python: "3.12" - arch: MacOSX-arm64 + + # MacOS compilation with the conda compilers (clang) will not + # work due to CEREAL polymorphic issues. + # - os: macos-latest + # python: "3.10" + # arch: MacOSX-x86_64 + # - os: macos-latest + # python: "3.12" + # arch: MacOSX-x86_64 + # - os: macos-latest + # python: "3.10" + # arch: MacOSX-arm64 + # - os: macos-latest + # python: "3.12" + # arch: MacOSX-arm64 steps: - name: Checkout uses: actions/checkout@v4 diff --git a/wheels/repair_wheel_linux.sh b/wheels/repair_wheel_linux.sh index 0b9d42f7..ac9ad891 100755 --- a/wheels/repair_wheel_linux.sh +++ b/wheels/repair_wheel_linux.sh @@ -16,7 +16,7 @@ scriptdir=$(pwd) popd >/dev/null 2>&1 # On Linux, we need to add this to LD_LIBRARY_PATH -spt3g_install=$(ls -d ${scriptdir}/../build/lib.*/so3g/spt3g_internal) +spt3g_install=$(ls -d ${scriptdir}/../build/temp.*/spt3g_install/lib) export LD_LIBRARY_PATH="/usr/local/lib":"${spt3g_install}":${LD_LIBRARY_PATH} auditwheel repair -w ${dest_dir} ${wheel} diff --git a/wheels/repair_wheel_macos.sh b/wheels/repair_wheel_macos.sh index a54191b7..e4662321 100755 --- a/wheels/repair_wheel_macos.sh +++ b/wheels/repair_wheel_macos.sh @@ -16,7 +16,7 @@ pushd $(dirname $0) >/dev/null 2>&1 scriptdir=$(pwd) popd >/dev/null 2>&1 -spt3g_install=$(ls -d ${scriptdir}/../build/temp.*/spt3g/spt3g) +spt3g_install=$(ls -d ${scriptdir}/../build/temp.*/spt3g_install/lib) export DYLD_LIBRARY_PATH="/usr/local/lib":"${spt3g_install}":${DYLD_LIBRARY_PATH} delocate-listdeps --all ${wheel} \ From b9a84d8e0b59ed662ceba80300fdda83b41bab2a Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Sun, 23 Feb 2025 23:51:25 -0800 Subject: [PATCH 06/10] More experimenting with dynamic library loading path during wheel repair. --- wheels/repair_wheel_linux.sh | 4 ++-- wheels/repair_wheel_macos.sh | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/wheels/repair_wheel_linux.sh b/wheels/repair_wheel_linux.sh index ac9ad891..f9571692 100755 --- a/wheels/repair_wheel_linux.sh +++ b/wheels/repair_wheel_linux.sh @@ -16,7 +16,7 @@ scriptdir=$(pwd) popd >/dev/null 2>&1 # On Linux, we need to add this to LD_LIBRARY_PATH -spt3g_install=$(ls -d ${scriptdir}/../build/temp.*/spt3g_install/lib) -export LD_LIBRARY_PATH="/usr/local/lib":"${spt3g_install}":${LD_LIBRARY_PATH} +spt3g_install=$(ls -d ${scriptdir}/../build/temp.*/spt3g_install) +export LD_LIBRARY_PATH="/usr/local/lib":"${spt3g_install}/lib":"${spt3g_install}/lib64":${LD_LIBRARY_PATH} auditwheel repair -w ${dest_dir} ${wheel} diff --git a/wheels/repair_wheel_macos.sh b/wheels/repair_wheel_macos.sh index e4662321..44f16d7f 100755 --- a/wheels/repair_wheel_macos.sh +++ b/wheels/repair_wheel_macos.sh @@ -16,8 +16,9 @@ pushd $(dirname $0) >/dev/null 2>&1 scriptdir=$(pwd) popd >/dev/null 2>&1 -spt3g_install=$(ls -d ${scriptdir}/../build/temp.*/spt3g_install/lib) -export DYLD_LIBRARY_PATH="/usr/local/lib":"${spt3g_install}":${DYLD_LIBRARY_PATH} +# spt3g_install=$(ls -d ${scriptdir}/../build/temp.*/spt3g_install/lib) +# export DYLD_LIBRARY_PATH="/usr/local/lib":"${spt3g_install}":${DYLD_LIBRARY_PATH} +export DYLD_LIBRARY_PATH="/usr/local/lib":${DYLD_LIBRARY_PATH} delocate-listdeps --all ${wheel} \ && delocate-wheel -v --require-archs ${delocate_archs} -w ${dest_dir} ${wheel} From 7d339b088d15dcdc551011bdf294bd8d9ce8ef6a Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Mon, 24 Feb 2025 12:47:27 -0800 Subject: [PATCH 07/10] Another attempt to fix repaired wheel linking --- wheels/repair_wheel_macos.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wheels/repair_wheel_macos.sh b/wheels/repair_wheel_macos.sh index 44f16d7f..ed12f7c4 100755 --- a/wheels/repair_wheel_macos.sh +++ b/wheels/repair_wheel_macos.sh @@ -18,7 +18,8 @@ popd >/dev/null 2>&1 # spt3g_install=$(ls -d ${scriptdir}/../build/temp.*/spt3g_install/lib) # export DYLD_LIBRARY_PATH="/usr/local/lib":"${spt3g_install}":${DYLD_LIBRARY_PATH} -export DYLD_LIBRARY_PATH="/usr/local/lib":${DYLD_LIBRARY_PATH} +spt3g_install=$(ls -d /project/build/temp.*/spt3g_install) +export DYLD_LIBRARY_PATH="/usr/local/lib":"${spt3g_install}/lib":"${spt3g_install}/lib64":${DYLD_LIBRARY_PATH} delocate-listdeps --all ${wheel} \ && delocate-wheel -v --require-archs ${delocate_archs} -w ${dest_dir} ${wheel} From 9cf35e5c163557e8ed745a1355075109d63d493e Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Mon, 24 Feb 2025 14:38:28 -0800 Subject: [PATCH 08/10] Another try at DYLD_LIBRARY_PATH to work around MacOS protections. --- wheels/repair_wheel_macos.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/wheels/repair_wheel_macos.sh b/wheels/repair_wheel_macos.sh index ed12f7c4..804e2eb0 100755 --- a/wheels/repair_wheel_macos.sh +++ b/wheels/repair_wheel_macos.sh @@ -19,8 +19,11 @@ popd >/dev/null 2>&1 # spt3g_install=$(ls -d ${scriptdir}/../build/temp.*/spt3g_install/lib) # export DYLD_LIBRARY_PATH="/usr/local/lib":"${spt3g_install}":${DYLD_LIBRARY_PATH} spt3g_install=$(ls -d /project/build/temp.*/spt3g_install) -export DYLD_LIBRARY_PATH="/usr/local/lib":"${spt3g_install}/lib":"${spt3g_install}/lib64":${DYLD_LIBRARY_PATH} +dylib_path="/usr/local/lib":"${spt3g_install}/lib":"${spt3g_install}/lib64":${DYLD_LIBRARY_PATH} -delocate-listdeps --all ${wheel} \ -&& delocate-wheel -v --require-archs ${delocate_archs} -w ${dest_dir} ${wheel} +# DYLD_LIBRARY_PATH must be set on the command invocation line. See: +# https://github.com/pypa/cibuildwheel/issues/816 +# +DYLD_LIBRARY_PATH=${dylib_path} delocate-listdeps --all ${wheel} && \ +DYLD_LIBRARY_PATH=${dylib_path} delocate-wheel -v --require-archs ${delocate_archs} -w ${dest_dir} ${wheel} From ca8a27104c5020af1fff6b7cb351ce70b15c2e28 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Mon, 24 Feb 2025 16:32:20 -0800 Subject: [PATCH 09/10] Use relative path --- wheels/repair_wheel_macos.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/wheels/repair_wheel_macos.sh b/wheels/repair_wheel_macos.sh index 804e2eb0..c17a2861 100755 --- a/wheels/repair_wheel_macos.sh +++ b/wheels/repair_wheel_macos.sh @@ -16,9 +16,7 @@ pushd $(dirname $0) >/dev/null 2>&1 scriptdir=$(pwd) popd >/dev/null 2>&1 -# spt3g_install=$(ls -d ${scriptdir}/../build/temp.*/spt3g_install/lib) -# export DYLD_LIBRARY_PATH="/usr/local/lib":"${spt3g_install}":${DYLD_LIBRARY_PATH} -spt3g_install=$(ls -d /project/build/temp.*/spt3g_install) +spt3g_install=$(ls -d ${scriptdir}/../build/temp.*/spt3g_install) dylib_path="/usr/local/lib":"${spt3g_install}/lib":"${spt3g_install}/lib64":${DYLD_LIBRARY_PATH} # DYLD_LIBRARY_PATH must be set on the command invocation line. See: From 877ea287c4da882a0e1a2a23f87d778f4ddb5b05 Mon Sep 17 00:00:00 2001 From: Theodore Kisner Date: Mon, 24 Feb 2025 21:04:42 -0800 Subject: [PATCH 10/10] Update setup.py --- setup.py | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/setup.py b/setup.py index 6c18271c..db60a147 100644 --- a/setup.py +++ b/setup.py @@ -188,7 +188,7 @@ def run(self): Perform build_cmake before doing the 'normal' stuff """ for extension in self.extensions: - if extension.name == "so3g.libso3g": + if extension.name == "so3g._libso3g": # We just trigger this on one of the extensions. build_cmake() # will actually build everything. self.build_cmake() @@ -216,13 +216,9 @@ def build_cmake(self): # CMake build directory for spt3g temp_spt3g = os.path.join(temp_build, "spt3g") - # The python module in the spt3g build directory. This contains - # the compiled libraries and symlinks to the python source. - spt3g_python_dir = os.path.join(temp_spt3g, "spt3g") - # Use CMake to install to the distutils build location install_so3g = os.path.dirname( - Path(self.get_ext_fullpath("so3g.libso3g")).resolve().parents[0] + Path(self.get_ext_fullpath("so3g._libso3g")).resolve().parents[0] ) # Fake install directory passed to spt3g cmake. @@ -244,9 +240,6 @@ def build_cmake(self): dlist3g = [ f"-DPython_EXECUTABLE={py_exe}", f"-DPython_INCLUDE_DIRS={py_incl}", - "-DPython_LIBRARIES=''", - "-DPython_RUNTIME_LIBRARY_DIRS=''", - "-DPython_LIBRARY_DIRS=''", f"-DPython_VERSION_MAJOR={py_maj}", f"-DPython_VERSION_MINOR={py_min}", "-DBoost_ARCHITECTURE=-x64", @@ -277,6 +270,13 @@ def build_cmake(self): self.debug, ) + # Move spt3g python directory into place. Remove any stale copy of the + # directory. + install_spt3g_internal = os.path.join(install_so3g, "so3g", "spt3g_internal") + if os.path.isdir(install_spt3g_internal): + shutil.rmtree(install_spt3g_internal) + os.rename(os.path.join(temp_spt3g, "spt3g"), install_spt3g_internal) + build_so3g( topdir, temp_so3g, @@ -288,25 +288,11 @@ def build_cmake(self): self.debug, ) - # Move spt3g python directory into place. Remove any stale copy of the - # directory. - install_spt3g_internal = os.path.join(install_so3g, "so3g", "spt3g_internal") - if os.path.isdir(install_spt3g_internal): - print(f"rm stale: {install_spt3g_internal}") - shutil.rmtree(install_spt3g_internal) - print(f"copy {spt3g_python_dir}, {install_spt3g_internal}") - shutil.copytree(spt3g_python_dir, install_spt3g_internal, symlinks=False) - self.cmake_build_done = True ext_modules = [ - CMakeExtension("so3g.libso3g"), - CMakeExtension("so3g.spt3g_internal.libspt3g-core"), - CMakeExtension("so3g.spt3g_internal.libspt3g-dfmux"), - CMakeExtension("so3g.spt3g_internal.libspt3g-calibration"), - CMakeExtension("so3g.spt3g_internal.libspt3g-gcp"), - CMakeExtension("so3g.spt3g_internal.libspt3g-maps"), + CMakeExtension("so3g._libso3g"), ] # Install the python scripts from spt3g