From ee6d17c474c1eb01dadfe316ea41c047df5c6ddc Mon Sep 17 00:00:00 2001 From: Nosheen Adil Date: Sat, 31 May 2025 09:27:39 -0700 Subject: [PATCH 01/10] IT-1579: fix unit tests in IDPS build --- isx/test/test_algos.py | 4 ++-- isx/test/test_file_io.py | 6 +++--- isx/test/test_file_metadata.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/isx/test/test_algos.py b/isx/test/test_algos.py index 347f7cb..c4dfbc5 100644 --- a/isx/test/test_algos.py +++ b/isx/test/test_algos.py @@ -1,4 +1,4 @@ -from test.utilities.setup import delete_files_silently, delete_dirs_silently, test_data_path, is_file +from .utilities.setup import delete_files_silently, delete_dirs_silently, test_data_path, is_file import os import csv @@ -9,7 +9,7 @@ import isx -from test.asserts import assert_csv_cell_metrics_are_close_by_path, assert_isxd_cellsets_are_close_by_path, \ +from .asserts import assert_csv_cell_metrics_are_close_by_path, assert_isxd_cellsets_are_close_by_path, \ assert_isxd_movies_are_close, assert_isxd_movies_are_close_by_path, assert_isxd_event_sets_are_close_by_path, \ assert_csv_files_are_equal_by_path, assert_csv_files_are_close_by_path, assert_txt_files_are_equal_by_path, \ assert_csv_pairwise_spatial_overlap_matrices_are_close_by_path, \ diff --git a/isx/test/test_file_io.py b/isx/test/test_file_io.py index 4df189e..daf707e 100644 --- a/isx/test/test_file_io.py +++ b/isx/test/test_file_io.py @@ -1,4 +1,4 @@ -from test.utilities.setup import delete_files_silently, delete_dirs_silently, test_data_path, is_file +from .utilities.setup import delete_files_silently, delete_dirs_silently, test_data_path, is_file import operator, shutil, os, platform import h5py @@ -9,8 +9,8 @@ import isx -from test.utilities.create_sample_data import write_sample_cellset, write_sample_vessel_diameter_set, write_sample_rbc_velocity_set -from test.asserts import assert_json_files_equal_by_path, assert_tiff_files_equal_by_path, \ +from .utilities.create_sample_data import write_sample_cellset, write_sample_vessel_diameter_set, write_sample_rbc_velocity_set +from .asserts import assert_json_files_equal_by_path, assert_tiff_files_equal_by_path, \ assert_csv_traces_are_close_by_path, compare_h5_groups, assert_csv_events_are_equal_by_path, \ assert_csv_files_are_equal_by_path, assert_isxd_images_are_close_by_path_nan_zero diff --git a/isx/test/test_file_metadata.py b/isx/test/test_file_metadata.py index 2e85600..c2901b8 100644 --- a/isx/test/test_file_metadata.py +++ b/isx/test/test_file_metadata.py @@ -1,6 +1,6 @@ import os -from test.utilities.setup import delete_files_silently, test_data_path +from .utilities.setup import delete_files_silently, test_data_path import pytest From 10c52d0292cbce9dcf29ccf0fd3f398dd8e4b5f5 Mon Sep 17 00:00:00 2001 From: Nosheen Adil Date: Tue, 3 Jun 2025 07:49:32 -0700 Subject: [PATCH 02/10] IT-1579: updates to api from IDPS --- isx/_internal.py | 15 ++++++++- isx/algo.py | 19 ++++++++--- isx/test/asserts.py | 9 +++++ isx/test/test_algos.py | 60 ++++++++++++++++++++++++++++++++-- isx/test/test_file_io.py | 8 ++--- isx/test/test_file_metadata.py | 2 -- 6 files changed, 99 insertions(+), 14 deletions(-) diff --git a/isx/_internal.py b/isx/_internal.py index a5a9811..d19cc3a 100644 --- a/isx/_internal.py +++ b/isx/_internal.py @@ -1064,6 +1064,17 @@ def get_core_version(): ctypes.c_double] c_api.isx_apply_cell_set.errcheck = _standard_errcheck + c_api.isx_apply_rois.argtypes = [ + ctypes.c_int, + CharPtrPtr, + CharPtrPtr, + ctypes.c_int, + Int64Ptr, + Int64Ptr, + ctypes.c_bool, + CharPtrPtr] + c_api.isx_apply_rois.errcheck = _standard_errcheck + c_api.isx_export_cell_contours.argtypes = [ ctypes.c_int, CharPtrPtr, @@ -1227,7 +1238,9 @@ def get_core_version(): ctypes.c_char_p, ctypes.c_bool, ctypes.c_double, - ctypes.c_size_t] + ctypes.c_size_t, + ctypes.c_bool, + CharPtrPtr] c_api.isx_estimate_vessel_diameter.errcheck = _standard_errcheck c_api.isx_estimate_rbc_velocity.argtypes = [ diff --git a/isx/algo.py b/isx/algo.py index fb8c68a..73d73b5 100644 --- a/isx/algo.py +++ b/isx/algo.py @@ -1157,7 +1157,8 @@ def estimate_vessel_diameter( height_estimate_rule = "independent", auto_accept_reject = True, rejection_threshold_fraction = 0.2, - rejection_threshold_count = 5): + rejection_threshold_count = 5, + vessel_names = []): """ Estimates blood vessel diameter along each input line over time @@ -1176,7 +1177,7 @@ def estimate_vessel_diameter( time_increment : float This specifies the time shift in seconds between consecutive measurements. When the time increment is smaller than the time window, consecutive windows will overlap. - The time increment must be greater than or equal to the time window. + The time increment must be less than or equal to the time window output_units : string in {'pixels', 'microns'} Output units for vessel diameter estimation. estimation_method : string in {'Non-Parametric FWHM', 'Parametric FWHM'} @@ -1200,6 +1201,9 @@ def estimate_vessel_diameter( rejection_threshold_count: int Parameter for auto accept/reject functionality. The number of threshold crossings allowed in a particular vessel diameter trace. + vessel_names: list + List of names to assign vessel in the output vessel sets. + If empty, then vessel will have default names. """ # File checks @@ -1234,6 +1238,11 @@ def estimate_vessel_diameter( if not output_units in output_units_map.keys(): raise ValueError('Invalid units. Valid units includes: {}'.format(*output_units_map.keys())) + + vessel_names_arr = isx._internal.list_to_ctypes_array(vessel_names, ctypes.c_char_p) + use_vessel_names = len(vessel_names) > 0 + if vessel_names and num_lines != len(vessel_names): + raise ValueError("Number of lines must equal number of vessel names.") isx._internal.c_api.isx_estimate_vessel_diameter( num_files, @@ -1248,7 +1257,9 @@ def estimate_vessel_diameter( height_estimate_rule.encode('utf-8'), auto_accept_reject, rejection_threshold_fraction, - rejection_threshold_count + rejection_threshold_count, + use_vessel_names, + vessel_names_arr ) return @@ -1279,7 +1290,7 @@ def estimate_rbc_velocity( time_increment : float This specifies the time shift in seconds between consecutive measurements. When the time increment is smaller than the time window, consecutive windows will overlap. - The time increment must be greater than or equal to the time window. + The time increment must be less than or equal to the time window. output_units : string in {'pixels', 'microns'} Output units for vessel velocity estimation. save_correlation_heatmaps: bool diff --git a/isx/test/asserts.py b/isx/test/asserts.py index 4d1164e..0218948 100644 --- a/isx/test/asserts.py +++ b/isx/test/asserts.py @@ -164,6 +164,8 @@ def assert_isxd_vesselsets_are_close_by_path(exp_vesselset_path, act_vesselset_p if assert_status: assert exp_vesselset.get_vessel_status(c) == act_vesselset.get_vessel_status(c) + assert exp_vesselset.get_vessel_name(c) == act_vesselset.get_vessel_name(c) + np.testing.assert_allclose(exp_vesselset.get_vessel_image_data(c), act_vesselset.get_vessel_image_data(c), rtol=relative_tolerance) np.testing.assert_allclose(exp_vesselset.get_vessel_trace_data(c), act_vesselset.get_vessel_trace_data(c), rtol=relative_tolerance) @@ -177,6 +179,13 @@ def assert_isxd_vesselsets_are_close_by_path(exp_vesselset_path, act_vesselset_p np.testing.assert_allclose(exp_vesselset.get_vessel_correlations_data(c, t), act_vesselset.get_vessel_correlations_data(c, t), rtol=relative_tolerance) +def assert_isxd_vesselsets_vessel_names(output_vessel_set_files, vessel_names): + vessel_sets = [isx.VesselSet.read(f) for f in output_vessel_set_files] + num_vessels = vessel_sets[0].num_vessels + for i in range(num_vessels): + for vessel_set in vessel_sets: + assert vessel_set.get_vessel_name(i) == vessel_names[i] + def assert_isxd_cellsets_are_close_range_by_path(exp_cellset_path, act_cellset_path, x_range, y_range, relative_tolerance=1e-05, absolute_tolerance=0): diff --git a/isx/test/test_algos.py b/isx/test/test_algos.py index c4dfbc5..6b2799a 100644 --- a/isx/test/test_algos.py +++ b/isx/test/test_algos.py @@ -1,7 +1,6 @@ from .utilities.setup import delete_files_silently, delete_dirs_silently, test_data_path, is_file import os -import csv import numpy as np import pandas as pd import pytest @@ -11,12 +10,13 @@ from .asserts import assert_csv_cell_metrics_are_close_by_path, assert_isxd_cellsets_are_close_by_path, \ assert_isxd_movies_are_close, assert_isxd_movies_are_close_by_path, assert_isxd_event_sets_are_close_by_path, \ - assert_csv_files_are_equal_by_path, assert_csv_files_are_close_by_path, assert_txt_files_are_equal_by_path, \ + assert_csv_files_are_equal_by_path, assert_csv_files_are_close_by_path, \ assert_csv_pairwise_spatial_overlap_matrices_are_close_by_path, \ assert_isxd_vesselsets_are_close_by_path, assert_csv_traces_are_close_by_path, \ assert_json_files_equal_by_path, assert_tiff_files_equal_by_path, \ assert_isxd_cellsets_trace_sums, \ - assert_isxd_cellsets_cell_names + assert_isxd_cellsets_cell_names, \ + assert_isxd_vesselsets_vessel_names @pytest.mark.skipif(not isx._is_with_algos, reason="Only for algo tests") class TestAlgorithms: @@ -1865,6 +1865,60 @@ def test_estimate_vessel_diameter_microns_probe_custom(self): assert not is_file(vs_out_file) + def test_estimate_vessel_diameter_vessel_names(self): + input_movie_files = [ + test_data_path + "/unit_test/bloodflow/bloodflow_movie_1.isxd", + test_data_path + "/unit_test/bloodflow/bloodflow_movie_2.isxd" + ] + vs_out_files = [ + test_data_path + "/unit_test/output/bloodflow_movie_1_vesselset.isxd", + test_data_path + "/unit_test/output/bloodflow_movie_2_vesselset.isxd" + ] + delete_files_silently(vs_out_files) + + test_contours = [ + [[96, 95], [222, 182]], + [[348, 301], [406, 311]], + [[439, 302], [482, 357]], + [[110, 355], [128, 409]] + ] + + vessel_names = [ + "v0", + "v1", + "v2", + "v3", + ] + + try: + isx.estimate_vessel_diameter( + input_movie_files, + vs_out_files, + test_contours, + time_window=1.5, + time_increment=0.5, + output_units="microns", + estimation_method="Non-Parametric FWHM", + auto_accept_reject=True, + rejection_threshold_fraction=0.2, + rejection_threshold_count=5, + vessel_names=vessel_names + ) + except Exception as error: + # Skip test if blood flow features are disabled in this version + if "Blood flow algorithms are not available in this version of the software. Please contact support in order to enable these features." in str(error): + return + else: + raise error + + # verify vessel names in output vessel set match input vessel names + assert_isxd_vesselsets_vessel_names( + vs_out_files, + vessel_names + ) + + delete_files_silently(vs_out_files) + def test_estimate_rbc_velocity(self): input_movie_files = [ test_data_path + "/unit_test/bloodflow/rbcv_movie_1-BP.isxd", diff --git a/isx/test/test_file_io.py b/isx/test/test_file_io.py index daf707e..28174b1 100644 --- a/isx/test/test_file_io.py +++ b/isx/test/test_file_io.py @@ -1,5 +1,5 @@ -from .utilities.setup import delete_files_silently, delete_dirs_silently, test_data_path, is_file -import operator, shutil, os, platform +from .utilities.setup import delete_files_silently, test_data_path, is_file +import shutil, os, platform import h5py import numpy as np @@ -10,8 +10,8 @@ import isx from .utilities.create_sample_data import write_sample_cellset, write_sample_vessel_diameter_set, write_sample_rbc_velocity_set -from .asserts import assert_json_files_equal_by_path, assert_tiff_files_equal_by_path, \ - assert_csv_traces_are_close_by_path, compare_h5_groups, assert_csv_events_are_equal_by_path, \ +from .asserts import assert_tiff_files_equal_by_path, \ + compare_h5_groups, assert_csv_events_are_equal_by_path, \ assert_csv_files_are_equal_by_path, assert_isxd_images_are_close_by_path_nan_zero data_types = ('float16', 'float32', 'float64', 'uint8', 'uint16', 'uint32', 'uint64', 'int16', 'int32', 'int64') diff --git a/isx/test/test_file_metadata.py b/isx/test/test_file_metadata.py index c2901b8..a390360 100644 --- a/isx/test/test_file_metadata.py +++ b/isx/test/test_file_metadata.py @@ -1,5 +1,3 @@ -import os - from .utilities.setup import delete_files_silently, test_data_path import pytest From 417ddb628e36a5d6f884ba245983bd956da3f149 Mon Sep 17 00:00:00 2001 From: Nosheen Adil Date: Wed, 4 Jun 2025 07:07:15 -0700 Subject: [PATCH 03/10] handle special case for windows --- isx/_internal.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/isx/_internal.py b/isx/_internal.py index d19cc3a..cd1f0d6 100644 --- a/isx/_internal.py +++ b/isx/_internal.py @@ -18,6 +18,10 @@ _lib_dir = os.path.join(_this_dir, 'lib') _is_windows = os.name == 'nt' if _is_windows: + # Special case for windows + # If built apart of IDPS app, all dlls need to be in top-level app dir + if not os.path.exists(_lib_dir): + _lib_dir = os.path.join(_this_dir, '..') _cwd = os.getcwd() os.chdir(_lib_dir) _isx_lib_name = os.path.join(_lib_dir, 'isxpublicapi.dll') From 0b1f5e19b02f1fc24d563f3b93072380e5979525 Mon Sep 17 00:00:00 2001 From: Nosheen Adil Date: Tue, 24 Jun 2025 13:32:28 -0700 Subject: [PATCH 04/10] IT-1579: support python 3.13 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 293bc99..26dd811 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ name='isx', version='2.0.1', packages=find_namespace_packages(), - python_requires='>=3.9,<3.13', + python_requires='>=3.9,<3.14', install_requires=[ 'h5py>=2.8.0', 'numpy>=1.14', From dd5c374808902f0bfbb43e69c145b1c20765dc10 Mon Sep 17 00:00:00 2001 From: Nosheen Adil Date: Tue, 24 Jun 2025 13:39:02 -0700 Subject: [PATCH 05/10] fix python3.13 unit tests --- isx/test/asserts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/isx/test/asserts.py b/isx/test/asserts.py index 0218948..f8ce726 100644 --- a/isx/test/asserts.py +++ b/isx/test/asserts.py @@ -134,7 +134,7 @@ def assert_isxd_cellsets_trace_sums(output_cell_set_files, expected_trace_sums): trace_sum = 0 for cell_set in cell_sets: trace = cell_set.get_cell_trace_data(i) - trace_sum += np.sum(trace) + trace_sum += np.sum(trace, dtype=float) assert round(trace_sum) == expected_trace_sums[i] From 159350c2f352caddbd412adb065f21a47ec4ba49 Mon Sep 17 00:00:00 2001 From: Nosheen Adil Date: Mon, 30 Jun 2025 17:12:18 -0700 Subject: [PATCH 06/10] remove use of anaconda --- .gitignore | 3 ++ Makefile | 85 ++++++++++++++++++++++++++--------------- jenkins/pipeline.groovy | 8 ++-- 3 files changed, 61 insertions(+), 35 deletions(-) diff --git a/.gitignore b/.gitignore index 2195721..402679a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ # secrets .ideas-github-token apiTestResults.xml +pyisx/lib/ +pyisx/bin/ +pyisx/pyvenv.cfg # IDE *.sublime-workspace diff --git a/Makefile b/Makefile index 4e508c1..61a2aee 100644 --- a/Makefile +++ b/Makefile @@ -24,11 +24,32 @@ ifndef THIRD_PARTY_DIR THIRD_PARTY_DIR=third_party endif -# Extract python version -ifndef PYTHON_VERSION - PYTHON_VERSION=$(shell python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))') +# Virtual environment vars +ifndef VENV_NAME + VENV_NAME=pyisx +endif + +# Define cmake generator based on OS +VENV_ACTIVATE = source ${VENV_NAME}/bin/activate + +ifndef PYTHON + PYTHON=python endif +# Check if the directory exists using wildcard and conditional +ifeq ($(wildcard $(VENV_NAME)/.),) + # Directory does not exist + PYTHON_VERSION=$(shell ${PYTHON} -c 'import sys; print(".".join(map(str, sys.version_info[:2])))') +else + # Directory exists + PYTHON_VERSION=$(shell ${VENV_ACTIVATE} && ${PYTHON} -c 'import sys; print(".".join(map(str, sys.version_info[:2])))') +endif + +# # Extract python version +# ifndef PYTHON_VERSION +# PYTHON_VERSION=$(shell ${PYTHON} -c 'import sys; print(".".join(map(str, sys.version_info[:2])))') +# endif + # Detect OS ifeq ($(OS), Windows_NT) DETECTED_OS = windows @@ -48,6 +69,8 @@ else _MACOSX_DEPLOYMENT_TARGET=10.11 else ifeq ($(PYTHON_VERSION), 3.12) _MACOSX_DEPLOYMENT_TARGET=10.15 + else ifeq ($(PYTHON_VERSION), 3.13) + _MACOSX_DEPLOYMENT_TARGET=10.15 endif endif endif @@ -82,17 +105,10 @@ else ifeq ($(DETECTED_OS), mac) CMAKE_GENERATOR = Xcode endif -# Virtual environment vars -ifndef VENV_NAME - VENV_NAME=pyisx -endif - -# Define cmake generator based on OS -ifeq ($(DETECTED_OS), windows) - VENV_ACTIVATE = source '$(shell conda info --base)/Scripts/activate' -else - VENV_ACTIVATE = source $(shell conda info --base)/bin/activate -endif +# ifeq ($(DETECTED_OS), windows) +# else +# VENV_ACTIVATE = source ${VENV_NAME}/bin/activate +# endif ifndef BUILD_API BUILD_API=0 @@ -113,27 +129,34 @@ clean: @rm -rf build @rm -rf docs/build @rm -rf wheelhouse + @rm -rf ${VENV_NAME} setup: ./scripts/setup -v --src ${REMOTE_DIR} --dst ${REMOTE_LOCAL_DIR} --remote-copy -ifeq ($(DETECTED_OS), mac) -env: - CONDA_SUBDIR=osx-64 conda create -y -n $(VENV_NAME) python=$(PYTHON_VERSION) && \ - $(VENV_ACTIVATE) $(VENV_NAME) && \ - conda config --env --set subdir osx-64 && \ - python -m pip install build -else +# ifeq ($(DETECTED_OS), mac) +# env: +# CONDA_SUBDIR=osx-64 conda create -y -n $(VENV_NAME) python=$(PYTHON_VERSION) && \ +# $(VENV_ACTIVATE) $(VENV_NAME) && \ +# conda config --env --set subdir osx-64 && \ +# python -m pip install build +# else +# env: +# conda create -y -n $(VENV_NAME) python=$(PYTHON_VERSION) && \ +# $(VENV_ACTIVATE) $(VENV_NAME) && \ +# python -m pip install build +# endif + env: - conda create -y -n $(VENV_NAME) python=$(PYTHON_VERSION) && \ - $(VENV_ACTIVATE) $(VENV_NAME) && \ - python -m pip install build -endif + echo ${PYTHON_VERSION} + ${PYTHON} -m venv ${VENV_NAME} + $(VENV_ACTIVATE) && python -m pip install build ifeq ($(DETECTED_OS), mac) build: export MACOSX_DEPLOYMENT_TARGET=${_MACOSX_DEPLOYMENT_TARGET} endif build: check_os + echo ${PYTHON_VERSION} mkdir -p $(BUILD_PATH) && \ cd $(BUILD_PATH) && \ THIRD_PARTY_DIR=$(THIRD_PARTY_DIR) cmake $(CMAKE_OPTIONS) -G "$(CMAKE_GENERATOR)" ../../../ @@ -147,25 +170,25 @@ else ifeq ($(DETECTED_OS), mac) cd $(BUILD_PATH) && \ xcodebuild -alltargets -configuration $(BUILD_TYPE) -project isx.xcodeproj CODE_SIGN_IDENTITY="" endif + $(VENV_ACTIVATE) && \ cd $(BUILD_PATH_BIN) && \ - $(VENV_ACTIVATE) $(VENV_NAME) && \ python -m build rebuild: clean build test: - $(VENV_ACTIVATE) $(VENV_NAME) && \ + $(VENV_ACTIVATE) && \ pip install --force-reinstall '$(shell ls $(BUILD_PATH_BIN)/dist/isx-*.whl)[test]' && \ cd build/Release && \ ISX_TEST_DATA_PATH='$(shell realpath $(TEST_DATA_DIR))' python -m pytest --disable-warnings -v -s --junit-xml=$(API_TEST_RESULTS_PATH) test $(TEST_ARGS) ifeq ($(BUILD_API), 1) docs: build - $(VENV_ACTIVATE) $(VENV_NAME) && \ + $(VENV_ACTIVATE) && \ pip install --force-reinstall '$(shell ls $(BUILD_PATH_BIN)/dist/isx-*.whl)[docs]' endif docs: - $(VENV_ACTIVATE) $(VENV_NAME) && \ + $(VENV_ACTIVATE) && \ sphinx-build docs docs/build repair-linux: @@ -177,12 +200,12 @@ repair-linux: ifeq ($(DETECTED_OS), linux) deploy: repair-linux - $(VENV_ACTIVATE) $(VENV_NAME) && \ + $(VENV_ACTIVATE) && \ pip install twine && \ twine upload '$(shell ls wheelhouse/isx-*.whl)' else deploy: - $(VENV_ACTIVATE) $(VENV_NAME) && \ + $(VENV_ACTIVATE) && \ pip install twine && \ twine upload '$(shell ls $(BUILD_PATH_BIN)/dist/isx-*.whl)' endif \ No newline at end of file diff --git a/jenkins/pipeline.groovy b/jenkins/pipeline.groovy index 9cad3a1..372dbc0 100644 --- a/jenkins/pipeline.groovy +++ b/jenkins/pipeline.groovy @@ -15,9 +15,9 @@ def run_command(command, os) { def run(os, python_version, deploy = false) { run_command("make clean", os) - run_command("make env PYTHON_VERSION=${python_version}", os) - run_command("make build THIRD_PARTY_DIR=${IDPS_REMOTE_EXT_COPY_DIR} PYTHON_VERSION=${python_version}", os) - run_command("make test TEST_DATA_DIR=${IDPS_REMOTE_EXT_COPY_DIR}/test_data_structured PYTHON_VERSION=${python_version}", os) + run_command("make env PYTHON=python${python_version}", os) + run_command("make build THIRD_PARTY_DIR=${IDPS_REMOTE_EXT_COPY_DIR}", os) + run_command("make test TEST_DATA_DIR=${IDPS_REMOTE_EXT_COPY_DIR}/test_data_structured", os) if (deploy) { run_command("make deploy", os) @@ -36,7 +36,7 @@ def run_all(os, deploy = false) { run_command("make setup REMOTE_DIR=${IDPS_REMOTE_EXT_DIR} REMOTE_LOCAL_DIR=${IDPS_REMOTE_EXT_COPY_DIR}", os) } - python_versions = ["3.9", "3.10", "3.11", "3.12"] + python_versions = ["3.9", "3.10", "3.11", "3.12", "3.13"] python_versions.each() { stage("Python ${it}") { run(os, it, deploy) From ce77363b0bb6e958ce3c267da28e494ebad3d757 Mon Sep 17 00:00:00 2001 From: Nosheen Adil Date: Mon, 30 Jun 2025 22:52:10 -0700 Subject: [PATCH 07/10] IT-1579: fix cicd build --- .gitignore | 4 +-- Makefile | 76 ++++++++++++++++++++++++++++-------------------------- README.md | 40 +++++----------------------- setup.py | 8 ++++++ 4 files changed, 56 insertions(+), 72 deletions(-) diff --git a/.gitignore b/.gitignore index 402679a..44e381a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,7 @@ # secrets .ideas-github-token apiTestResults.xml -pyisx/lib/ -pyisx/bin/ -pyisx/pyvenv.cfg +venv/ # IDE *.sublime-workspace diff --git a/Makefile b/Makefile index 61a2aee..1b7a29f 100644 --- a/Makefile +++ b/Makefile @@ -26,11 +26,22 @@ endif # Virtual environment vars ifndef VENV_NAME - VENV_NAME=pyisx + VENV_NAME=venv endif -# Define cmake generator based on OS -VENV_ACTIVATE = source ${VENV_NAME}/bin/activate +# Detect OS +ifeq ($(OS), Windows_NT) + DETECTED_OS = windows + VENV_ACTIVATE = source ${VENV_NAME}/Scripts/activate +else + VENV_ACTIVATE = source ${VENV_NAME}/bin/activate + UNAME_S = $(shell uname -s) + ifeq ($(UNAME_S), Linux) + DETECTED_OS = linux + else ifeq ($(UNAME_S), Darwin) + DETECTED_OS = mac + endif +endif ifndef PYTHON PYTHON=python @@ -50,28 +61,18 @@ endif # PYTHON_VERSION=$(shell ${PYTHON} -c 'import sys; print(".".join(map(str, sys.version_info[:2])))') # endif -# Detect OS -ifeq ($(OS), Windows_NT) - DETECTED_OS = windows -else - UNAME_S = $(shell uname -s) - ifeq ($(UNAME_S), Linux) - DETECTED_OS = linux - else ifeq ($(UNAME_S), Darwin) - DETECTED_OS = mac - # Set the macOS deployment version based on python version - ifeq ($(PYTHON_VERSION), 3.9) - _MACOSX_DEPLOYMENT_TARGET=10.11 - else ifeq ($(PYTHON_VERSION), 3.10) - _MACOSX_DEPLOYMENT_TARGET=10.11 - else ifeq ($(PYTHON_VERSION), 3.11) - _MACOSX_DEPLOYMENT_TARGET=10.11 - else ifeq ($(PYTHON_VERSION), 3.12) - _MACOSX_DEPLOYMENT_TARGET=10.15 - else ifeq ($(PYTHON_VERSION), 3.13) - _MACOSX_DEPLOYMENT_TARGET=10.15 - endif +ifeq ($(DETECTED_OS), mac) + ifeq ($(PYTHON_VERSION), 3.9) + _MACOSX_DEPLOYMENT_TARGET=10.11 + else ifeq ($(PYTHON_VERSION), 3.10) + _MACOSX_DEPLOYMENT_TARGET=10.11 + else ifeq ($(PYTHON_VERSION), 3.11) + _MACOSX_DEPLOYMENT_TARGET=10.11 + else ifeq ($(PYTHON_VERSION), 3.12) + _MACOSX_DEPLOYMENT_TARGET=10.15 + else ifeq ($(PYTHON_VERSION), 3.13) + _MACOSX_DEPLOYMENT_TARGET=10.15 endif endif @@ -147,16 +148,19 @@ setup: # python -m pip install build # endif +ifeq ($(DETECTED_OS), mac) env: - echo ${PYTHON_VERSION} ${PYTHON} -m venv ${VENV_NAME} - $(VENV_ACTIVATE) && python -m pip install build + $(VENV_ACTIVATE) && python -m pip install '.[build,test,docs,deploy]' +else + sh -c "${PYTHON} -m venv ${VENV_NAME}" + $(VENV_ACTIVATE) && python -m pip install '.[build,test,docs,deploy]' +endif ifeq ($(DETECTED_OS), mac) build: export MACOSX_DEPLOYMENT_TARGET=${_MACOSX_DEPLOYMENT_TARGET} endif build: check_os - echo ${PYTHON_VERSION} mkdir -p $(BUILD_PATH) && \ cd $(BUILD_PATH) && \ THIRD_PARTY_DIR=$(THIRD_PARTY_DIR) cmake $(CMAKE_OPTIONS) -G "$(CMAKE_GENERATOR)" ../../../ @@ -175,22 +179,24 @@ endif python -m build rebuild: clean build - -test: + +install: + $(VENV_ACTIVATE) && \ + pip install --force-reinstall --no-deps '$(shell ls $(BUILD_PATH_BIN)/dist/isx-*.whl)' + +test: install $(VENV_ACTIVATE) && \ - pip install --force-reinstall '$(shell ls $(BUILD_PATH_BIN)/dist/isx-*.whl)[test]' && \ cd build/Release && \ ISX_TEST_DATA_PATH='$(shell realpath $(TEST_DATA_DIR))' python -m pytest --disable-warnings -v -s --junit-xml=$(API_TEST_RESULTS_PATH) test $(TEST_ARGS) ifeq ($(BUILD_API), 1) -docs: build - $(VENV_ACTIVATE) && \ - pip install --force-reinstall '$(shell ls $(BUILD_PATH_BIN)/dist/isx-*.whl)[docs]' +docs: install endif docs: $(VENV_ACTIVATE) && \ sphinx-build docs docs/build +# Used for fixing linux wheel installs before deploying to pypi repair-linux: docker run \ -v $(shell pwd):/io \ @@ -201,11 +207,9 @@ repair-linux: ifeq ($(DETECTED_OS), linux) deploy: repair-linux $(VENV_ACTIVATE) && \ - pip install twine && \ twine upload '$(shell ls wheelhouse/isx-*.whl)' else deploy: $(VENV_ACTIVATE) && \ - pip install twine && \ twine upload '$(shell ls $(BUILD_PATH_BIN)/dist/isx-*.whl)' -endif \ No newline at end of file +endif diff --git a/README.md b/README.md index f0cd90a..c685455 100644 --- a/README.md +++ b/README.md @@ -20,14 +20,8 @@ Pre-built binaries of this API can be installed from [PyPi](https://pypi.org/pro pip install isx ``` -> **Note:** For Apple Silicon (i.e., macOS arm64 architecture), the package is currently not natively supported. However, it's possible to use anaconda to configure an x86 environment and use the project. +> **Note**: Currently, pyisx is only supported for x86 architectures, which can be problematic, specifically on the newer Mac computers with Apple Silicon. For usage with Apple Silicon, the Rosetta software must be installed, and the Terminal app must be configured to use this software for automatic translation of x86 binaries to arm64. Read more [here](https://support.apple.com/en-us/102527) on how to configure Rosetta on Mac computers. -```bash -CONDA_SUBDIR=osx-64 conda create -n python= -conda activate -conda config --env --set subdir osx-64 -pip install isx -``` Replace `` with a name for the conda environment, and `` with the python version to use. @@ -60,43 +54,23 @@ Follow the setup instructions for the C++ [isxcore](https://github.com/inscopix/ 2. Setup python virtual environment -Create a python virtual environment, specifying the desired python version. -This guide uses anaconda for demonstration, but other tools like virtualenv or poetry can also be used. +Create a python virtual environment using venv, specifying the desired python version. ```bash -conda create -n python= -conda activate +make env PYTHON=python3.13 ``` -Replace `` with a name for the conda environment, and `` with the python version to use. - -> **Note**: On macOS systems with Apple Silicon, the conda environment is configured differently, since `isxcore` is currently only built for x86 architectures. - -```bash -CONDA_SUBDIR=osx-64 conda create -n python= -conda activate -conda config --env --set subdir osx-64 -``` - -Replace `` with a name for the conda environment, and `` with the python version to use. - -3. Install build & test dependencies - -Inside the virtual environment install the following dependencies: - -```bash -conda install -y build pytest -``` +> **Note**: Currently, pyisx is only supported for x86 architectures, which can be problematic, specifically on the newer Mac computers with Apple Silicon. For usage with Apple Silicon, the Rosetta software must be installed, and the Terminal app must be configured to use this software for automatic translation of x86 binaries to arm64. Read more [here](https://support.apple.com/en-us/102527) on how to configure Rosetta on Mac computers. -> **Note**: For python 3.12 the `build` package must be installed used `pip` instead. +3. Build the package -4. Build the package +Once the virtual environment is setup, the package can be built. ```bash make build THIRD_PARTY_DIR=/path/to/third/party/dir ``` -5. Run the unit tests +4. Run the unit tests ```bash make test THIRD_PARTY_DIR=/path/to/third/party/dir TEST_DATA_DIR=/path/to/test/data/dir diff --git a/setup.py b/setup.py index 26dd811..6892ad8 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,10 @@ 'openpyxl>=3.0.10', # Required for pandas Excel support ], extras_require={ + 'build': [ + # Optional dependencies for building wheel file with binaries + 'build' + ], 'test': [ # Optional dependencies for testing 'pytest', @@ -29,6 +33,10 @@ 'sphinx', 'sphinx_rtd_theme', 'myst_parser' + ], + 'deploy': [ + # Optional dependencies for deployment + 'twine' ] }, include_package_data=True, From 162e75166a3aec275d253671c80d04e624aa662f Mon Sep 17 00:00:00 2001 From: Nosheen Adil Date: Mon, 30 Jun 2025 23:23:21 -0700 Subject: [PATCH 08/10] IT-1579: fix windows build --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 1b7a29f..f6b88b9 100644 --- a/Makefile +++ b/Makefile @@ -148,12 +148,13 @@ setup: # python -m pip install build # endif -ifeq ($(DETECTED_OS), mac) +ifeq ($(DETECTED_OS), windows) env: - ${PYTHON} -m venv ${VENV_NAME} + sh -c "${PYTHON} -m venv ${VENV_NAME}" $(VENV_ACTIVATE) && python -m pip install '.[build,test,docs,deploy]' else - sh -c "${PYTHON} -m venv ${VENV_NAME}" +env: + ${PYTHON} -m venv ${VENV_NAME} $(VENV_ACTIVATE) && python -m pip install '.[build,test,docs,deploy]' endif From 97592038f91028ed9d04cea3dbb2d5c70de0d4b0 Mon Sep 17 00:00:00 2001 From: Nosheen Adil Date: Tue, 1 Jul 2025 09:48:31 -0700 Subject: [PATCH 09/10] IT-1579: doc updates and clean up --- Makefile | 30 ------------------------------ README.md | 3 --- docs/installation.md | 11 +---------- docs/overview.md | 9 +-------- setup.py | 2 +- 5 files changed, 3 insertions(+), 52 deletions(-) diff --git a/Makefile b/Makefile index f6b88b9..2dbecc1 100644 --- a/Makefile +++ b/Makefile @@ -56,11 +56,6 @@ else PYTHON_VERSION=$(shell ${VENV_ACTIVATE} && ${PYTHON} -c 'import sys; print(".".join(map(str, sys.version_info[:2])))') endif -# # Extract python version -# ifndef PYTHON_VERSION -# PYTHON_VERSION=$(shell ${PYTHON} -c 'import sys; print(".".join(map(str, sys.version_info[:2])))') -# endif - # Set the macOS deployment version based on python version ifeq ($(DETECTED_OS), mac) ifeq ($(PYTHON_VERSION), 3.9) @@ -106,15 +101,6 @@ else ifeq ($(DETECTED_OS), mac) CMAKE_GENERATOR = Xcode endif -# ifeq ($(DETECTED_OS), windows) -# else -# VENV_ACTIVATE = source ${VENV_NAME}/bin/activate -# endif - -ifndef BUILD_API - BUILD_API=0 -endif - check_os: @echo "Verifying detected OS" ifndef DETECTED_OS @@ -135,19 +121,6 @@ clean: setup: ./scripts/setup -v --src ${REMOTE_DIR} --dst ${REMOTE_LOCAL_DIR} --remote-copy -# ifeq ($(DETECTED_OS), mac) -# env: -# CONDA_SUBDIR=osx-64 conda create -y -n $(VENV_NAME) python=$(PYTHON_VERSION) && \ -# $(VENV_ACTIVATE) $(VENV_NAME) && \ -# conda config --env --set subdir osx-64 && \ -# python -m pip install build -# else -# env: -# conda create -y -n $(VENV_NAME) python=$(PYTHON_VERSION) && \ -# $(VENV_ACTIVATE) $(VENV_NAME) && \ -# python -m pip install build -# endif - ifeq ($(DETECTED_OS), windows) env: sh -c "${PYTHON} -m venv ${VENV_NAME}" @@ -190,9 +163,6 @@ test: install cd build/Release && \ ISX_TEST_DATA_PATH='$(shell realpath $(TEST_DATA_DIR))' python -m pytest --disable-warnings -v -s --junit-xml=$(API_TEST_RESULTS_PATH) test $(TEST_ARGS) -ifeq ($(BUILD_API), 1) -docs: install -endif docs: $(VENV_ACTIVATE) && \ sphinx-build docs docs/build diff --git a/README.md b/README.md index c685455..e5bafab 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,6 @@ pip install isx > **Note**: Currently, pyisx is only supported for x86 architectures, which can be problematic, specifically on the newer Mac computers with Apple Silicon. For usage with Apple Silicon, the Rosetta software must be installed, and the Terminal app must be configured to use this software for automatic translation of x86 binaries to arm64. Read more [here](https://support.apple.com/en-us/102527) on how to configure Rosetta on Mac computers. - -Replace `` with a name for the conda environment, and `` with the python version to use. - ## Supported Platforms This library has been built and tested on the following operating systems, for python versions `3.9 - 3.12`. diff --git a/docs/installation.md b/docs/installation.md index 7c0d7f7..78eb5b2 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -8,18 +8,9 @@ pip install isx ``` ::: {attention} -For Apple Silicon (i.e., macOS with arm64 architecture), the package is currently not natively supported. However, it's possible to use [anaconda](https://www.anaconda.com/) to configure an x86 environment and use the project. +Currently, pyisx is only supported for x86 architectures, which can be problematic, specifically on the newer Mac computers with Apple Silicon. For usage with Apple Silicon, the Rosetta software must be installed, and the Terminal app must be configured to use this software for automatic translation of x86 binaries to arm64. Read more [here](https://support.apple.com/en-us/102527) on how to configure Rosetta on Mac computers. ::: -```bash -CONDA_SUBDIR=osx-64 conda create -n python= -conda activate -conda config --env --set subdir osx-64 -pip install isx -``` - -Replace `` with a name for the conda environment, and `` with the python version to use. - ## Supported Platforms This package has been built and tested on the following operating systems, for python versions `3.9 - 3.12`: diff --git a/docs/overview.md b/docs/overview.md index d0d0c27..ed1dc27 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -20,16 +20,9 @@ pip install isx ``` ::: {attention} -For Apple Silicon (i.e., macOS with arm64 architecture), the package is currently not natively supported. However, it's possible to use [anaconda](https://www.anaconda.com/) to configure an x86 environment and use the project. +Currently, pyisx is only supported for x86 architectures, which can be problematic, specifically on the newer Mac computers with Apple Silicon. For usage with Apple Silicon, the Rosetta software must be installed, and the Terminal app must be configured to use this software for automatic translation of x86 binaries to arm64. Read more [here](https://support.apple.com/en-us/102527) on how to configure Rosetta on Mac computers. ::: -```bash -CONDA_SUBDIR=osx-64 conda create -n python= -conda activate -conda config --env --set subdir osx-64 -pip install isx -``` - Please refer to the [Installation](#installation) guide for more details. ## File Types diff --git a/setup.py b/setup.py index 6892ad8..46972ac 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='isx', - version='2.0.1', + version='2.1.0', packages=find_namespace_packages(), python_requires='>=3.9,<3.14', install_requires=[ From f5662383d9a017220cbeb18d562f733ba00c701a Mon Sep 17 00:00:00 2001 From: Nosheen Adil Date: Thu, 3 Jul 2025 15:57:22 -0700 Subject: [PATCH 10/10] IT-1579: improve docs --- README.md | 17 +++++++++++++++++ docs/installation.md | 34 ++++++++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e5bafab..7a8d842 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,23 @@ make env PYTHON=python3.13 > **Note**: Currently, pyisx is only supported for x86 architectures, which can be problematic, specifically on the newer Mac computers with Apple Silicon. For usage with Apple Silicon, the Rosetta software must be installed, and the Terminal app must be configured to use this software for automatic translation of x86 binaries to arm64. Read more [here](https://support.apple.com/en-us/102527) on how to configure Rosetta on Mac computers. +In the Rosetta terminal, install Homebrew. This will install Homebrew to the `/usr/local` directory, indicating an x86 installation +```bash +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +``` + +Next, in Rosetta terminal, install the desired python version using the x86 installation of brew: +``` +/usr/local/bin/brew install python@3.13 +``` + +To verify the installation worked correctly run the following command: +``` +python3.13 -c "import sysconfig; print(sysconfig.get_platform())" +``` + +The output will be `macosx-13.0-x86_64` is running x86 version of python, otherwise the output will instead be `macosx-13.0-arm64` if running the arm version of python. + 3. Build the package Once the virtual environment is setup, the package can be built. diff --git a/docs/installation.md b/docs/installation.md index 78eb5b2..c415564 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -7,13 +7,43 @@ Pre-built binaries of this API can be installed from [PyPi](https://pypi.org/pro pip install isx ``` +## Installation on Apple Silicon + ::: {attention} -Currently, pyisx is only supported for x86 architectures, which can be problematic, specifically on the newer Mac computers with Apple Silicon. For usage with Apple Silicon, the Rosetta software must be installed, and the Terminal app must be configured to use this software for automatic translation of x86 binaries to arm64. Read more [here](https://support.apple.com/en-us/102527) on how to configure Rosetta on Mac computers. +Currently, pyisx is only supported for x86 architectures, which can be problematic, specifically on the newer Mac computers with Apple Silicon. The following section describes one option for how to use this package on newer Mac computers. ::: +For usage with Apple Silicon, the Rosetta software must be installed, and the Terminal app must be configured to use this software for automatic translation of x86 binaries to arm64. Read more [here](https://support.apple.com/en-us/102527) on how to configure Rosetta on Mac computers. + +Next, install an x86 version of python with the following steps. + +First, in the Rosetta terminal, install Homebrew, which is a package manager for macOS. This will install Homebrew to the `/usr/local` directory, indicating an x86 installation: +```bash +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +``` + +Next, in the Rosetta terminal, install the desired python version using the x86 installation of brew: +```bash +/usr/local/bin/brew install python@3.13 +``` + +Verify the installation worked correctly by running the following command: +```bash +python3.13 -c "import sysconfig; print(sysconfig.get_platform())" +``` + +The output will be `macosx-13.0-x86_64` if running the x86 version of python, otherwise the output will instead be `macosx-13.0-arm64` if running the arm version of python. + +If the output indicates the x86 version of python, you can install the `pyisx` package. +For example, the following snippet demonstrates how to create a [virtual environment](https://docs.python.org/3/library/venv.html) using the x86 version of python, and then install the pyisx package within the virtual environment. +```bash +python3.13 -m venv venv +source venv/bin/activate && python -m pip install pyisx +``` + ## Supported Platforms -This package has been built and tested on the following operating systems, for python versions `3.9 - 3.12`: +This package has been built and tested on the following operating systems, for python versions `3.9 - 3.13`: | OS | Version | Architecture | | --------- | ------- | ----- |