From d9fd109f22790287d08926db46939111651b0dcb Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 13 Mar 2026 10:35:44 +0100 Subject: [PATCH 1/7] remove allow_failure for python 3.14 --- .github/workflows/python-package-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index da00acbf..4e3b87b2 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -18,7 +18,7 @@ jobs: dl1dh-version: ['latest', 'nightly'] max-parallel: 6 runs-on: ${{ matrix.os }} - continue-on-error: ${{ matrix.dl1dh-version == 'nightly' || matrix.python-version == '3.14' }} + continue-on-error: ${{ matrix.dl1dh-version == 'nightly'}} steps: - uses: actions/checkout@v4 From 9b212094189f18bdc0b6c975d092dd82363d7271 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 27 Mar 2026 17:52:59 +0100 Subject: [PATCH 2/7] bug fix config instrument storage before tool wrote incompatible subarray descriptions which resulted in errors when merging files --- ctlearn/tools/predict_model.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/ctlearn/tools/predict_model.py b/ctlearn/tools/predict_model.py index dba16638..f14151ef 100644 --- a/ctlearn/tools/predict_model.py +++ b/ctlearn/tools/predict_model.py @@ -90,9 +90,6 @@ from ctlearn.utils import validate_trait_dict # Convienient constants for column names and table keys -CONFIG_INSTRUMENT_SUBARRAY_LAYOUT = "/configuration/instrument/subarray/layout" -CONFIG_INSTRUMENT_TEL = "/configuration/instrument/telescope" -CONFIG_INSTRUMENT_TEL_CAMERA = "/configuration/instrument/telescope/camera" SUBARRAY_EVENT_KEYS = ["obs_id", "event_id"] TEL_EVENT_KEYS = ["obs_id", "event_id", "tel_id"] TEL_ITER_GROUPS = [ @@ -519,7 +516,13 @@ def _ensure_subarray_consistency(self): if input_subarray == self.dl1dh_reader.subarray: return - self.dl1dh_reader.subarray.to_hdf(self.output_path, overwrite=True) + # From the merger tool a SubarrayDescription for the full array is already stored + # in the output file. We need to remove it to avoid conflicts when storing + # the new SubarrayDescription for the selected telescopes. + with tables.open_file(self.output_path, mode="a") as h5file: + h5file.remove_node("/configuration/instrument", recursive=True) + selected_subarray = input_subarray.select_subarray(set(self.dl1dh_reader.tel_ids)) + selected_subarray.to_hdf(self.output_path) self.log.info("SubarrayDescription was stored in '%s'", self.output_path) tel_trigger_table = read_table( @@ -611,19 +614,6 @@ def prune_group(group, valid_ids): if group is not None: prune_group(group, tel_ids) - # Camera configuration tables - layout_node = getattr(h5_file.root, CONFIG_INSTRUMENT_SUBARRAY_LAYOUT, None) - camera_group = getattr(h5_file.root, CONFIG_INSTRUMENT_TEL_CAMERA, None) - if not (layout_node and camera_group): - return - # layout can be either a Table or a Group containing a Table - layout_table = ( - layout_node - if isinstance(layout_node, tables.Table) - else next(layout_node._f_iter_nodes("Table")) - ) - camera_indices = set(layout_table.col("camera_index")) - prune_group(camera_group, camera_indices) def _create_nan_table(self, nonexample_identifiers, columns, shapes, reco_task): """ From fd6858786f6a21bfb9002e13c77042e14941dd74 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Fri, 27 Mar 2026 17:55:11 +0100 Subject: [PATCH 3/7] bug fix for older data format without event_type in subarray trigger table --- ctlearn/tools/predict_model.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ctlearn/tools/predict_model.py b/ctlearn/tools/predict_model.py index f14151ef..acfa39be 100644 --- a/ctlearn/tools/predict_model.py +++ b/ctlearn/tools/predict_model.py @@ -541,9 +541,12 @@ def _ensure_subarray_consistency(self): ) subarray_trigger_table = tel_trigger_table.copy() - subarray_trigger_table.keep_columns( - SUBARRAY_EVENT_KEYS + ["time", "event_type"] - ) + subarray_columns = SUBARRAY_EVENT_KEYS + ["time"] + # In older data formats the event type is not included in the trigger table, so we need to + # check if it is present before keeping the column to be backwards compatible. + if "event_type" in subarray_trigger_table.colnames: + subarray_columns.append("event_type") + subarray_trigger_table.keep_columns(subarray_columns) subarray_trigger_table = unique( subarray_trigger_table, keys=SUBARRAY_EVENT_KEYS ) From 7fdaddf193a873a1bce6a2f2aa0ab9c6b51f7ee6 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 7 Apr 2026 10:08:08 +0200 Subject: [PATCH 4/7] added TensorBoard to pyproject since TF 2.21 removed it from their deps --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index bab81d9f..f2a5c057 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "scikit-learn", "numba", "tensorflow>=2.16", + "tensorboard", "pydot", "setuptools", "ctapipe[all]>=0.29", From a845dcd40b3b55c2691701f85a18248f1f43e039 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 7 Apr 2026 10:23:23 +0200 Subject: [PATCH 5/7] added TF lowest and latest version to CI matrix --- .github/workflows/python-package-conda.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 4e3b87b2..364f7c0a 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -16,6 +16,7 @@ jobs: os: [ubuntu-22.04] python-version: ['3.12', '3.13', '3.14'] dl1dh-version: ['latest', 'nightly'] + tensorflow-version: ['latest', '2.16.*'] max-parallel: 6 runs-on: ${{ matrix.os }} continue-on-error: ${{ matrix.dl1dh-version == 'nightly'}} @@ -50,6 +51,11 @@ jobs: else pip install dl1-data-handler fi + if [ "${{ matrix.tensorflow-version }}" = "latest" ]; then + pip install --upgrade tensorflow + else + pip install "tensorflow==${{ matrix.tensorflow-version }}" + fi - name: Add MKL_THREADING_LAYER variable run: echo "MKL_THREADING_LAYER=GNU" >> $GITHUB_ENV From 0d40d531aa3cb79afb3ba40fedab5c8ca2528ad0 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Tue, 7 Apr 2026 10:29:58 +0200 Subject: [PATCH 6/7] TF 2.16 only avail for python 3.12 --- .github/workflows/python-package-conda.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 364f7c0a..c5163881 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -17,6 +17,11 @@ jobs: python-version: ['3.12', '3.13', '3.14'] dl1dh-version: ['latest', 'nightly'] tensorflow-version: ['latest', '2.16.*'] + exclude: + - python-version: '3.13' + tensorflow-version: '2.16.*' + - python-version: '3.14' + tensorflow-version: '2.16.*' max-parallel: 6 runs-on: ${{ matrix.os }} continue-on-error: ${{ matrix.dl1dh-version == 'nightly'}} From fb0589b4be45dbce5fed357eb461008b644d04d7 Mon Sep 17 00:00:00 2001 From: TjarkMiener Date: Wed, 1 Jul 2026 13:28:53 +0000 Subject: [PATCH 7/7] move back to allowing to fail python 3.14 --- .github/workflows/python-package-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index c5163881..bea20d02 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -24,7 +24,7 @@ jobs: tensorflow-version: '2.16.*' max-parallel: 6 runs-on: ${{ matrix.os }} - continue-on-error: ${{ matrix.dl1dh-version == 'nightly'}} + continue-on-error: ${{ matrix.dl1dh-version == 'nightly' || matrix.python-version == '3.14' }} steps: - uses: actions/checkout@v4