From 1ac363efc956761c57a50c75fd670a778bf307bd Mon Sep 17 00:00:00 2001 From: Chris Meyer <34664+cmeyer@users.noreply.github.com> Date: Mon, 14 Apr 2025 14:05:23 -0700 Subject: [PATCH 1/3] Fix and test xdata.clone_with_data method. --- nion/data/DataAndMetadata.py | 6 +++++- nion/data/test/ExtendedData_test.py | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/nion/data/DataAndMetadata.py b/nion/data/DataAndMetadata.py index 73c855a..5c31c51 100644 --- a/nion/data/DataAndMetadata.py +++ b/nion/data/DataAndMetadata.py @@ -620,7 +620,11 @@ def clone_with_data(self, data: _ImageDataType) -> DataAndMetadata: data=data, intensity_calibration=self.intensity_calibration, dimensional_calibrations=self.dimensional_calibrations, - data_descriptor=self.data_descriptor) + metadata=self.metadata, + timestamp=self.timestamp, + data_descriptor=self.data_descriptor, + timezone=self.timezone, + timezone_offset=self.timezone_offset) @property def data_shape_and_dtype(self) -> typing.Optional[typing.Tuple[ShapeType, numpy.typing.DTypeLike]]: diff --git a/nion/data/test/ExtendedData_test.py b/nion/data/test/ExtendedData_test.py index 9f0994e..58caefe 100644 --- a/nion/data/test/ExtendedData_test.py +++ b/nion/data/test/ExtendedData_test.py @@ -1,4 +1,5 @@ # standard libraries +import datetime import h5py import logging import os @@ -143,6 +144,26 @@ def test_convert_to_array(self) -> None: data2[:] = xdata[:] self.assertTrue(numpy.array_equal(data2, xdata.data)) + def test_clone_with_data(self) -> None: + xdata = DataAndMetadata.new_data_and_metadata( + data=numpy.ones((10, 11, 12)), + intensity_calibration=Calibration.Calibration(0.1, 0.2, "I"), + dimensional_calibrations=[Calibration.Calibration(0.11, 0.22, "S"), Calibration.Calibration(0.11, 0.22, "A"), Calibration.Calibration(0.111, 0.222, "B")], + data_descriptor=DataAndMetadata.DataDescriptor(True, 0, 2), + metadata={"test": "test"}, + timestamp=datetime.datetime(2013, 11, 18, 14, 5, 4, 0), + timezone="America/Los_Angeles", + timezone_offset="-0700" + ) + xdata_clone = xdata.clone_with_data(numpy.ones((12, 11, 10))) + self.assertEqual(Calibration.Calibration(0.1, 0.2, "I"), xdata_clone.intensity_calibration) + self.assertEqual([Calibration.Calibration(0.11, 0.22, "S"), Calibration.Calibration(0.11, 0.22, "A"), Calibration.Calibration(0.111, 0.222, "B")], xdata_clone.dimensional_calibrations) + self.assertEqual(DataAndMetadata.DataDescriptor(True, 0, 2), xdata_clone.data_descriptor) + self.assertEqual({"test": "test"}, xdata_clone.metadata) + self.assertEqual(datetime.datetime(2013, 11, 18, 14, 5, 4, 0), xdata_clone.timestamp) + self.assertEqual("America/Los_Angeles", xdata_clone.timezone) + self.assertEqual("-0700", xdata_clone.timezone_offset) + if __name__ == '__main__': logging.getLogger().setLevel(logging.DEBUG) From cf4015620f45d45f39cf3de076da47dd85954012 Mon Sep 17 00:00:00 2001 From: Chris Meyer <34664+cmeyer@users.noreply.github.com> Date: Mon, 14 Apr 2025 14:12:51 -0700 Subject: [PATCH 2/3] Fix issue with promote_constant allowing data parameter. --- nion/data/DataAndMetadata.py | 4 +++- nion/data/test/ExtendedData_test.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/nion/data/DataAndMetadata.py b/nion/data/DataAndMetadata.py index 5c31c51..215f1bc 100644 --- a/nion/data/DataAndMetadata.py +++ b/nion/data/DataAndMetadata.py @@ -1344,7 +1344,9 @@ def promote_constant(data: _DataAndMetadataOrConstant, shape: ShapeType) -> Data # return data and metadata or constant with shape in form of data and metadata if isinstance(data, DataAndMetadata): return data - return new_data_and_metadata(data=numpy.full(shape, data)) + elif isinstance(data, numbers.Complex): + return new_data_and_metadata(data=numpy.full(shape, data)) + raise Exception(f"Unable to convert {data} to DataAndMetadata or constant.") def new_data_and_metadata(data: _ImageDataType, diff --git a/nion/data/test/ExtendedData_test.py b/nion/data/test/ExtendedData_test.py index 58caefe..b9151c2 100644 --- a/nion/data/test/ExtendedData_test.py +++ b/nion/data/test/ExtendedData_test.py @@ -4,6 +4,7 @@ import logging import os import shutil +import typing import unittest # third party libraries @@ -164,6 +165,19 @@ def test_clone_with_data(self) -> None: self.assertEqual("America/Los_Angeles", xdata_clone.timezone) self.assertEqual("-0700", xdata_clone.timezone_offset) + def test_promote_constant(self) -> None: + xdata = DataAndMetadata.new_data_and_metadata(numpy.random.randn(5,4)) + p1 = DataAndMetadata.promote_constant(xdata, xdata.data_shape) + p2 = DataAndMetadata.promote_constant(5.6, (5,4)) + self.assertTrue(numpy.array_equal(xdata.data, p1.data)) + self.assertTrue(numpy.array_equal(numpy.full((5, 4), 5.6), p2.data)) + failed = False + try: + DataAndMetadata.promote_constant(typing.cast(float, numpy.zeros((3,3))), (5,4)) + except Exception as e: + failed = True + self.assertTrue(failed) + if __name__ == '__main__': logging.getLogger().setLevel(logging.DEBUG) From 23d8ad2a190ab3117b9feda7507117ed8c24cad5 Mon Sep 17 00:00:00 2001 From: Chris Meyer <34664+cmeyer@users.noreply.github.com> Date: Mon, 14 Apr 2025 14:26:27 -0700 Subject: [PATCH 3/3] Ensure element data always returns actual ndarray. Test. --- nion/data/Core.py | 4 ++++ nion/data/DataAndMetadata.py | 7 +++++++ nion/data/test/Core_test.py | 13 +++++++++++++ 3 files changed, 24 insertions(+) diff --git a/nion/data/Core.py b/nion/data/Core.py index 30891c1..e5a7c59 100755 --- a/nion/data/Core.py +++ b/nion/data/Core.py @@ -2024,6 +2024,8 @@ def function_element_data_no_copy(data_and_metadata: DataAndMetadata._DataAndMet flag16: bool = True) -> typing.Tuple[typing.Optional[DataAndMetadata.DataAndMetadata], bool]: # extract an element (2d or 1d data element) from data and metadata using the indexes and slices. # flag16 is for backwards compatibility with 0.15.2 and earlier. new callers should set it to False. + # always return an ndarray, never a slice into another type of array (h5py). this helps ensure the display pipeline + # works correctly by ensuring the data is always a numpy array and allow downstream operations will work. data_and_metadata = DataAndMetadata.promote_ndarray(data_and_metadata) result: typing.Optional[DataAndMetadata.DataAndMetadata] = data_and_metadata dimensional_shape = data_and_metadata.dimensional_shape @@ -2065,6 +2067,8 @@ def function_element_data_no_copy(data_and_metadata: DataAndMetadata._DataAndMet next_dimension += collection_dimension_count + datum_dimension_count if result and functools.reduce(operator.mul, result.dimensional_shape) == 0: result = None + # ensure element data is a ndarray and not a slice into another array type (h5py) + result = DataAndMetadata.promote_ndarray_actual(result) if result else None return result, modified diff --git a/nion/data/DataAndMetadata.py b/nion/data/DataAndMetadata.py index 215f1bc..dd567cb 100644 --- a/nion/data/DataAndMetadata.py +++ b/nion/data/DataAndMetadata.py @@ -1329,6 +1329,13 @@ def promote_ndarray(data: _DataAndMetadataLike) -> DataAndMetadata: raise Exception(f"Unable to convert {data} to DataAndMetadata.") +def promote_ndarray_actual(data: _DataAndMetadataLike) -> DataAndMetadata: + maybe_array = promote_ndarray(data) + if not isinstance(maybe_array.data, numpy.ndarray) and hasattr(maybe_array.data, "__array__"): + return maybe_array.clone_with_data(numpy.array(maybe_array.data)) + return maybe_array + + def determine_shape(*datas: _DataAndMetadataOrConstant) -> typing.Optional[ShapeType]: # return the common shape between datas or None if they don't match, ignore constants shape: typing.Optional[ShapeType] = None diff --git a/nion/data/test/Core_test.py b/nion/data/test/Core_test.py index f7d1829..9ccfd9b 100755 --- a/nion/data/test/Core_test.py +++ b/nion/data/test/Core_test.py @@ -1042,6 +1042,19 @@ def test_operations_using_copy_on_h5py_array(self) -> None: Core.function_rebin_2d(d, (2, 2)) Core.function_resample_2d(d, (3, 3)) + def test_element_data_returns_ndarray(self) -> None: + bio = io.BytesIO() + with h5py.File(bio, "w") as f: + dataset = f.create_dataset("data", data=numpy.ones((5, 6), dtype=numpy.float32)) + xdata = DataAndMetadata.new_data_and_metadata(data=dataset) + element, _ = Core.function_element_data_no_copy(xdata, 0, (0, 0)) + assert element + # test whether inline math works, implying it is a numpy array + elementp1 = element.data + 4 + # test directly its type + self.assertIsInstance(element.data, numpy.ndarray) + + if __name__ == '__main__': logging.getLogger().setLevel(logging.DEBUG) unittest.main()