diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 9119868..5e79d76 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -40,7 +40,7 @@ jobs: - uses: actions/setup-python@v4 name: Install Python with: - python-version: '3.9' + python-version: '3.11' - name: Install cibuildwheel run: | @@ -57,6 +57,8 @@ jobs: CIBW_BEFORE_ALL_MACOS: brew install nasm && bash ci-utils/install_prereq_linux.sh && mkdir -p /tmp/bfiocpp_bld && + brew uninstall --ignore-dependencies openssl || true && + sudo rm -rf /usr/local/include/openssl && cp -r local_install /tmp/bfiocpp_bld CIBW_BEFORE_ALL_LINUX: curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 -o nasm-2.15.05.tar.bz2 && tar -xjf nasm-2.15.05.tar.bz2 && @@ -76,7 +78,11 @@ jobs: ON_GITHUB="TRUE" BFIOCPP_DEP_DIR="/tmp/bfiocpp_bld/local_install" CMAKE_ARGS="-DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw -DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON" - CIBW_ENVIRONMENT_LINUX: LD_LIBRARY_PATH="/tmp/bfiocpp_bld/local_install/lib:/tmp/bfiocpp_bld/local_install/lib64:$LD_LIBRARY_PATH" ON_GITHUB="TRUE" BFIOCPP_DEP_DIR="/tmp/bfiocpp_bld/local_install" + CIBW_ENVIRONMENT_LINUX: >- + LD_LIBRARY_PATH="/tmp/bfiocpp_bld/local_install/lib:/tmp/bfiocpp_bld/local_install/lib64:$LD_LIBRARY_PATH" + ON_GITHUB="TRUE" + BFIOCPP_DEP_DIR="/tmp/bfiocpp_bld/local_install" + CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON" CIBW_REPAIR_WHEEL_COMMAND_MACOS: DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\bfiocpp\\bin;$PATH" ON_GITHUB="TRUE" BFIOCPP_DEP_DIR="C:\\TEMP\\bfiocpp_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}" diff --git a/.github/workflows/wheel_build.yml b/.github/workflows/wheel_build.yml index b56549f..595ca05 100644 --- a/.github/workflows/wheel_build.yml +++ b/.github/workflows/wheel_build.yml @@ -39,13 +39,13 @@ jobs: - uses: actions/setup-python@v4 name: Install Python with: - python-version: '3.9' + python-version: '3.11' - name: Install cibuildwheel run: | python -m pip install cibuildwheel delvewheel wheel - - - name: Building wheels + + - name: Building wheels run: | python -m cibuildwheel --output-dir dist env: @@ -56,6 +56,8 @@ jobs: CIBW_BEFORE_ALL_MACOS: brew install nasm && bash ci-utils/install_prereq_linux.sh && mkdir -p /tmp/bfiocpp_bld && + brew uninstall --ignore-dependencies openssl || true && + sudo rm -rf /usr/local/include/openssl && cp -r local_install /tmp/bfiocpp_bld CIBW_BEFORE_ALL_LINUX: curl -L https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.bz2 -o nasm-2.15.05.tar.bz2 && tar -xjf nasm-2.15.05.tar.bz2 && @@ -75,7 +77,11 @@ jobs: ON_GITHUB="TRUE" BFIOCPP_DEP_DIR="/tmp/bfiocpp_bld/local_install" CMAKE_ARGS="-DCMAKE_CXX_FLAGS=-Wno-missing-template-arg-list-after-template-kw -DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON" - CIBW_ENVIRONMENT_LINUX: LD_LIBRARY_PATH="/tmp/bfiocpp_bld/local_install/lib:/tmp/bfiocpp_bld/local_install/lib64:$LD_LIBRARY_PATH" ON_GITHUB="TRUE" BFIOCPP_DEP_DIR="/tmp/bfiocpp_bld/local_install" + CIBW_ENVIRONMENT_LINUX: >- + LD_LIBRARY_PATH="/tmp/bfiocpp_bld/local_install/lib:/tmp/bfiocpp_bld/local_install/lib64:$LD_LIBRARY_PATH" + ON_GITHUB="TRUE" + BFIOCPP_DEP_DIR="/tmp/bfiocpp_bld/local_install" + CMAKE_ARGS="-DTENSORSTORE_USE_SYSTEM_JPEG=ON -DTENSORSTORE_USE_SYSTEM_ZLIB=ON -DTENSORSTORE_USE_SYSTEM_PNG=ON" CIBW_REPAIR_WHEEL_COMMAND_MACOS: DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-listdeps {wheel} && DYLD_LIBRARY_PATH=$REPAIR_LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} CIBW_ENVIRONMENT_WINDOWS: PATH="$TEMP\\bfiocpp\\bin;$PATH" ON_GITHUB="TRUE" BFIOCPP_DEP_DIR="C:\\TEMP\\bfiocpp_bld\\local_install" CMAKE_ARGS="-DCMAKE_GENERATOR=Ninja" CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel}" diff --git a/CMakeLists.txt b/CMakeLists.txt index 4b78f90..1b910f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,8 +37,8 @@ include(FetchContent) FetchContent_Declare( tensorstore - URL "https://github.com/google/tensorstore/archive/refs/tags/v0.1.36.tar.gz" - URL_HASH SHA256=5857582b1b68e5a3f470d2ee8e9a7fa1ad6e6c7e0867c7c69e5a523ae00bc002 + URL "https://github.com/google/tensorstore/archive/refs/tags/v0.1.80.tar.gz" + URL_HASH SHA256=94866de34b6139d77d30e828a50f9e8df98e7dd68e848393470879aeb50ea7bf ) # Additional FetchContent_Declare calls as needed... diff --git a/ci-utils/install_prereq_linux.sh b/ci-utils/install_prereq_linux.sh index c7f5d4b..9587920 100755 --- a/ci-utils/install_prereq_linux.sh +++ b/ci-utils/install_prereq_linux.sh @@ -22,32 +22,37 @@ cmake -DCMAKE_INSTALL_PREFIX=../../$LOCAL_INSTALL_DIR/ -DPYBIND11_TEST=OFF .. make install -j4 cd ../../ + +curl -L https://github.com/madler/zlib/releases/download/v1.3.1/zlib131.zip -o zlib131.zip +unzip zlib131.zip +cd zlib-1.3.1 +mkdir build_man +cd build_man +cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local .. +cmake --build . +cmake --build . --target install +cd ../../ + +curl -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/refs/tags/3.1.0.zip -o 3.1.0.zip +unzip 3.1.0.zip +cd libjpeg-turbo-3.1.0 +mkdir build_man +cd build_man +cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_STATIC=FALSE -DCMAKE_BUILD_TYPE=Release .. if [[ "$OSTYPE" == "darwin"* ]]; then - curl -L https://github.com/madler/zlib/releases/download/v1.3.1/zlib131.zip -o zlib131.zip - unzip zlib131.zip - cd zlib-1.3.1 - mkdir build_man - cd build_man - cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local .. - cmake --build . - cmake --build . --target install - cd ../../ - - curl -L https://github.com/libjpeg-turbo/libjpeg-turbo/archive/refs/tags/3.1.0.zip -o 3.1.0.zip - unzip 3.1.0.zip - cd libjpeg-turbo-3.1.0 - mkdir build_man - cd build_man - cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_STATIC=FALSE -DCMAKE_BUILD_TYPE=Release .. - sudo make install -j4 - cd ../../ + sudo make install -j4 +else + make install -j4 +fi +cd ../../ +make install -j4 +cd ../../ - curl -L https://github.com/glennrp/libpng/archive/refs/tags/v1.6.53.zip -o v1.6.53.zip - unzip v1.6.53.zip - cd libpng-1.6.53 - mkdir build_man - cd build_man - cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local .. - make install -j4 - cd ../../ -fi \ No newline at end of file +curl -L https://github.com/glennrp/libpng/archive/refs/tags/v1.6.53.zip -o v1.6.53.zip +unzip v1.6.53.zip +cd libpng-1.6.53 +mkdir build_man +cd build_man +cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local .. +make install -j4 +cd ../../ diff --git a/setup.py b/setup.py index 578f6d5..a687839 100644 --- a/setup.py +++ b/setup.py @@ -78,18 +78,6 @@ def build_extension(self, ext): ["cmake", ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env ) - if platform.system() == "Linux": - rl = r"s/^#ifdef __has_builtin$/#if defined(__has_builtin)" - rf = r"\&\& defined(__clang__)/" - rsed = rl + rf - path_header = "_deps/tensorstore-src/tensorstore/internal/type_traits.h" - result = subprocess.call(["sed", "-i", - rsed, - path_header], - cwd=self.build_temp, env=env) - if result != 0: - print("--------------- TensorStore patch failed!") - subprocess.check_call( ["cmake", "--build", "."] + build_args, cwd=self.build_temp ) diff --git a/src/cpp/ts_driver/ometiff/driver.cc b/src/cpp/ts_driver/ometiff/driver.cc index 4e0df34..fd128fa 100644 --- a/src/cpp/ts_driver/ometiff/driver.cc +++ b/src/cpp/ts_driver/ometiff/driver.cc @@ -2,6 +2,7 @@ #include "metadata.h" #include "tensorstore/driver/driver.h" +#include "tensorstore/driver/driver_spec.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "tensorstore/context.h" @@ -10,7 +11,11 @@ #include "tensorstore/driver/registry.h" #include "tensorstore/index.h" #include "tensorstore/index_space/index_transform_builder.h" +#include "tensorstore/index_space/transform_broadcastable_array.h" +#include "tensorstore/internal/async_write_array.h" #include "tensorstore/internal/cache/chunk_cache.h" +#include "tensorstore/internal/chunk_grid_specification.h" +#include "tensorstore/contiguous_layout.h" #include "tensorstore/internal/json_binding/json_binding.h" #include "tensorstore/internal/path.h" #include "tensorstore/kvstore/kvstore.h" @@ -19,6 +24,7 @@ #include "tensorstore/util/future.h" #include +#include #include #include @@ -86,8 +92,7 @@ class OmeTiffDriverSpec Future Open( - internal::OpenTransactionPtr transaction, - ReadWriteMode read_write_mode) const override; + internal::DriverOpenRequest request) const override; }; // we need OMETiff Metadata @@ -143,12 +148,17 @@ class DataCache : public internal_kvs_backed_chunk_driver::DataCache { using Base = internal_kvs_backed_chunk_driver::DataCache; public: - explicit DataCache(Initializer initializer, std::string key_prefix) - : Base(initializer, - GetChunkGridSpecification(*static_cast( - initializer.metadata.get()))), + explicit DataCache(Initializer&& initializer, std::string key_prefix) + : Base(std::move(initializer), + GetChunkGridSpecification( + *static_cast( + initializer.metadata.get()))), key_prefix_(std::move(key_prefix)) {} + const OmeTiffMetadata& metadata() const { + return *static_cast(initial_metadata().get()); + } + absl::Status ValidateMetadataCompatibility( const void* existing_metadata_ptr, const void* new_metadata_ptr) override { @@ -195,47 +205,70 @@ class DataCache : public internal_kvs_backed_chunk_driver::DataCache { static internal::ChunkGridSpecification GetChunkGridSpecification( const OmeTiffMetadata& metadata) { - SharedArray fill_value( - internal::AllocateAndConstructSharedElements(1, value_init, - metadata.dtype), - StridedLayout<>(metadata.chunk_layout.shape(), - GetConstantVector(metadata.rank))); - return internal::ChunkGridSpecification( - {internal::ChunkGridSpecification::Component(std::move(fill_value), - // Since all dimensions are - // resizable, just specify - // unbounded - // `component_bounds`. - Box<>(metadata.rank))}); + const DimensionIndex rank = metadata.rank; + + // Create fill value (zero-initialized) + auto fill_value = AllocateArray(span{}, c_order, value_init, + metadata.dtype); + + // Valid data bounds - unbounded since all dimensions are resizable + Box<> valid_data_bounds(rank); + + // Broadcast fill value to the valid bounds + auto chunk_fill_value = BroadcastArray(fill_value, valid_data_bounds).value(); + + // Cell chunk shape from metadata + std::vector cell_chunk_shape(metadata.chunk_layout.shape().begin(), + metadata.chunk_layout.shape().end()); + + // Chunked to cell dimensions mapping (identity for OmeTiff) + std::vector chunked_to_cell_dimensions(rank); + std::iota(chunked_to_cell_dimensions.begin(), + chunked_to_cell_dimensions.end(), static_cast(0)); + + // Create layout order buffer for C order (identity permutation) + DimensionIndex layout_order_buffer[kMaxRank]; + for (DimensionIndex i = 0; i < rank; ++i) { + layout_order_buffer[i] = i; + } + + // Build component list with AsyncWriteArray::Spec + internal::ChunkGridSpecification::ComponentList components; + components.emplace_back( + internal::AsyncWriteArray::Spec{std::move(chunk_fill_value), + std::move(valid_data_bounds), + ContiguousLayoutPermutation<>( + span(layout_order_buffer, rank))}, + std::move(cell_chunk_shape), chunked_to_cell_dimensions); + + return internal::ChunkGridSpecification{std::move(components)}; } - Result, 1>> DecodeChunk( - const void* metadata, span chunk_indices, + Result, 1>> DecodeChunk( + span chunk_indices, absl::Cord data) override { TENSORSTORE_ASSIGN_OR_RETURN( auto array, - internal_ometiff::DecodeChunk(*static_cast(metadata), - std::move(data))); - return absl::InlinedVector, 1>{ - std::move(array)}; + internal_ometiff::DecodeChunk(metadata(), std::move(data))); + return absl::InlinedVector, 1>{ + SharedArray(std::move(array))}; } Result EncodeChunk( - const void* metadata, span chunk_indices, - span> component_arrays) override { + span chunk_indices, + span> component_arrays) override { return absl::Cord(); } - std::string GetChunkStorageKey(const void* metadata_ptr, - span cell_indices) override { + std::string GetChunkStorageKey(span cell_indices) override { // OMETiff is always 5D. So need to add some check here - const auto& metadata = *static_cast(metadata_ptr); + const auto& md = metadata(); - size_t ifd = metadata.GetIfdIndex(cell_indices[2],cell_indices[1],cell_indices[0]); + size_t ifd = md.GetIfdIndex(cell_indices[2],cell_indices[1],cell_indices[0]); std::string key = StrCat(key_prefix_, "__TAG__/" ); - auto& chunk_shape = metadata.chunk_shape; + auto& chunk_shape = md.chunk_shape; // StrAppend(&key, cell_indices.empty() ? 0 : cell_indices[0]); StrAppend(&key, "_", cell_indices[3]*chunk_shape[3]); @@ -276,12 +309,12 @@ class DataCache : public internal_kvs_backed_chunk_driver::DataCache { return absl::OkStatus(); } - Result GetChunkLayout(const void* metadata_ptr, - std::size_t component_index) override { - const auto& metadata = *static_cast(metadata_ptr); + Result GetChunkLayoutFromMetadata(const void* metadata_ptr, + size_t component_index) override { + const auto& md = *static_cast(metadata_ptr); ChunkLayout chunk_layout; TENSORSTORE_RETURN_IF_ERROR(SetChunkLayoutFromMetadata( - metadata.rank, metadata.chunk_shape, chunk_layout)); + md.rank, md.chunk_shape, chunk_layout)); TENSORSTORE_RETURN_IF_ERROR(chunk_layout.Finalize()); return chunk_layout; } @@ -292,11 +325,16 @@ class DataCache : public internal_kvs_backed_chunk_driver::DataCache { std::string key_prefix_; }; -class OmeTiffDriver : public internal_kvs_backed_chunk_driver::RegisteredKvsDriver< - OmeTiffDriver, OmeTiffDriverSpec> { - using Base = - internal_kvs_backed_chunk_driver::RegisteredKvsDriver; +// Forward declaration for the mixin +class OmeTiffDriver; + +using OmeTiffDriverBase = internal_kvs_backed_chunk_driver::RegisteredKvsDriver< + OmeTiffDriver, OmeTiffDriverSpec, DataCache, + internal::ChunkCacheReadWriteDriverMixin< + OmeTiffDriver, internal_kvs_backed_chunk_driver::KvsChunkedDriverBase>>; + +class OmeTiffDriver : public OmeTiffDriverBase { + using Base = OmeTiffDriverBase; public: using Base::Base; @@ -333,7 +371,7 @@ class OmeTiffDriver::OpenState : public OmeTiffDriver::OpenStateBase { Result> Create( - const void* existing_metadata) override { + const void* existing_metadata, CreateOptions options) override { if (existing_metadata) { return absl::AlreadyExistsError(""); } @@ -345,8 +383,8 @@ class OmeTiffDriver::OpenState : public OmeTiffDriver::OpenStateBase { return metadata; } - std::unique_ptr GetDataCache( - DataCache::Initializer initializer) override { + std::unique_ptr GetDataCache( + internal_kvs_backed_chunk_driver::DataCacheInitializer&& initializer) override { return std::make_unique(std::move(initializer), spec().store.path); } @@ -364,9 +402,8 @@ class OmeTiffDriver::OpenState : public OmeTiffDriver::OpenStateBase { }; Future OmeTiffDriverSpec::Open( - internal::OpenTransactionPtr transaction, - ReadWriteMode read_write_mode) const { - return OmeTiffDriver::Open(std::move(transaction), this, read_write_mode); + internal::DriverOpenRequest request) const { + return OmeTiffDriver::Open(this, std::move(request)); } } // namespace diff --git a/src/cpp/ts_driver/ometiff/metadata.cc b/src/cpp/ts_driver/ometiff/metadata.cc index 266fb43..7cc07b8 100644 --- a/src/cpp/ts_driver/ometiff/metadata.cc +++ b/src/cpp/ts_driver/ometiff/metadata.cc @@ -18,7 +18,7 @@ #include "tensorstore/internal/json_binding/json_binding.h" #include "tensorstore/internal/json_binding/std_optional.h" #include "tensorstore/internal/json_metadata_matching.h" -#include "tensorstore/internal/type_traits.h" +#include "absl/meta/type_traits.h" #include "tensorstore/serialization/fwd.h" #include "tensorstore/serialization/json_bindable.h" @@ -51,7 +51,7 @@ absl::Status ValidateMetadata(OmeTiffMetadata& metadata) { constexpr auto MetadataJsonBinder = [](auto maybe_optional) { return [=](auto is_loading, const auto& options, auto* obj, auto* j) { - using T = internal::remove_cvref_t; + using T = absl::remove_cvref_t; DimensionIndex* rank = nullptr; if constexpr (is_loading) { rank = &obj->rank; diff --git a/src/cpp/ts_driver/tiled_tiff/tiled_tiff_key_value_store.cc b/src/cpp/ts_driver/tiled_tiff/tiled_tiff_key_value_store.cc index a642838..53e5355 100644 --- a/src/cpp/ts_driver/tiled_tiff/tiled_tiff_key_value_store.cc +++ b/src/cpp/ts_driver/tiled_tiff/tiled_tiff_key_value_store.cc @@ -33,11 +33,13 @@ #include "tensorstore/internal/json_binding/bindable.h" #include "tensorstore/internal/json_binding/json_binding.h" #include "tensorstore/internal/metrics/counter.h" -#include "tensorstore/internal/os_error_code.h" +#include "tensorstore/internal/os/error_code.h" +#include "tensorstore/internal/os/file_descriptor.h" +#include "tensorstore/internal/os/file_info.h" +#include "tensorstore/internal/os/file_util.h" #include "tensorstore/internal/path.h" -#include "tensorstore/internal/type_traits.h" +#include "tensorstore/internal/uri_utils.h" #include "tensorstore/kvstore/byte_range.h" -#include "tensorstore/kvstore/file/unique_handle.h" #include "tensorstore/kvstore/file/util.h" #include "tensorstore/kvstore/generation.h" #include "tensorstore/kvstore/key_range.h" @@ -54,9 +56,6 @@ #include "tensorstore/util/status.h" #include "tensorstore/util/str_cat.h" -// Include these last to reduce impact of macros. -#include "tensorstore/kvstore/file/posix_file_util.h" -#include "tensorstore/kvstore/file/windows_file_util.h" namespace tensorstore { namespace { @@ -67,13 +66,15 @@ using ::tensorstore::internal::GetLastErrorCode; using ::tensorstore::internal::GetOsErrorStatusCode; using ::tensorstore::internal::OsErrorCode; using ::tensorstore::internal::StatusFromOsError; -using ::tensorstore::internal_file_util::FileDescriptor; -using ::tensorstore::internal_file_util::FileInfo; -using ::tensorstore::internal_file_util::GetFileInfo; using ::tensorstore::internal_file_util::IsKeyValid; -using ::tensorstore::internal_file_util::kLockSuffix; using ::tensorstore::internal_file_util::LongestDirectoryPrefix; -using ::tensorstore::internal_file_util::UniqueFileDescriptor; +using ::tensorstore::internal_os::kLockSuffix; +using ::tensorstore::internal_os::FileDescriptor; +using ::tensorstore::internal_os::FileInfo; +using ::tensorstore::internal_os::GetFileInfo; +using ::tensorstore::internal_os::OpenFileWrapper; +using ::tensorstore::internal_os::OpenFlags; +using ::tensorstore::internal_os::UniqueFileDescriptor; using ::tensorstore::kvstore::ReadResult; // auto& tiled_tiff_bytes_read = internal_metrics::Counter::New( @@ -94,9 +95,9 @@ absl::Status ValidateKey(std::string_view key) { // Encode in the generation fields that uniquely identify the file. StorageGeneration GetFileGeneration(const FileInfo& info) { - return StorageGeneration::FromValues(internal_file_util::GetDeviceId(info), - internal_file_util::GetFileId(info), - internal_file_util::GetMTime(info)); + return StorageGeneration::FromValues(info.GetDeviceId(), + info.GetFileId(), + absl::ToUnixNanos(info.GetMTime())); } /// Returns a absl::Status for the current errno value. The message is composed @@ -108,10 +109,10 @@ absl::Status StatusFromErrno(std::string_view a = {}, std::string_view b = {}, absl::Status VerifyRegularFile(FileDescriptor fd, FileInfo* info, const char* path) { - if (!internal_file_util::GetFileInfo(fd, info)) { - return StatusFromErrno("Error getting file information: ", path); - } - if (!internal_file_util::IsRegularFile(*info)) { + TENSORSTORE_RETURN_IF_ERROR(GetFileInfo(fd, info), + tensorstore::MaybeAnnotateStatus(_, + tensorstore::StrCat("Error getting file information: ", path))); + if (!info->IsRegularFile()) { return absl::FailedPreconditionError( tensorstore::StrCat("Not a regular file: ", path)); } @@ -121,19 +122,18 @@ absl::Status VerifyRegularFile(FileDescriptor fd, FileInfo* info, Result OpenValueFile(const char* path, StorageGeneration* generation, std::int64_t* size = nullptr) { - UniqueFileDescriptor fd = - internal_file_util::OpenExistingFileForReading(path); - if (!fd.valid()) { - auto error = GetLastErrorCode(); - if (GetOsErrorStatusCode(error) == absl::StatusCode::kNotFound) { + auto fd_result = OpenFileWrapper(path, OpenFlags::DefaultRead); + if (!fd_result.ok()) { + if (fd_result.status().code() == absl::StatusCode::kNotFound) { *generation = StorageGeneration::NoValue(); - return fd; + return UniqueFileDescriptor{}; } - return StatusFromOsError(error, "Error opening file: ", path); + return fd_result.status(); } + UniqueFileDescriptor fd = std::move(*fd_result); FileInfo info; TENSORSTORE_RETURN_IF_ERROR(VerifyRegularFile(fd.get(), &info, path)); - if (size) *size = internal_file_util::GetSize(info); + if (size) *size = info.GetSize(); *generation = GetFileGeneration(info); return fd; } @@ -210,9 +210,9 @@ struct ReadTask { read_result.state = ReadResult::kMissing; return read_result; } - if (read_result.stamp.generation == options.if_not_equal || - (!StorageGeneration::IsUnknown(options.if_equal) && - read_result.stamp.generation != options.if_equal)) { + if (read_result.stamp.generation == options.generation_conditions.if_not_equal || + (!StorageGeneration::IsUnknown(options.generation_conditions.if_equal) && + read_result.stamp.generation != options.generation_conditions.if_equal)) { return read_result; }