diff --git a/.github/actions/overwrite-package-version/action.yml b/.github/actions/overwrite-package-version/action.yml index 8a2739456e..aed736ecf9 100644 --- a/.github/actions/overwrite-package-version/action.yml +++ b/.github/actions/overwrite-package-version/action.yml @@ -25,7 +25,7 @@ runs: using: "composite" steps: - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: '3.12' diff --git a/.github/actions/setup-builder/action.yml b/.github/actions/setup-builder/action.yml index 532174590f..e961ed6335 100644 --- a/.github/actions/setup-builder/action.yml +++ b/.github/actions/setup-builder/action.yml @@ -26,8 +26,8 @@ runs: using: "composite" steps: - name: Setup specified Rust toolchain - shell: bash if: ${{ inputs.rust-version != '' }} + shell: bash env: RUST_VERSION: ${{ inputs.rust-version }} run: | diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a94ae0151b..03235972dd 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -23,6 +23,8 @@ updates: schedule: interval: "weekly" day: "sunday" + cooldown: + default-days: 7 # Maintain dependencies for iceberg - package-ecosystem: "cargo" @@ -42,3 +44,5 @@ updates: patterns: - "arrow*" - "parquet" + cooldown: + default-days: 7 diff --git a/.github/workflows/asf-allowlist-check.yml b/.github/workflows/asf-allowlist-check.yml new file mode 100644 index 0000000000..d4e84c5922 --- /dev/null +++ b/.github/workflows/asf-allowlist-check.yml @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Verifies all GitHub Actions refs are on the ASF allowlist. +# Actions not on the allowlist silently fail with "Startup failure" — no logs, +# no notifications, and PRs may appear green because no checks ran. +# See https://github.com/apache/infrastructure-actions/issues/574 +name: "ASF Allowlist Check" + +on: + pull_request: + paths: + - ".github/**" + push: + branches: + - main + paths: + - ".github/**" + +permissions: + contents: read + +jobs: + asf-allowlist-check: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + # Intentionally unpinned to always use the latest allowlist from the ASF. + - uses: apache/infrastructure-actions/allowlist-check@main # zizmor: ignore[unpinned-uses] diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index abe0c377c5..68731cbed3 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -44,11 +44,13 @@ jobs: runs-on: ubuntu-latest if: github.repository == 'apache/iceberg-rust' steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Setup Rust toolchain uses: ./.github/actions/setup-builder with: rust-version: stable - - uses: rustsec/audit-check@v2.0.0 + - uses: rustsec/audit-check@69366f33c96575abad1ee0dba8212993eecbe998 # v2.0.0 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/bindings_python_ci.yml b/.github/workflows/bindings_python_ci.yml index 78e1a9615c..a7abfcbeed 100644 --- a/.github/workflows/bindings_python_ci.yml +++ b/.github/workflows/bindings_python_ci.yml @@ -47,7 +47,9 @@ jobs: check-rust: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Check format working-directory: "bindings/python" run: cargo fmt --all -- --check @@ -58,8 +60,10 @@ jobs: check-python: runs-on: ubuntu-slim steps: - - uses: actions/checkout@v6 - - uses: astral-sh/setup-uv@v7 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 with: version: "0.9.3" enable-cache: true @@ -85,16 +89,18 @@ jobs: - macos-latest - windows-latest steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: 3.12 - - uses: PyO3/maturin-action@v1 + - uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1 with: working-directory: "bindings/python" command: build - args: --out dist - - uses: astral-sh/setup-uv@v7 + args: --out dist -i python3.12 # Explicitly set interpreter; manylinux containers have multiple Pythons and maturin may pick an older one + - uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 # v7.3.1 with: version: "0.9.3" enable-cache: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea2257b676..071d6dbcbf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,19 +53,21 @@ jobs: - ubuntu-latest - macos-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - name: Check License Header - uses: apache/skywalking-eyes/header@v0.8.0 + uses: apache/skywalking-eyes/header@61275cc80d0798a405cb070f7d3a8aaf7cf2c2c1 # v0.8.0 - name: Check toml format run: make check-toml - name: Install protoc - uses: arduino/setup-protoc@v3 + uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3 with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -91,16 +93,18 @@ jobs: - macos-latest - windows-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - name: Cache Rust artifacts - uses: Swatinem/rust-cache@v2 + uses: swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 - name: Install protoc - uses: arduino/setup-protoc@v3 + uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3 with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -117,13 +121,15 @@ jobs: - macos-latest - windows-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - name: Cache Rust artifacts - uses: Swatinem/rust-cache@v2 + uses: swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 - name: Build run: cargo build -p iceberg --no-default-features @@ -138,24 +144,26 @@ jobs: - { name: "doc", args: "--doc --all-features --workspace" } name: Tests (${{ matrix.test-suite.name }}) steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - name: Install protoc - uses: arduino/setup-protoc@v3 + uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Cache Rust artifacts - uses: Swatinem/rust-cache@v2 + uses: swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: key: ${{ matrix.test-suite.name }} - name: Install cargo-nextest if: matrix.test-suite.name == 'default' - uses: taiki-e/install-action@v2 + uses: taiki-e/install-action@0fde6d128a3d980ceac30be8c8b8739abd963b81 # v2.70.0 with: tool: cargo-nextest @@ -164,6 +172,7 @@ jobs: run: make docker-up - name: Run tests + shell: bash env: # Disable debug info to speed up compilation and reduce artifact size RUSTFLAGS: "-C debuginfo=0" @@ -182,9 +191,11 @@ jobs: name: Verify MSRV runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Install protoc - uses: arduino/setup-protoc@v3 + uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Get MSRV diff --git a/.github/workflows/ci_typos.yml b/.github/workflows/ci_typos.yml index 8031cd8ca9..9373c7295d 100644 --- a/.github/workflows/ci_typos.yml +++ b/.github/workflows/ci_typos.yml @@ -43,6 +43,8 @@ jobs: env: FORCE_COLOR: 1 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Check typos - uses: crate-ci/typos@v1.44.0 + uses: crate-ci/typos@631208b7aac2daa8b707f55e7331f9112b0e062d # v1.44.0 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 17bfd8bf3d..81bc6b16f8 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -41,14 +41,16 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@v4 + uses: github/codeql-action/init@c10b8064de6f491fea524254123dbe5e09572f13 # v4 with: languages: actions - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 + uses: github/codeql-action/analyze@c10b8064de6f491fea524254123dbe5e09572f13 # v4 with: category: "/language:actions" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c1c9046154..83e1031d17 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -32,9 +32,9 @@ permissions: jobs: publish: runs-on: ubuntu-latest + environment: publish strategy: - # Publish package one by one instead of flooding the registry - max-parallel: 15 + max-parallel: 1 # Publish package one by one instead of flooding the registry matrix: # Order here is sensitive, as it will be used to determine the order of publishing package: @@ -47,7 +47,9 @@ jobs: - "crates/catalog/sql" - "crates/integrations/datafusion" steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Get MSRV id: get-msrv @@ -62,6 +64,19 @@ jobs: working-directory: ${{ matrix.package }} # Only publish if it's a tag and the tag is not a pre-release if: ${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-') }} - run: cargo publish --all-features + run: cargo publish --all-features # zizmor: ignore[use-trusted-publishing] -- https://github.com/apache/iceberg-rust/issues/1539 + shell: bash env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + + # Trigger Python release after crate publishing completes. + # Only runs for tag pushes; for manual Python releases, use workflow_dispatch on release_python.yml directly. + release-python: + needs: [publish] + if: ${{ startsWith(github.ref, 'refs/tags/') }} + permissions: + contents: read + id-token: write # Required for PyPI trusted publishing in the called workflow + uses: ./.github/workflows/release_python.yml + with: + release_tag: ${{ github.ref_name }} diff --git a/.github/workflows/release_python.yml b/.github/workflows/release_python.yml index 5d97e23a2e..b19fa165dc 100644 --- a/.github/workflows/release_python.yml +++ b/.github/workflows/release_python.yml @@ -18,10 +18,12 @@ name: Publish Python 🐍 distribution 📦 to PyPI on: - workflow_run: - workflows: ["Publish"] # Trigger this workflow after the "publish.yml" workflow completes - types: - - completed + workflow_call: + inputs: + release_tag: + description: 'Release tag (e.g., v0.4.0 or v0.4.0-rc.1)' + required: true + type: string workflow_dispatch: inputs: release_tag: @@ -33,37 +35,24 @@ permissions: contents: read jobs: - check-cargo-publish: - runs-on: ubuntu-latest - # Only run if the triggering workflow succeeded OR if manually triggered - if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} - steps: - - run: echo 'The Publish workflow passed or was manually triggered' - validate-release-tag: runs-on: ubuntu-latest - needs: [check-cargo-publish] outputs: cargo-version: ${{ steps.validate.outputs.cargo-version }} is-rc: ${{ steps.validate.outputs.is-rc }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 if: ${{ github.event_name == 'workflow_dispatch' }} + with: + persist-credentials: false - name: Validate release tag format id: validate - # Use input for workflow_dispatch, otherwise use `workflow_run.head_branch` - # Note, `workflow_run.head_branch` does not contain `refs/tags/` prefix, just the tag name, i.e. `v0.4.0` or `v0.4.0-rc.1` # Valid formats: v.. OR v..-rc. + shell: bash env: - DISPATCH_RELEASE_TAG: ${{ github.event.inputs.release_tag }} - RUN_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + RELEASE_TAG: ${{ inputs.release_tag }} run: | - if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then - RELEASE_TAG="$DISPATCH_RELEASE_TAG" - else - RELEASE_TAG="$RUN_HEAD_BRANCH" - fi echo "Validating release tag: $RELEASE_TAG" if [[ ! "$RELEASE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$ ]]; then echo "❌ Invalid release tag format: $RELEASE_TAG" @@ -114,7 +103,9 @@ jobs: runs-on: ubuntu-latest needs: [validate-release-tag] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Install toml-cli if: ${{ needs.validate-release-tag.outputs.is-rc == 'true' }} @@ -124,19 +115,22 @@ jobs: if: ${{ needs.validate-release-tag.outputs.is-rc == 'true' }} working-directory: "bindings/python" run: | - echo "Setting cargo version to: ${{ needs.validate-release-tag.outputs.cargo-version }}" - toml set Cargo.toml package.version ${{ needs.validate-release-tag.outputs.cargo-version }} > Cargo.toml.tmp + echo "Setting cargo version to: ${NEEDS_VALIDATE_RELEASE_TAG_OUTPUTS_CARGO_VERSION}" + toml set Cargo.toml package.version "${NEEDS_VALIDATE_RELEASE_TAG_OUTPUTS_CARGO_VERSION}" > Cargo.toml.tmp # doing this explicitly to avoid issue in Windows where `mv` does not overwrite existing file rm Cargo.toml mv Cargo.toml.tmp Cargo.toml + shell: bash + env: + NEEDS_VALIDATE_RELEASE_TAG_OUTPUTS_CARGO_VERSION: ${{ needs.validate-release-tag.outputs.cargo-version }} - - uses: PyO3/maturin-action@v1 + - uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1 with: working-directory: "bindings/python" command: sdist args: -o dist - name: Upload sdist - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 with: name: wheels-sdist path: bindings/python/dist @@ -158,7 +152,9 @@ jobs: } - { os: ubuntu-latest, target: "armv7l" } steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Install toml-cli if: ${{ needs.validate-release-tag.outputs.is-rc == 'true' }} @@ -167,14 +163,17 @@ jobs: - name: Set cargo version for RC if: ${{ needs.validate-release-tag.outputs.is-rc == 'true' }} working-directory: "bindings/python" + shell: bash + env: + CARGO_VERSION: ${{ needs.validate-release-tag.outputs.cargo-version }} run: | - echo "Setting cargo version to: ${{ needs.validate-release-tag.outputs.cargo-version }}" - toml set Cargo.toml package.version ${{ needs.validate-release-tag.outputs.cargo-version }} > Cargo.toml.tmp + echo "Setting cargo version to: $CARGO_VERSION" + toml set Cargo.toml package.version "$CARGO_VERSION" > Cargo.toml.tmp # doing this explicitly to avoid issue in Windows where `mv` does not overwrite existing file rm Cargo.toml mv Cargo.toml.tmp Cargo.toml - - uses: actions/setup-python@v6 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: 3.12 - name: Get MSRV @@ -185,15 +184,15 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: ${{ steps.get-msrv.outputs.msrv }} - - uses: PyO3/maturin-action@v1 + - uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1 with: target: ${{ matrix.target }} manylinux: ${{ matrix.manylinux || 'auto' }} working-directory: "bindings/python" command: build - args: --release -o dist + args: --release -o dist -i python3.12 # Explicitly set interpreter; manylinux containers have multiple Pythons and maturin may pick an older one - name: Upload wheels - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 with: name: wheels-${{ matrix.os }}-${{ matrix.target }} path: bindings/python/dist @@ -212,13 +211,13 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: wheels-* merge-multiple: true path: bindings/python/dist - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 with: skip-existing: true packages-dir: bindings/python/dist diff --git a/.github/workflows/release_python_nightly.yml b/.github/workflows/release_python_nightly.yml index 6f9655e29c..66ae0e1db2 100644 --- a/.github/workflows/release_python_nightly.yml +++ b/.github/workflows/release_python_nightly.yml @@ -27,6 +27,7 @@ permissions: jobs: set-version: + if: github.repository == 'apache/iceberg-rust' || github.event_name == 'workflow_dispatch' # Run on schedule for apache repo, or on manual dispatch from any repo runs-on: ubuntu-latest outputs: timestamp: ${{ steps.set-ts.outputs.TIMESTAMP }} @@ -37,30 +38,30 @@ jobs: sdist: needs: set-version - if: github.repository == 'apache/iceberg-rust' # Only run for apache repo runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - uses: ./.github/actions/overwrite-package-version # Overwrite package version with timestamp with: timestamp: ${{ needs.set-version.outputs.TIMESTAMP }} - - uses: PyO3/maturin-action@v1 + - uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1 with: working-directory: "bindings/python" command: sdist args: -o dist - name: Upload sdist - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 with: name: wheels-sdist path: bindings/python/dist wheels: needs: set-version - if: github.repository == 'apache/iceberg-rust' # Only run for apache repo runs-on: "${{ matrix.os }}" strategy: max-parallel: 15 @@ -76,13 +77,15 @@ jobs: } - { os: ubuntu-latest, target: "armv7l" } steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - uses: ./.github/actions/overwrite-package-version # Overwrite package version with timestamp with: timestamp: ${{ needs.set-version.outputs.TIMESTAMP }} - - uses: actions/setup-python@v6 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 with: python-version: 3.12 @@ -95,21 +98,22 @@ jobs: with: rust-version: ${{ steps.get-msrv.outputs.msrv }} - - uses: PyO3/maturin-action@v1 + - uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1 with: target: ${{ matrix.target }} manylinux: ${{ matrix.manylinux || 'auto' }} working-directory: "bindings/python" command: build - args: --release -o dist + args: --release -o dist -i python3.12 # Explicitly set interpreter; manylinux containers have multiple Pythons and maturin may pick an older one - name: Upload wheels - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 with: name: wheels-${{ matrix.os }}-${{ matrix.target }} path: bindings/python/dist testpypi-publish: + if: github.repository == 'apache/iceberg-rust' # Only run for apache repo needs: [sdist, wheels] runs-on: ubuntu-latest @@ -122,7 +126,7 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: wheels-* merge-multiple: true @@ -132,7 +136,7 @@ jobs: - name: Publish to TestPyPI id: publish-testpypi continue-on-error: true - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 with: repository-url: https://test.pypi.org/legacy/ skip-existing: true diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index e2afce4c71..c3d3f18294 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -32,7 +32,7 @@ jobs: if: github.repository_owner == 'apache' runs-on: ubuntu-24.04 steps: - - uses: actions/stale@v10.2.0 + - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 with: # stale issues stale-issue-label: 'stale,security' diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index 59bd2c6f2c..71fb9503c9 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -39,15 +39,17 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false - name: Setup mdBook - uses: peaceiris/actions-mdbook@v2 + uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2 with: mdbook-version: "0.4.36" - name: Install protoc - uses: arduino/setup-protoc@v3 + uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3 with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -64,7 +66,7 @@ jobs: cp -r target/doc ./website/book/api - name: Deploy to gh-pages - uses: peaceiris/actions-gh-pages@v4.0.0 + uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0 if: github.event_name == 'push' && github.ref_name == 'main' with: github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml new file mode 100644 index 0000000000..313835fcbe --- /dev/null +++ b/.github/workflows/zizmor.yml @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: GitHub Actions Security Analysis with zizmor 🌈 + +on: + push: + branches: ["main"] + pull_request: + branches: ["**"] + +permissions: {} + +jobs: + zizmor: + name: Run zizmor 🌈 + runs-on: ubuntu-latest + permissions: {} + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Run zizmor 🌈 + uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2 + with: + advanced-security: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 57f5f73bb4..54bb48a8ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/) and this project adheres to [Semantic Versioning](https://semver.org/). -## [v0.9.0] - 2026-03-09 +## [v0.9.0] - 2026-03-10 ### Breaking Changes @@ -34,6 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). * feat!(io): Implement Storage for OpenDal (#2080) by @CTTY in https://github.com/apache/iceberg-rust/pull/2080 * **Compatibility:** + * chore: Bumping MSRV to 1.92.0 (#2224) by @CTTY in https://github.com/apache/iceberg-rust/pull/2224 * chore: Update MSRV to 1.91.0 (#2204) by @xanderbailey in https://github.com/apache/iceberg-rust/pull/2204 * **Dependency Updates:** @@ -42,6 +43,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### All Changes +* feat(reader): support timestamp type in create_column (#2180) by @chenzl25 in https://github.com/apache/iceberg-rust/pull/2180 +* chore: Bumping MSRV to 1.92.0 (#2224) by @CTTY in https://github.com/apache/iceberg-rust/pull/2224 * perf(reader): Avoid second `create_parquet_record_batch_stream_builder()` call for migrated tables (#2176) by @mbutrovich in https://github.com/apache/iceberg-rust/pull/2176 * doc: Update IO feature flag and examples (#2214) by @CTTY in https://github.com/apache/iceberg-rust/pull/2214 * chore(deps): Bump uuid from 1.21.0 to 1.22.0 (#2221) by @dependabot[bot] in https://github.com/apache/iceberg-rust/pull/2221 diff --git a/Cargo.lock b/Cargo.lock index f6064e23bc..d3b5bb6646 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + [[package]] name = "aes" version = "0.8.4" @@ -16,7 +26,21 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", ] [[package]] @@ -75,9 +99,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -90,15 +114,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -125,9 +149,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "apache-avro" @@ -153,16 +177,16 @@ dependencies = [ "snap", "strum", "strum_macros", - "thiserror 2.0.17", + "thiserror 2.0.18", "uuid", "zstd", ] [[package]] name = "ar_archive_writer" -version = "0.2.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c269894b6fe5e9d7ada0cf69b5bf847ff35bc25fc271f08e1d080fce80339a" +checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" dependencies = [ "object", ] @@ -187,9 +211,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb372a7cbcac02a35d3fb7b3fc1f969ec078e871f9bb899bf00a2e1809bec8a3" +checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c" dependencies = [ "arrow-arith", "arrow-array", @@ -208,9 +232,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "57.2.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "288015089e7931843c80ed4032c5274f02b37bcb720c4a42096d50b390e70372" +checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59" dependencies = [ "arrow-array", "arrow-buffer", @@ -222,9 +246,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8955af33b25f3b175ee10af580577280b4bd01f7e823d94c7cdef7cf8c9aef" +checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" dependencies = [ "ahash", "arrow-buffer", @@ -241,9 +265,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c697ddca96183182f35b3a18e50b9110b11e916d7b7799cbfd4d34662f2c56c2" +checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" dependencies = [ "bytes", "half", @@ -253,9 +277,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "57.2.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8e372ed52bd4ee88cc1e6c3859aa7ecea204158ac640b10e187936e7e87074" +checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d" dependencies = [ "arrow-array", "arrow-buffer", @@ -275,9 +299,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2275877a0e5e7e7c76954669366c2aa1a829e340ab1f612e647507860906fb6b" +checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe" dependencies = [ "arrow-array", "arrow-cast", @@ -290,9 +314,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fdd994a9d28e6365aa78e15da3f3950c0fdcea6b963a12fa1c391afb637b304" +checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" dependencies = [ "arrow-buffer", "arrow-schema", @@ -303,9 +327,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf7df950701ab528bf7c0cf7eeadc0445d03ef5d6ffc151eaae6b38a58feff1" +checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0" dependencies = [ "arrow-array", "arrow-buffer", @@ -319,9 +343,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "371ffd66fa77f71d7628c63f209c9ca5341081051aa32f9c8020feb0def787c0" +checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8" dependencies = [ "arrow-array", "arrow-buffer", @@ -330,7 +354,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.12.1", + "indexmap 2.13.0", "itoa", "lexical-core", "memchr", @@ -343,9 +367,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d8f1870e03d4cbed632959498bcc84083b5a24bded52905ae1695bd29da45b" +checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5" dependencies = [ "arrow-array", "arrow-buffer", @@ -356,9 +380,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "169676f317157dc079cc5def6354d16db63d8861d61046d2f3883268ced6f99f" +checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e" dependencies = [ "arrow-array", "arrow-buffer", @@ -369,9 +393,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c872d36b7bf2a6a6a2b40de9156265f0242910791db366a2c17476ba8330d68" +checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" dependencies = [ "serde_core", "serde_json", @@ -379,9 +403,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68bf3e3efbd1278f770d67e5dc410257300b161b93baedb3aae836144edcaf4b" +checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" dependencies = [ "ahash", "arrow-array", @@ -393,9 +417,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "57.3.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85e968097061b3c0e9fe3079cf2e703e487890700546b5b0647f60fca1b5a8d8" +checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e" dependencies = [ "arrow-array", "arrow-buffer", @@ -438,13 +462,12 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.36" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98ec5f6c2f8bc326c994cb9e241cc257ddaba9afa8555a43cffbb5dd86efaa37" +checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1" dependencies = [ "compression-codecs", "compression-core", - "futures-core", "pin-project-lite", "tokio", ] @@ -505,9 +528,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-config" -version = "1.8.12" +version = "1.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96571e6996817bf3d58f6b569e4b9fd2e9d2fcf9f7424eed07b2ce9bb87535e5" +checksum = "11493b0bad143270fb8ad284a096dd529ba91924c5409adeac856cc1bf047dbc" dependencies = [ "aws-credential-types", "aws-runtime", @@ -525,7 +548,7 @@ dependencies = [ "fastrand", "hex", "http 1.4.0", - "ring", + "sha1", "time", "tokio", "tracing", @@ -535,9 +558,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.11" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd362783681b15d136480ad555a099e82ecd8e2d10a841e14dfd0078d67fee3" +checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -547,9 +570,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.15.2" +version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88aab2464f1f25453baa7a07c84c5b7684e274054ba06817f382357f77a288" +checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" dependencies = [ "aws-lc-sys", "zeroize", @@ -557,9 +580,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.35.0" +version = "0.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45afffdee1e7c9126814751f88dddc747f41d91da16c9551a0f1e8a11e788a1" +checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" dependencies = [ "cc", "cmake", @@ -569,9 +592,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.17" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d81b5b2898f6798ad58f484856768bca817e3cd9de0974c24ae0f1113fe88f1b" +checksum = "5fc0651c57e384202e47153c1260b84a9936e19803d747615edf199dc3b98d17" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -582,9 +605,10 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", + "bytes-utils", "fastrand", - "http 0.2.12", - "http-body 0.4.6", + "http 1.4.0", + "http-body 1.0.1", "percent-encoding", "pin-project-lite", "tracing", @@ -593,15 +617,16 @@ dependencies = [ [[package]] name = "aws-sdk-glue" -version = "1.134.0" +version = "1.142.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef38f029c1e39b96e47df528c90c3c8965e0cae883c43d7e0232f3444f6cd251" +checksum = "3962675ec1f2012ae6439814e784557550fa239a4a291bd4f33d8f514d4fdb5b" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -609,21 +634,23 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", + "http 1.4.0", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-s3tables" -version = "1.47.0" +version = "1.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4252a364a800f00ce90e0fba31906ea812f263ab9f72c08ac5517eb8e33fb5a" +checksum = "2e0ec266873694efc365debded01f44e27a0de3946a3ac15d24c489759e5ddf8" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -631,21 +658,23 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", + "http 1.4.0", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-sso" -version = "1.91.0" +version = "1.97.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee6402a36f27b52fe67661c6732d684b2635152b676aa2babbfb5204f99115d" +checksum = "9aadc669e184501caaa6beafb28c6267fc1baef0810fb58f9b205485ca3f2567" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -653,21 +682,23 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", + "http 1.4.0", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-ssooidc" -version = "1.93.0" +version = "1.99.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a45a7f750bbd170ee3677671ad782d90b894548f4e4ae168302c57ec9de5cb3e" +checksum = "1342a7db8f358d3de0aed2007a0b54e875458e39848d54cc1d46700b2bfcb0a8" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -675,21 +706,23 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", + "http 1.4.0", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-sts" -version = "1.95.0" +version = "1.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55542378e419558e6b1f398ca70adb0b2088077e79ad9f14eb09441f2f7b2164" +checksum = "ab41ad64e4051ecabeea802d6a17845a91e83287e1dd249e6963ea1ba78c428a" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", "aws-smithy-json", + "aws-smithy-observability", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -698,15 +731,16 @@ dependencies = [ "aws-types", "fastrand", "http 0.2.12", + "http 1.4.0", "regex-lite", "tracing", ] [[package]] name = "aws-sigv4" -version = "1.3.7" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e523e1c4e8e7e8ff219d732988e22bfeae8a1cafdbe6d9eca1546fa080be7c" +checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -726,9 +760,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.7" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ee19095c7c4dda59f1697d028ce704c24b2d33c6718790c7f1d5a3015b4107c" +checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc" dependencies = [ "futures-util", "pin-project-lite", @@ -737,9 +771,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.62.6" +version = "0.63.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "826141069295752372f8203c17f28e30c464d22899a43a0c9fd9c458d469c88b" +checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -747,9 +781,9 @@ dependencies = [ "bytes-utils", "futures-core", "futures-util", - "http 0.2.12", "http 1.4.0", - "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", "percent-encoding", "pin-project-lite", "pin-utils", @@ -758,57 +792,51 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.1.5" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59e62db736db19c488966c8d787f52e6270be565727236fd5579eaa301e7bc4a" +checksum = "6a2f165a7feee6f263028b899d0a181987f4fa7179a6411a32a439fba7c5f769" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "h2 0.3.27", - "h2 0.4.13", - "http 0.2.12", + "h2", "http 1.4.0", - "http-body 0.4.6", - "hyper 0.14.32", - "hyper 1.8.1", - "hyper-rustls 0.24.2", - "hyper-rustls 0.27.7", + "hyper", + "hyper-rustls", "hyper-util", "pin-project-lite", - "rustls 0.21.12", - "rustls 0.23.36", + "rustls", "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls", "tower", "tracing", ] [[package]] name = "aws-smithy-json" -version = "0.61.9" +version = "0.62.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49fa1213db31ac95288d981476f78d05d9cbb0353d22cdf3472cc05bb02f6551" +checksum = "9648b0bb82a2eedd844052c6ad2a1a822d1f8e3adee5fbf668366717e428856a" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-observability" -version = "0.1.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f616c3f2260612fe44cede278bafa18e73e6479c4e393e2c4518cf2a9a228a" +checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c" dependencies = [ "aws-smithy-runtime-api", ] [[package]] name = "aws-smithy-query" -version = "0.60.9" +version = "0.60.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae5d689cf437eae90460e944a58b5668530d433b4ff85789e69d2f2a556e057d" +checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" dependencies = [ "aws-smithy-types", "urlencoding", @@ -816,9 +844,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.9.5" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a392db6c583ea4a912538afb86b7be7c5d8887d91604f50eb55c262ee1b4a5f5" +checksum = "028999056d2d2fd58a697232f9eec4a643cf73a71cf327690a7edad1d2af2110" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -832,6 +860,7 @@ dependencies = [ "http 1.4.0", "http-body 0.4.6", "http-body 1.0.1", + "http-body-util", "pin-project-lite", "pin-utils", "tokio", @@ -840,9 +869,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.9.3" +version = "1.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0d43d899f9e508300e587bf582ba54c27a452dd0a9ea294690669138ae14a2" +checksum = "876ab3c9c29791ba4ba02b780a3049e21ec63dabda09268b175272c3733a79e6" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -857,9 +886,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.3.5" +version = "1.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "905cb13a9895626d49cf2ced759b062d913834c7482c38e49557eac4e6193f01" +checksum = "9d73dbfbaa8e4bc57b9045137680b958d274823509a360abfd8e1d514d40c95c" dependencies = [ "base64-simd", "bytes", @@ -883,18 +912,18 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.13" +version = "0.60.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b2f670422ff42bf7065031e72b45bc52a3508bd089f743ea90731ca2b6ea57" +checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.11" +version = "1.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d980627d2dd7bfc32a3c025685a033eeab8d365cc840c631ef59d1b8f428164" +checksum = "47c8323699dd9b3c8d5b3c13051ae9cdef58fd179957c882f8374dd8725962d9" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -933,9 +962,9 @@ dependencies = [ [[package]] name = "base64ct" -version = "1.8.2" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d809780667f4410e7c41b07f52439b94d2bdf8528eeedc287fa38d3b7f95d82" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "bigdecimal" @@ -959,9 +988,9 @@ checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" [[package]] name = "bitflags" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" dependencies = [ "serde_core", ] @@ -977,15 +1006,16 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.2" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" +checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", + "cpufeatures 0.3.0", ] [[package]] @@ -1018,9 +1048,9 @@ dependencies = [ [[package]] name = "bon" -version = "3.8.1" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebeb9aaf9329dff6ceb65c689ca3db33dbf15f324909c60e4e5eef5701ce31b1" +checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" dependencies = [ "bon-macros", "rustversion", @@ -1028,11 +1058,11 @@ dependencies = [ [[package]] name = "bon-macros" -version = "3.8.1" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645" +checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" dependencies = [ - "darling 0.21.3", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", @@ -1064,15 +1094,15 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.19.1" +version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" [[package]] name = "bytemuck" -version = "1.24.0" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" [[package]] name = "byteorder" @@ -1119,9 +1149,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.51" +version = "1.2.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a0aeaff4ff1a90589618835a598e545176939b97874f7abc7851caa0618f203" +checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" dependencies = [ "find-msvc-tools", "jobserver", @@ -1141,11 +1171,22 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.0", +] + [[package]] name = "chrono" -version = "0.4.42" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "iana-time-zone", "js-sys", @@ -1177,9 +1218,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.54" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" dependencies = [ "clap_builder", "clap_derive", @@ -1187,9 +1228,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.54" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstream", "anstyle", @@ -1199,9 +1240,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.49" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" +checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" dependencies = [ "heck", "proc-macro2", @@ -1211,9 +1252,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.6" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "clipboard-win" @@ -1226,33 +1267,33 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" dependencies = [ "cc", ] [[package]] name = "colorchoice" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "colored" -version = "3.0.0" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "comfy-table" -version = "7.2.1" +version = "7.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03b7db8e0b4b2fdad6c551e634134e99ec000e5c8c3b6856c65e8bbaded7a3b" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" dependencies = [ "unicode-segmentation", "unicode-width 0.2.2", @@ -1260,9 +1301,9 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.35" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0f7ac3e5b97fdce45e8922fb05cae2c37f7bbd63d30dd94821dacfd8f3f2bf2" +checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" dependencies = [ "bzip2", "compression-core", @@ -1290,13 +1331,12 @@ dependencies = [ [[package]] name = "console" -version = "0.16.2" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03e45a4a8926227e4197636ba97a9fc9b00477e9f4bd711395687c5f0734bec4" +checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" dependencies = [ "encode_unicode", "libc", - "once_cell", "unicode-width 0.2.2", "windows-sys 0.61.2", ] @@ -1322,16 +1362,16 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "once_cell", "tiny-keccak", ] [[package]] name = "constant_time_eq" -version = "0.3.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" [[package]] name = "core-foundation" @@ -1358,6 +1398,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.4.0" @@ -1437,6 +1486,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -1461,6 +1511,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "darling" version = "0.20.11" @@ -1473,12 +1532,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core 0.21.3", - "darling_macro 0.21.3", + "darling_core 0.23.0", + "darling_macro 0.23.0", ] [[package]] @@ -1497,11 +1556,10 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ - "fnv", "ident_case", "proc-macro2", "quote", @@ -1522,11 +1580,11 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core 0.21.3", + "darling_core 0.23.0", "quote", "syn", ] @@ -1547,9 +1605,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d12ee9fdc6cdb5898c7691bb994f0ba606c4acc93a2258d78bb9f26ff8158bb3" +checksum = "de9f8117889ba9503440f1dd79ebab32ba52ccf1720bb83cd718a29d4edc0d16" dependencies = [ "arrow", "arrow-schema", @@ -1603,9 +1661,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "462dc9ef45e5d688aeaae49a7e310587e81b6016b9d03bace5626ad0043e5a9e" +checksum = "be893b73a13671f310ffcc8da2c546b81efcc54c22e0382c0a28aa3537017137" dependencies = [ "arrow", "async-trait", @@ -1628,9 +1686,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b96dbf1d728fc321817b744eb5080cdd75312faa6980b338817f68f3caa4208" +checksum = "830487b51ed83807d6b32d6325f349c3144ae0c9bf772cf2a712db180c31d5e6" dependencies = [ "arrow", "async-trait", @@ -1651,9 +1709,9 @@ dependencies = [ [[package]] name = "datafusion-cli" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e765fd9f2ce5b451861a7858987f47ce11bb963dff7785f667e68dcea212c3de" +checksum = "8735220c84a731c3917dce75ec837a8376eddf5462b0c5dbaf5a2e354c9b6e05" dependencies = [ "arrow", "async-trait", @@ -1679,9 +1737,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3237a6ff0d2149af4631290074289cae548c9863c885d821315d54c6673a074a" +checksum = "0d7663f3af955292f8004e74bcaf8f7ea3d66cc38438749615bb84815b61a293" dependencies = [ "ahash", "apache-avro", @@ -1691,7 +1749,8 @@ dependencies = [ "half", "hashbrown 0.16.1", "hex", - "indexmap 2.12.1", + "indexmap 2.13.0", + "itertools 0.14.0", "libc", "log", "object_store", @@ -1705,9 +1764,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70b5e34026af55a1bfccb1ef0a763cf1f64e77c696ffcf5a128a278c31236528" +checksum = "5f590205c7e32fe1fea48dd53ffb406e56ae0e7a062213a3ac848db8771641bd" dependencies = [ "futures", "log", @@ -1716,9 +1775,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b2a6be734cc3785e18bbf2a7f2b22537f6b9fb960d79617775a51568c281842" +checksum = "fde1e030a9dc87b743c806fbd631f5ecfa2ccaa4ffb61fa19144a07fea406b79" dependencies = [ "arrow", "async-compression", @@ -1751,9 +1810,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1739b9b07c9236389e09c74f770e88aff7055250774e9def7d3f4f56b3dcc7be" +checksum = "331ebae7055dc108f9b54994b93dff91f3a17445539efe5b74e89264f7b36e15" dependencies = [ "arrow", "arrow-ipc", @@ -1775,9 +1834,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "828088c2fb681cc0e06fb42f541f76c82a0c10278f9fd6334e22c8d1e3574ee7" +checksum = "49dda81c79b6ba57b1853a9158abc66eb85a3aa1cede0c517dabec6d8a4ed3aa" dependencies = [ "apache-avro", "arrow", @@ -1795,9 +1854,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c73bc54b518bbba7c7650299d07d58730293cfba4356f6f428cc94c20b7600" +checksum = "9e0d475088325e2986876aa27bb30d0574f72a22955a527d202f454681d55c5c" dependencies = [ "arrow", "async-trait", @@ -1818,9 +1877,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37812c8494c698c4d889374ecfabbff780f1f26d9ec095dd1bddfc2a8ca12559" +checksum = "ea1520d81f31770f3ad6ee98b391e75e87a68a5bb90de70064ace5e0a7182fe8" dependencies = [ "arrow", "async-trait", @@ -1835,14 +1894,16 @@ dependencies = [ "datafusion-session", "futures", "object_store", + "serde_json", "tokio", + "tokio-stream", ] [[package]] name = "datafusion-datasource-parquet" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2210937ecd9f0e824c397e73f4b5385c97cd1aff43ab2b5836fcfd2d321523fb" +checksum = "95be805d0742ab129720f4c51ad9242cd872599cdb076098b03f061fcdc7f946" dependencies = [ "arrow", "async-trait", @@ -1870,22 +1931,24 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c825f969126bc2ef6a6a02d94b3c07abff871acf4d6dd759ce1255edb7923ce" +checksum = "5c93ad9e37730d2c7196e68616f3f2dd3b04c892e03acd3a8eeca6e177f3c06a" [[package]] name = "datafusion-execution" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa03ef05a2c2f90dd6c743e3e111078e322f4b395d20d4b4d431a245d79521ae" +checksum = "9437d3cd5d363f9319f8122182d4d233427de79c7eb748f23054c9aaa0fdd8df" dependencies = [ "arrow", + "arrow-buffer", "async-trait", "chrono", "dashmap", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr-common", "futures", "log", "object_store", @@ -1898,9 +1961,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef33934c1f98ee695cc51192cc5f9ed3a8febee84fdbcd9131bf9d3a9a78276f" +checksum = "67164333342b86521d6d93fa54081ee39839894fb10f7a700c099af96d7552cf" dependencies = [ "arrow", "async-trait", @@ -1911,7 +1974,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "paste", "recursive", @@ -1921,22 +1984,22 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "000c98206e3dd47d2939a94b6c67af4bfa6732dd668ac4fafdbde408fd9134ea" +checksum = "ab05fdd00e05d5a6ee362882546d29d6d3df43a6c55355164a7fbee12d163bc9" dependencies = [ "arrow", "datafusion-common", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "paste", ] [[package]] name = "datafusion-functions" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "379b01418ab95ca947014066248c22139fe9af9289354de10b445bd000d5d276" +checksum = "04fb863482d987cf938db2079e07ab0d3bb64595f28907a6c2f8671ad71cca7e" dependencies = [ "arrow", "arrow-buffer", @@ -1955,6 +2018,7 @@ dependencies = [ "itertools 0.14.0", "log", "md-5", + "memchr", "num-traits", "rand 0.9.2", "regex", @@ -1965,9 +2029,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd00d5454ba4c3f8ebbd04bd6a6a9dc7ced7c56d883f70f2076c188be8459e4c" +checksum = "829856f4e14275fb376c104f27cbf3c3b57a9cfe24885d98677525f5e43ce8d6" dependencies = [ "ahash", "arrow", @@ -1981,14 +2045,15 @@ dependencies = [ "datafusion-physical-expr-common", "half", "log", + "num-traits", "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aec06b380729a87210a4e11f555ec2d729a328142253f8d557b87593622ecc9f" +checksum = "08af79cc3d2aa874a362fb97decfcbd73d687190cb096f16a6c85a7780cce311" dependencies = [ "ahash", "arrow", @@ -1999,9 +2064,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904f48d45e0f1eb7d0eb5c0f80f2b5c6046a85454364a6b16a2e0b46f62e7dff" +checksum = "465ae3368146d49c2eda3e2c0ef114424c87e8a6b509ab34c1026ace6497e790" dependencies = [ "arrow", "arrow-ord", @@ -2015,16 +2080,18 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", + "hashbrown 0.16.1", "itertools 0.14.0", + "itoa", "log", "paste", ] [[package]] name = "datafusion-functions-table" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9a0d20e2b887e11bee24f7734d780a2588b925796ac741c3118dd06d5aa77f0" +checksum = "6156e6b22fcf1784112fc0173f3ae6e78c8fdb4d3ed0eace9543873b437e2af6" dependencies = [ "arrow", "async-trait", @@ -2038,9 +2105,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3414b0a07e39b6979fe3a69c7aa79a9f1369f1d5c8e52146e66058be1b285ee" +checksum = "ca7baec14f866729012efb89011a6973f3a346dc8090c567bfcd328deff551c1" dependencies = [ "arrow", "datafusion-common", @@ -2056,9 +2123,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf2feae63cd4754e31add64ce75cae07d015bce4bb41cd09872f93add32523a" +checksum = "159228c3280d342658466bb556dc24de30047fe1d7e559dc5d16ccc5324166f9" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2066,9 +2133,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4fe888aeb6a095c4bcbe8ac1874c4b9a4c7ffa2ba849db7922683ba20875aaf" +checksum = "e5427e5da5edca4d21ea1c7f50e1c9421775fe33d7d5726e5641a833566e7578" dependencies = [ "datafusion-doc", "quote", @@ -2077,9 +2144,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a6527c063ae305c11be397a86d8193936f4b84d137fe40bd706dfc178cf733c" +checksum = "89099eefcd5b223ec685c36a41d35c69239236310d71d339f2af0fa4383f3f46" dependencies = [ "arrow", "chrono", @@ -2087,7 +2154,7 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-physical-expr", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "log", "recursive", @@ -2097,9 +2164,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb028323dd4efd049dd8a78d78fe81b2b969447b39c51424167f973ac5811d9" +checksum = "0f222df5195d605d79098ef37bdd5323bff0131c9d877a24da6ec98dfca9fe36" dependencies = [ "ahash", "arrow", @@ -2110,20 +2177,20 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.16.1", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "parking_lot", "paste", - "petgraph 0.8.3", + "petgraph", "recursive", "tokio", ] [[package]] name = "datafusion-physical-expr-adapter" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78fe0826aef7eab6b4b61533d811234a7a9e5e458331ebbf94152a51fc8ab433" +checksum = "40838625d63d9c12549d81979db3dd675d159055eb9135009ba272ab0e8d0f64" dependencies = [ "arrow", "datafusion-common", @@ -2136,9 +2203,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfccd388620734c661bd8b7ca93c44cdd59fecc9b550eea416a78ffcbb29475f" +checksum = "eacbcc4cfd502558184ed58fa3c72e775ec65bf077eef5fd2b3453db676f893c" dependencies = [ "ahash", "arrow", @@ -2146,16 +2213,16 @@ dependencies = [ "datafusion-common", "datafusion-expr-common", "hashbrown 0.16.1", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "parking_lot", ] [[package]] name = "datafusion-physical-optimizer" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde5fa10e73259a03b705d5fddc136516814ab5f441b939525618a4070f5a059" +checksum = "d501d0e1d0910f015677121601ac177ec59272ef5c9324d1147b394988f40941" dependencies = [ "arrow", "datafusion-common", @@ -2172,9 +2239,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1098760fb29127c24cc9ade3277051dc73c9ed0ac0131bd7bcd742e0ad7470" +checksum = "463c88ad6f1ecab1810f4c9f046898bee035b370137eb79b2b2db925e270631d" dependencies = [ "ahash", "arrow", @@ -2193,9 +2260,10 @@ dependencies = [ "futures", "half", "hashbrown 0.16.1", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "log", + "num-traits", "parking_lot", "pin-project-lite", "tokio", @@ -2203,9 +2271,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d0fef4201777b52951edec086c21a5b246f3c82621569ddb4a26f488bc38a9" +checksum = "2857618a0ecbd8cd0cf29826889edd3a25774ec26b2995fc3862095c95d88fc6" dependencies = [ "arrow", "datafusion-common", @@ -2220,9 +2288,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f71f1e39e8f2acbf1c63b0e93756c2e970a64729dab70ac789587d6237c4fde0" +checksum = "ef8637e35022c5c775003b3ab1debc6b4a8f0eb41b069bdd5475dd3aa93f6eba" dependencies = [ "async-trait", "datafusion-common", @@ -2234,39 +2302,44 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556c431f5f2259620c8223254c0ef57aa9a85c576d4da0166157260f71eb0e25" +checksum = "923a8b871962a9d860f036f743a20af50ff04729f1da2468ed220dab4f61c97d" dependencies = [ "arrow", "bigdecimal", "chrono", "crc32fast", + "datafusion", "datafusion-catalog", "datafusion-common", "datafusion-execution", "datafusion-expr", "datafusion-functions", + "datafusion-functions-aggregate", "datafusion-functions-nested", "log", "percent-encoding", "rand 0.9.2", + "serde_json", "sha1", + "sha2", "url", ] [[package]] name = "datafusion-sql" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f44693cfcaeb7a9f12d71d1c576c3a6dc025a12cef209375fa2d16fb3b5670ee" +checksum = "12d9e9f16a1692a11c94bcc418191fa15fd2b4d72a0c1a0c607db93c0b84dd81" dependencies = [ "arrow", "bigdecimal", "chrono", "datafusion-common", "datafusion-expr", - "indexmap 2.12.1", + "datafusion-functions-nested", + "indexmap 2.13.0", "log", "recursive", "regex", @@ -2275,9 +2348,9 @@ dependencies = [ [[package]] name = "datafusion-sqllogictest" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d388fec80647198ae041d314dd7d9e2305207836ecec3ad48908eac6844cdef" +checksum = "a43746bd59e7f2655be4c5553ede4a1ceb1cd34005932fa9e2bd0641c714c46e" dependencies = [ "arrow", "async-trait", @@ -2295,15 +2368,15 @@ dependencies = [ "sqllogictest", "sqlparser", "tempfile", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", ] [[package]] name = "datafusion-substrait" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6042adacd0bd64e56c22f6a7f9ce0ce1793dd367c899d868179d029f110d9215" +checksum = "d5e5656a7e63d51dd3e5af3dbd347ea83bbe993a77c66b854b74961570d16490" dependencies = [ "async-recursion", "async-trait", @@ -2317,7 +2390,6 @@ dependencies = [ "substrait", "tokio", "url", - "uuid", ] [[package]] @@ -2333,9 +2405,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.5.5" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ "powerfmt", "serde_core", @@ -2408,7 +2480,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2424,9 +2496,9 @@ dependencies = [ [[package]] name = "dissimilar" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8975ffdaa0ef3661bfe02dbdcc06c9f829dfafe6a3c474de366a8d5e44276921" +checksum = "aeda16ab4059c5fd2a83f2b9c9e9c981327b18aa8e3b313f7e6563799d4f093e" [[package]] name = "dlv-list" @@ -2516,9 +2588,9 @@ dependencies = [ [[package]] name = "env_filter" -version = "0.1.4" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2" +checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" dependencies = [ "log", "regex", @@ -2526,9 +2598,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.8" +version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" dependencies = [ "anstream", "anstyle", @@ -2545,9 +2617,9 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "erased-serde" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e8918065695684b2b0702da20382d5ae6065cf3327bc2d6436bd49a71ce9f3" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" dependencies = [ "serde", "serde_core", @@ -2561,7 +2633,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2638,9 +2710,9 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "faststr" -version = "0.2.32" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baec6a0289d7f1fe5665586ef7340af82e3037207bef60f5785e57569776f0c8" +checksum = "1ca7d44d22004409a61c393afb3369c8f7bb74abcae49fe249ee01dcc3002113" dependencies = [ "bytes", "rkyv", @@ -2661,9 +2733,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.6" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "fixedbitset" @@ -2683,13 +2755,13 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.5" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", - "libz-rs-sys", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -2732,9 +2804,12 @@ dependencies = [ [[package]] name = "fragile" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619" +checksum = "8878864ba14bb86e818a412bfd6f18f9eabd4ec0f008a28e8f7eb61db532fcf9" +dependencies = [ + "futures-core", +] [[package]] name = "fs-err" @@ -2753,9 +2828,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" dependencies = [ "futures-channel", "futures-core", @@ -2768,9 +2843,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", "futures-sink", @@ -2778,15 +2853,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] name = "futures-executor" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" dependencies = [ "futures-core", "futures-task", @@ -2806,15 +2881,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", @@ -2823,21 +2898,27 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-timer" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-channel", "futures-core", @@ -2847,7 +2928,6 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] @@ -2863,9 +2943,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", "js-sys", @@ -2883,24 +2963,35 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 6.0.0", + "rand_core 0.10.0", "wasip2", "wasip3", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "glob" version = "0.3.3" @@ -2919,25 +3010,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "h2" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap 2.12.1", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "h2" version = "0.4.13" @@ -2950,7 +3022,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.4.0", - "indexmap 2.12.1", + "indexmap 2.13.0", "slab", "tokio", "tokio-util", @@ -3138,66 +3210,26 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" -version = "0.14.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.3.27", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.5.10", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", - "h2 0.4.13", + "h2", "http 1.4.0", "http-body 1.0.1", "httparse", "httpdate", "itoa", "pin-project-lite", - "pin-utils", "smallvec", "tokio", "want", ] -[[package]] -name = "hyper-rustls" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" -dependencies = [ - "futures-util", - "http 0.2.12", - "hyper 0.14.32", - "log", - "rustls 0.21.12", - "tokio", - "tokio-rustls 0.24.1", -] - [[package]] name = "hyper-rustls" version = "0.27.7" @@ -3205,36 +3237,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ "http 1.4.0", - "hyper 1.8.1", + "hyper", "hyper-util", - "rustls 0.23.36", + "rustls", "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls", "tower-service", - "webpki-roots 1.0.5", + "webpki-roots 1.0.6", ] [[package]] name = "hyper-util" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ "base64", "bytes", "futures-channel", - "futures-core", "futures-util", "http 1.4.0", "http-body 1.0.1", - "hyper 1.8.1", + "hyper", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", "tokio", "tower-service", "tracing", @@ -3242,9 +3273,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.64" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -3268,6 +3299,7 @@ dependencies = [ name = "iceberg" version = "0.9.0" dependencies = [ + "aes-gcm", "anyhow", "apache-avro", "array-init", @@ -3321,6 +3353,7 @@ dependencies = [ "typetag", "url", "uuid", + "zeroize", "zstd", ] @@ -3345,7 +3378,6 @@ dependencies = [ "iceberg_test_utils", "serde_json", "tokio", - "tracing", ] [[package]] @@ -3382,6 +3414,10 @@ dependencies = [ "iceberg-catalog-rest", "iceberg-catalog-s3tables", "iceberg-catalog-sql", + "iceberg-storage-opendal", + "iceberg_test_utils", + "reqwest", + "rstest", "sqlx", "tempfile", "tokio", @@ -3404,7 +3440,6 @@ dependencies = [ "serde_derive", "serde_json", "tokio", - "tracing", "typed-builder", "uuid", ] @@ -3535,6 +3570,7 @@ dependencies = [ "async-trait", "bytes", "cfg-if", + "futures", "iceberg", "iceberg_test_utils", "opendal", @@ -3556,12 +3592,13 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -3569,9 +3606,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -3582,9 +3619,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -3596,15 +3633,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -3616,15 +3653,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -3681,9 +3718,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", "hashbrown 0.16.1", @@ -3693,9 +3730,9 @@ dependencies = [ [[package]] name = "indicatif" -version = "0.18.3" +version = "0.18.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9375e112e4b463ec1b1c6c011953545c65a30164fbab5b581df32b3abf0dcb88" +checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb" dependencies = [ "console", "portable-atomic", @@ -3732,24 +3769,24 @@ dependencies = [ [[package]] name = "inventory" -version = "0.3.22" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "009ae045c87e7082cb72dab0ccd01ae075dd00141ddc108f43a0ea150a9e7227" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" dependencies = [ "rustversion", ] [[package]] name = "ipnet" -version = "2.11.0" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "iri-string" -version = "0.7.10" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" dependencies = [ "memchr", "serde", @@ -3781,15 +3818,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.18" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e67e8da4c49d6d9909fe03361f9b620f58898859f5c7aded68351e85e71ecf50" +checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -3797,14 +3834,14 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "jiff-static" -version = "0.2.18" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78" +checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2", "quote", @@ -3813,9 +3850,9 @@ dependencies = [ [[package]] name = "jiff-tzdb" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68971ebff725b9e2ca27a601c5eb38a4c5d64422c4cbab0c535f248087eda5c2" +checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" [[package]] name = "jiff-tzdb-platform" @@ -3838,10 +3875,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.83" +version = "0.3.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -3941,24 +3980,24 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.179" +version = "0.2.184" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5a2d376baa530d1238d133232d15e239abad80d05838b4b59354e5268af431f" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" [[package]] name = "liblzma" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73c36d08cad03a3fbe2c4e7bb3a9e84c57e4ee4135ed0b065cade3d98480c648" +checksum = "b6033b77c21d1f56deeae8014eb9fbe7bdf1765185a6c508b5ca82eeaed7f899" dependencies = [ "liblzma-sys", ] [[package]] name = "liblzma-sys" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b9596486f6d60c3bbe644c0e1be1aa6ccc472ad630fe8927b456973d7cb736" +checksum = "9f2db66f3268487b5033077f266da6777d057949b8f93c8ad82e441df25e6186" dependencies = [ "cc", "libc", @@ -3967,9 +4006,9 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libmimalloc-sys" @@ -3983,13 +4022,14 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.12" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" +checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" dependencies = [ "bitflags", "libc", - "redox_syscall 0.7.0", + "plain", + "redox_syscall 0.7.3", ] [[package]] @@ -4005,9 +4045,9 @@ dependencies = [ [[package]] name = "libtest-mimic" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5297962ef19edda4ce33aaa484386e0a5b3d7f2f4e037cbeee00503ef6b29d33" +checksum = "14e6ba06f0ade6e504aff834d7c34298e5155c6baca353cc6a4aaff2f9fd7f33" dependencies = [ "anstream", "anstyle", @@ -4015,15 +4055,6 @@ dependencies = [ "escape8259", ] -[[package]] -name = "libz-rs-sys" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415" -dependencies = [ - "zlib-rs", -] - [[package]] name = "linked-hash-map" version = "0.5.6" @@ -4043,15 +4074,15 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "lock_api" @@ -4076,9 +4107,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" dependencies = [ "twox-hash", ] @@ -4095,9 +4126,15 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "memo-map" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d1115007560874e373613744c6fba374c17688327a71c1476d1a5954cc857b" [[package]] name = "memoffset" @@ -4132,10 +4169,11 @@ dependencies = [ [[package]] name = "minijinja" -version = "2.17.1" +version = "2.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea5ea1e90055f200af6b8e52a4a34e05e77e7fee953a9fb40c631efdc43cab1" +checksum = "805bfd7352166bae857ee569628b52bcd85a1cecf7810861ebceb1686b72b75d" dependencies = [ + "memo-map", "serde", ] @@ -4151,9 +4189,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "wasi", @@ -4188,9 +4226,9 @@ dependencies = [ [[package]] name = "mockito" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e0603425789b4a70fcc4ac4f5a46a566c116ee3e2a6b768dc623f7719c611de" +checksum = "90820618712cab19cfc46b274c6c22546a82affcb3c3bdf0f29e3db8e1bb92c0" dependencies = [ "assert-json-diff", "bytes", @@ -4199,7 +4237,7 @@ dependencies = [ "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.8.1", + "hyper", "hyper-util", "log", "pin-project-lite", @@ -4213,9 +4251,9 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.12" +version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3dec6bd31b08944e08b58fd99373893a6c17054d6f3ea5006cc894f4f4eee2a" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" dependencies = [ "async-lock", "crossbeam-channel", @@ -4332,7 +4370,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4373,9 +4411,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" [[package]] name = "num-integer" @@ -4409,9 +4447,9 @@ dependencies = [ [[package]] name = "num_enum" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" dependencies = [ "num_enum_derive", "rustversion", @@ -4419,9 +4457,9 @@ dependencies = [ [[package]] name = "num_enum_derive" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -4431,42 +4469,44 @@ dependencies = [ [[package]] name = "object" -version = "0.32.2" +version = "0.37.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" dependencies = [ "memchr", ] [[package]] name = "object_store" -version = "0.12.4" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c1be0c6c22ec0817cdc77d3842f721a17fd30ab6965001415b5402a74e6b740" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", "base64", "bytes", "chrono", "form_urlencoded", - "futures", + "futures-channel", + "futures-core", + "futures-util", "http 1.4.0", "http-body-util", "humantime", - "hyper 1.8.1", + "hyper", "itertools 0.14.0", "md-5", "parking_lot", "percent-encoding", - "quick-xml 0.38.4", - "rand 0.9.2", + "quick-xml 0.39.2", + "rand 0.10.0", "reqwest", "ring", - "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tracing", "url", @@ -4477,9 +4517,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -4487,6 +4527,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "opendal" version = "0.55.0" @@ -4499,7 +4545,7 @@ dependencies = [ "bytes", "crc32c", "futures", - "getrandom 0.2.16", + "getrandom 0.2.17", "http 1.4.0", "http-body 1.0.1", "jiff", @@ -4518,9 +4564,9 @@ dependencies = [ [[package]] name = "openssl-probe" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f50d9b3dabb09ecd771ad0aa242ca6894994c130308ca3d7684634df8037391" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "option-ext" @@ -4566,9 +4612,9 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "owo-colors" -version = "4.2.3" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c6901729fa79e91a0913333229e9ca5dc725089d1c363b2f4b4760709dc4a52" +checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" [[package]] name = "parking" @@ -4601,14 +4647,13 @@ dependencies = [ [[package]] name = "parquet" -version = "57.2.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6a2926a30477c0b95fea6c28c3072712b139337a242c2cc64817bdc20a8854" +checksum = "7d3f9f2205199603564127932b89695f52b62322f541d0fc7179d57c2e1c9877" dependencies = [ "ahash", "arrow-array", "arrow-buffer", - "arrow-cast", "arrow-data", "arrow-ipc", "arrow-schema", @@ -4715,16 +4760,6 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "petgraph" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" -dependencies = [ - "fixedbitset", - "indexmap 2.12.1", -] - [[package]] name = "petgraph" version = "0.8.3" @@ -4733,7 +4768,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap 2.12.1", + "indexmap 2.13.0", "serde", ] @@ -4779,18 +4814,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.10" +version = "1.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.10" +version = "1.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" dependencies = [ "proc-macro2", "quote", @@ -4799,9 +4834,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pin-utils" @@ -4853,26 +4888,44 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" dependencies = [ "portable-atomic", ] [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -4894,9 +4947,9 @@ dependencies = [ [[package]] name = "predicates" -version = "3.1.3" +version = "3.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" dependencies = [ "anstyle", "predicates-core", @@ -4904,15 +4957,15 @@ dependencies = [ [[package]] name = "predicates-core" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727e462b119fe9c93fd0eb1429a5f7647394014cf3c04ab2c0350eeb09095ffa" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" [[package]] name = "predicates-tree" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" dependencies = [ "predicates-core", "termtree", @@ -4940,27 +4993,27 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.23.10+spec-1.0.0", + "toml_edit 0.25.10+spec-1.1.0", ] [[package]] name = "proc-macro2" -version = "1.0.105" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" dependencies = [ "bytes", "prost-derive", @@ -4968,16 +5021,15 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.13.0", + "itertools 0.14.0", "log", "multimap", - "once_cell", - "petgraph 0.7.1", + "petgraph", "prettyplease", "prost", "prost-types", @@ -4988,12 +5040,12 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn", @@ -5001,18 +5053,18 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" dependencies = [ "prost", ] [[package]] name = "psm" -version = "0.1.28" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d11f2fedc3b7dafdc2851bc52f277377c5473d378859be234bc7ebb593144d01" +checksum = "3852766467df634d74f0b2d7819bf8dc483a0eb2e3b0f50f756f9cfe8b0d18d8" dependencies = [ "ar_archive_writer", "cc", @@ -5064,6 +5116,16 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.39.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.9" @@ -5076,9 +5138,9 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.36", - "socket2 0.5.10", - "thiserror 2.0.17", + "rustls", + "socket2 0.6.3", + "thiserror 2.0.18", "tokio", "tracing", "web-time", @@ -5096,10 +5158,10 @@ dependencies = [ "rand 0.9.2", "ring", "rustc-hash", - "rustls 0.23.36", + "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.17", + "thiserror 2.0.18", "tinyvec", "tracing", "web-time", @@ -5114,16 +5176,16 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.3", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] name = "quote" -version = "1.0.43" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -5134,6 +5196,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "radix_trie" version = "0.2.1" @@ -5172,7 +5240,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.3", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.0", ] [[package]] @@ -5192,7 +5271,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.3", + "rand_core 0.9.5", ] [[package]] @@ -5201,19 +5280,25 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "serde", ] [[package]] name = "rand_core" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" + [[package]] name = "recursive" version = "0.1.1" @@ -5245,9 +5330,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.7.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f3fe0889e69e2ae9e41f4d6c4c0181701d00e4697b356fb1f74173a5e0ee27" +checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" dependencies = [ "bitflags", ] @@ -5258,9 +5343,9 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "libredox", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -5285,9 +5370,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.2" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", @@ -5297,9 +5382,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -5308,15 +5393,15 @@ dependencies = [ [[package]] name = "regex-lite" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "regress" @@ -5328,6 +5413,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "rend" version = "0.5.3" @@ -5345,7 +5436,7 @@ dependencies = [ "base64", "chrono", "form_urlencoded", - "getrandom 0.2.16", + "getrandom 0.2.17", "hex", "hmac", "home", @@ -5376,19 +5467,19 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "h2 0.4.13", + "h2", "http 1.4.0", "http-body 1.0.1", "http-body-util", - "hyper 1.8.1", - "hyper-rustls 0.27.7", + "hyper", + "hyper-rustls", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.36", + "rustls", "rustls-native-certs", "rustls-pki-types", "serde", @@ -5396,7 +5487,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls 0.26.4", + "tokio-rustls", "tokio-util", "tower", "tower-http", @@ -5406,7 +5497,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.5", + "webpki-roots 1.0.6", ] [[package]] @@ -5417,7 +5508,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.16", + "getrandom 0.2.17", "libc", "untrusted", "windows-sys 0.52.0", @@ -5425,13 +5516,13 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.8.13" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2e88acca7157d83d789836a3987dafc12bc3d88a050e54b8fe9ea4aaa29d20" +checksum = "1a30e631b7f4a03dee9056b8ef6982e8ba371dd5bedb74d3ec86df4499132c70" dependencies = [ "bytes", "hashbrown 0.16.1", - "indexmap 2.12.1", + "indexmap 2.13.0", "munge", "ptr_meta", "rancor", @@ -5443,9 +5534,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.8.13" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6dffea3c91fa91a3c0fc8a061b0e27fef25c6304728038a6d6bcb1c58ba9bd" +checksum = "8100bb34c0a1d0f907143db3149e6b4eea3c33b9ee8b189720168e818303986f" dependencies = [ "proc-macro2", "quote", @@ -5483,6 +5574,35 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rstest" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", +] + +[[package]] +name = "rstest_macros" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn", + "unicode-ident", +] + [[package]] name = "rust-ini" version = "0.21.3" @@ -5495,9 +5615,9 @@ dependencies = [ [[package]] name = "rustc-hash" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" dependencies = [ "rand 0.8.5", ] @@ -5513,40 +5633,28 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", -] - -[[package]] -name = "rustls" -version = "0.21.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" -dependencies = [ - "log", - "ring", - "rustls-webpki 0.101.7", - "sct", + "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.23.36" +version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.8", + "rustls-webpki", "subtle", "zeroize", ] @@ -5563,20 +5671,11 @@ dependencies = [ "security-framework", ] -[[package]] -name = "rustls-pemfile" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" -dependencies = [ - "rustls-pki-types", -] - [[package]] name = "rustls-pki-types" -version = "1.13.2" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21e6f2ab2928ca4291b86736a8bd920a277a399bba1589409d72154ff87c1282" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ "web-time", "zeroize", @@ -5584,19 +5683,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.7" +version = "0.103.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "rustls-webpki" -version = "0.103.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ "aws-lc-rs", "ring", @@ -5634,9 +5723,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "salsa20" @@ -5658,9 +5747,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -5691,9 +5780,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54e910108742c57a770f492731f99be216a52fadd361b06c8fb59d74ccc267d2" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" dependencies = [ "dyn-clone", "ref-cast", @@ -5730,21 +5819,11 @@ dependencies = [ "sha2", ] -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "security-framework" -version = "3.5.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ "bitflags", "core-foundation", @@ -5755,9 +5834,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.15.0" +version = "2.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" dependencies = [ "core-foundation-sys", "libc", @@ -5874,9 +5953,9 @@ dependencies = [ [[package]] name = "serde_tokenstream" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64060d864397305347a78851c51588fd283767e7e7589829e8121d65512340f1" +checksum = "d7c49585c52c01f13c5c2ebb333f14f6885d76daa768d8a037d28017ec538c69" dependencies = [ "proc-macro2", "quote", @@ -5898,17 +5977,17 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.17.0" +version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "381b283ce7bc6b476d903296fb59d0d36633652b633b27f64db4fb46dcbfc3b9" +checksum = "dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f" dependencies = [ "base64", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.12.1", + "indexmap 2.13.0", "schemars 0.9.0", - "schemars 1.2.0", + "schemars 1.2.1", "serde_core", "serde_json", "serde_with_macros", @@ -5917,11 +5996,11 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.17.0" +version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6d4e30573c8cb306ed6ab1dca8423eec9a463ea0e155f45399455e0368b27e0" +checksum = "d3db8978e608f1fe7357e211969fd9abdcae80bac1ba7a3369bb7eb6b404eb65" dependencies = [ - "darling 0.21.3", + "darling 0.23.0", "proc-macro2", "quote", "syn", @@ -5933,7 +6012,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.13.0", "itoa", "ryu", "serde", @@ -5947,7 +6026,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -5958,7 +6037,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -5999,9 +6078,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" [[package]] name = "simdutf8" @@ -6017,27 +6096,27 @@ checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" [[package]] name = "simple_asn1" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" +checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", ] [[package]] name = "siphasher" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" [[package]] name = "slab" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" @@ -6066,19 +6145,19 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "sonic-number" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a74044c092f4f43ca7a6cfd62854cf9fb5ac8502b131347c990bf22bef1dfe" +checksum = "3775c3390edf958191f1ab1e8c5c188907feebd0f3ce1604cb621f72961dbf32" dependencies = [ "cfg-if", ] @@ -6100,14 +6179,14 @@ dependencies = [ "simdutf8", "sonic-number", "sonic-simd", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] name = "sonic-simd" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5707edbfb34a40c9f2a55fa09a49101d9fec4e0cc171ce386086bd9616f34257" +checksum = "f99e664ecd2d85a68c87e3c7a3cfe691f647ea9e835de984aba4d54a41f817d4" dependencies = [ "cfg-if", ] @@ -6133,9 +6212,9 @@ dependencies = [ [[package]] name = "sqllogictest" -version = "0.28.4" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3566426f72a13e393aa34ca3d542c5b0eb86da4c0db137ee9b5cfccc6179e52d" +checksum = "d03b2262a244037b0b510edbd25a8e6c9fb8d73ee0237fc6cc95a54c16f94a82" dependencies = [ "async-trait", "educe", @@ -6152,15 +6231,15 @@ dependencies = [ "similar", "subst", "tempfile", - "thiserror 2.0.17", + "thiserror 2.0.18", "tracing", ] [[package]] name = "sqlparser" -version = "0.59.0" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f" +checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" dependencies = [ "log", "recursive", @@ -6169,9 +6248,9 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.3.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", @@ -6209,17 +6288,17 @@ dependencies = [ "futures-util", "hashbrown 0.15.5", "hashlink", - "indexmap 2.12.1", + "indexmap 2.13.0", "log", "memchr", "once_cell", "percent-encoding", - "rustls 0.23.36", + "rustls", "serde", "serde_json", "sha2", "smallvec", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-stream", "tracing", @@ -6299,7 +6378,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.17", + "thiserror 2.0.18", "tracing", "whoami", ] @@ -6336,7 +6415,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror 2.0.17", + "thiserror 2.0.18", "tracing", "whoami", ] @@ -6360,7 +6439,7 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror 2.0.17", + "thiserror 2.0.18", "tracing", "url", ] @@ -6373,9 +6452,9 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1f8b29fb42aafcea4edeeb6b2f2d7ecd0d969c48b4cf0d2e64aafc471dd6e59" +checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" dependencies = [ "cc", "cfg-if", @@ -6465,9 +6544,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.114" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -6502,15 +6581,15 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tempfile" -version = "3.25.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.1", + "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6530,11 +6609,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.17", + "thiserror-impl 2.0.18", ] [[package]] @@ -6550,9 +6629,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", @@ -6621,9 +6700,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -6631,9 +6710,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -6646,9 +6725,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.49.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ "bytes", "libc", @@ -6656,39 +6735,29 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.1", + "socket2 0.6.3", "tokio-macros", "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" dependencies = [ "proc-macro2", "quote", "syn", ] -[[package]] -name = "tokio-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" -dependencies = [ - "rustls 0.21.12", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls 0.23.36", + "rustls", "tokio", ] @@ -6701,6 +6770,7 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", + "tokio-util", ] [[package]] @@ -6739,9 +6809,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.7.5+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ "serde_core", ] @@ -6752,33 +6822,33 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.13.0", "serde", "serde_spanned", "toml_datetime 0.6.11", "toml_write", - "winnow", + "winnow 0.7.15", ] [[package]] name = "toml_edit" -version = "0.23.10+spec-1.0.0" +version = "0.25.10+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" +checksum = "a82418ca169e235e6c399a84e395ab6debeb3bc90edc959bf0f48647c6a32d1b" dependencies = [ - "indexmap 2.12.1", - "toml_datetime 0.7.5+spec-1.1.0", + "indexmap 2.13.0", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", - "winnow", + "winnow 1.0.1", ] [[package]] name = "toml_parser" -version = "1.0.6+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "winnow", + "winnow 1.0.1", ] [[package]] @@ -6789,9 +6859,9 @@ checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] name = "tower" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", @@ -6878,9 +6948,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ "nu-ansi-term", "sharded-slab", @@ -6984,7 +7054,7 @@ dependencies = [ "serde", "serde_json", "syn", - "thiserror 2.0.17", + "thiserror 2.0.18", "unicode-ident", ] @@ -7013,9 +7083,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-normalization" @@ -7034,9 +7104,9 @@ checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" [[package]] name = "unicode-width" @@ -7062,6 +7132,16 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -7106,11 +7186,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.22.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" +checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" dependencies = [ - "getrandom 0.4.1", + "getrandom 0.4.2", "js-sys", "serde_core", "wasm-bindgen", @@ -7153,7 +7233,7 @@ dependencies = [ "pin-project", "rand 0.9.2", "socket2 0.5.10", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tokio-stream", "tower", @@ -7184,7 +7264,7 @@ dependencies = [ "rustc-hash", "scopeguard", "sonic-rs", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tracing", "volo", @@ -7223,11 +7303,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.1+wasi-0.2.4" +version = "1.0.2+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" dependencies = [ - "wit-bindgen 0.46.0", + "wit-bindgen", ] [[package]] @@ -7236,7 +7316,7 @@ version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ - "wit-bindgen 0.51.0", + "wit-bindgen", ] [[package]] @@ -7247,9 +7327,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.106" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" dependencies = [ "cfg-if", "once_cell", @@ -7260,22 +7340,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.56" +version = "0.4.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e" dependencies = [ - "cfg-if", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.106" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -7283,9 +7360,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.106" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" dependencies = [ "bumpalo", "proc-macro2", @@ -7296,9 +7373,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.106" +version = "0.2.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" dependencies = [ "unicode-ident", ] @@ -7320,7 +7397,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap 2.12.1", + "indexmap 2.13.0", "wasm-encoder", "wasmparser", ] @@ -7346,15 +7423,15 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ "bitflags", "hashbrown 0.15.5", - "indexmap 2.12.1", + "indexmap 2.13.0", "semver", ] [[package]] name = "web-sys" -version = "0.3.83" +version = "0.3.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +checksum = "cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a" dependencies = [ "js-sys", "wasm-bindgen", @@ -7376,14 +7453,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.5", + "webpki-roots 1.0.6", ] [[package]] name = "webpki-roots" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" dependencies = [ "rustls-pki-types", ] @@ -7404,7 +7481,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7699,18 +7776,21 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" dependencies = [ "memchr", ] [[package]] -name = "wit-bindgen" -version = "0.46.0" +name = "winnow" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" +dependencies = [ + "memchr", +] [[package]] name = "wit-bindgen" @@ -7740,7 +7820,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap 2.12.1", + "indexmap 2.13.0", "prettyplease", "syn", "wasm-metadata", @@ -7771,7 +7851,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", "bitflags", - "indexmap 2.12.1", + "indexmap 2.13.0", "log", "serde", "serde_derive", @@ -7790,7 +7870,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.12.1", + "indexmap 2.13.0", "log", "semver", "serde", @@ -7820,9 +7900,9 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -7831,9 +7911,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", @@ -7843,18 +7923,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.33" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.33" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", @@ -7863,18 +7943,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", @@ -7890,9 +7970,9 @@ checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -7901,9 +7981,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -7912,9 +7992,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", @@ -7923,15 +8003,15 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.5.5" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" [[package]] name = "zmij" -version = "1.0.12" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fc5a66a20078bf1251bde995aa2fdcc4b800c70b5d92dd2c62abc5c60f679f8" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" [[package]] name = "zstd" diff --git a/Cargo.toml b/Cargo.toml index 1d87bc6733..778e69c9d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,22 +40,24 @@ repository = "https://github.com/apache/iceberg-rust" rust-version = "1.92" [workspace.dependencies] +aes = { version = "0.8", features = ["zeroize"] } +aes-gcm = "0.10" anyhow = "1.0.72" apache-avro = { version = "0.21", features = ["zstandard"] } array-init = "2" -arrow-arith = "57.1" -arrow-array = "57.1" -arrow-buffer = "57.1" -arrow-cast = "57.1" -arrow-ord = "57.1" -arrow-schema = "57.1" -arrow-select = "57.1" -arrow-string = "57.1" +arrow-arith = "58" +arrow-array = "58" +arrow-buffer = "58" +arrow-cast = "58" +arrow-ord = "58" +arrow-schema = "58" +arrow-select = "58" +arrow-string = "58" as-any = "0.3.2" async-trait = "0.1.89" aws-config = "1.8.7" -aws-sdk-glue = "1.39" -aws-sdk-s3tables = "1.28.0" +aws-sdk-glue = { version = "1.85", default-features = false, features = ["default-https-client", "rt-tokio"] } +aws-sdk-s3tables = { version = "1.28", default-features = false, features = ["default-https-client", "rt-tokio"] } backon = "1.5.1" base64 = "0.22.1" bimap = "0.6" @@ -64,9 +66,9 @@ cfg-if = "1" chrono = "0.4.41" clap = { version = "4.5.48", features = ["derive", "cargo"] } dashmap = "6" -datafusion = "52.1" -datafusion-cli = "52.1" -datafusion-sqllogictest = "52.1" +datafusion = "53.0.0" +datafusion-cli = "53.0.0" +datafusion-sqllogictest = "53.0.0" derive_builder = "0.20" dirs = "6" enum-ordinalize = "4.3.0" @@ -103,13 +105,14 @@ murmur3 = "0.5.2" once_cell = "1.20" opendal = "0.55.0" ordered-float = "4" -parquet = "57.1" +parquet = "58" pilota = "0.11.10" pretty_assertions = "1.4" rand = "0.8.5" regex = "1.11.3" reqwest = { version = "0.12.12", default-features = false, features = ["json"] } roaring = { version = "0.11" } +rstest = "0.26" fastnum = { version = "0.7", default-features = false, features = ["std", "serde"] } serde = { version = "1.0.219", features = ["rc"] } serde_bytes = "0.11.17" @@ -117,7 +120,7 @@ serde_derive = "1.0.219" serde_json = "1.0.142" serde_repr = "0.1.16" serde_with = "3.4" -sqllogictest = "0.28.3" +sqllogictest = "0.29" sqlx = { version = "0.8.1", default-features = false } stacker = "0.1.20" strum = "0.27.2" @@ -133,4 +136,5 @@ url = "2.5.7" uuid = { version = "1.18", features = ["v7"] } volo = "0.10.6" volo-thrift = "0.10.8" +zeroize = "1.7" zstd = "0.13.3" diff --git a/README.md b/README.md index 693180c6df..4839a855c0 100644 --- a/README.md +++ b/README.md @@ -27,13 +27,18 @@ Rust implementation of [Apache Iceberg™](https://iceberg.apache.org/). The Apache Iceberg Rust project is composed of the following components: -| Name | Release | Docs | -|--------------------------|-----------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| -| [iceberg] | [![iceberg image]][iceberg link] | [![docs release]][iceberg release docs] [![docs dev]][iceberg dev docs] | -| [iceberg-datafusion] | [![iceberg-datafusion image]][iceberg-datafusion link] | [![docs release]][iceberg-datafusion release docs] [![docs dev]][iceberg-datafusion dev docs] | -| [iceberg-catalog-glue] | [![iceberg-catalog-glue image]][iceberg-catalog-glue link] | [![docs release]][iceberg-catalog-glue release docs] [![docs dev]][iceberg-catalog-glue dev docs] | -| [iceberg-catalog-hms] | [![iceberg-catalog-hms image]][iceberg-catalog-hms link] | [![docs release]][iceberg-catalog-hms release docs] [![docs dev]][iceberg-catalog-hms dev docs] | -| [iceberg-catalog-rest] | [![iceberg-catalog-rest image]][iceberg-catalog-rest link] | [![docs release]][iceberg-catalog-rest release docs] [![docs dev]][iceberg-catalog-rest dev docs] | +| Name | Release | Docs | +|-------------------------------|--------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------| +| [iceberg] | [![iceberg image]][iceberg link] | [![docs release]][iceberg release docs] [![docs dev]][iceberg dev docs] | +| [iceberg-catalog-loader] | [![iceberg-catalog-loader image]][iceberg-catalog-loader link] | [![docs release]][iceberg-catalog-loader release docs] [![docs dev]][iceberg-catalog-loader dev docs] | +| [iceberg-catalog-glue] | [![iceberg-catalog-glue image]][iceberg-catalog-glue link] | [![docs release]][iceberg-catalog-glue release docs] [![docs dev]][iceberg-catalog-glue dev docs] | +| [iceberg-catalog-hms] | [![iceberg-catalog-hms image]][iceberg-catalog-hms link] | [![docs release]][iceberg-catalog-hms release docs] [![docs dev]][iceberg-catalog-hms dev docs] | +| [iceberg-catalog-rest] | [![iceberg-catalog-rest image]][iceberg-catalog-rest link] | [![docs release]][iceberg-catalog-rest release docs] [![docs dev]][iceberg-catalog-rest dev docs] | +| [iceberg-catalog-s3tables] | [![iceberg-catalog-s3tables image]][iceberg-catalog-s3tables link] | [![docs release]][iceberg-catalog-s3tables release docs] [![docs dev]][iceberg-catalog-s3tables dev docs] | +| [iceberg-catalog-sql] | [![iceberg-catalog-sql image]][iceberg-catalog-sql link] | [![docs release]][iceberg-catalog-sql release docs] [![docs dev]][iceberg-catalog-sql dev docs] | +| [iceberg-cache-moka] | [![iceberg-cache-moka image]][iceberg-cache-moka link] | [![docs release]][iceberg-cache-moka release docs] [![docs dev]][iceberg-cache-moka dev docs] | +| [iceberg-datafusion] | [![iceberg-datafusion image]][iceberg-datafusion link] | [![docs release]][iceberg-datafusion release docs] [![docs dev]][iceberg-datafusion dev docs] | +| [iceberg-storage-opendal] | [![iceberg-storage-opendal image]][iceberg-storage-opendal link] | [![docs release]][iceberg-storage-opendal release docs] [![docs dev]][iceberg-storage-opendal dev docs] | [docs release]: https://img.shields.io/badge/docs-release-blue [docs dev]: https://img.shields.io/badge/docs-dev-blue @@ -61,13 +66,42 @@ The Apache Iceberg Rust project is composed of the following components: [iceberg-catalog-hms release docs]: https://docs.rs/iceberg-catalog-hms [iceberg-catalog-hms dev docs]: https://rust.iceberg.apache.org/api/iceberg_catalog_hms/ - [iceberg-catalog-rest]: crates/catalog/rest/README.md [iceberg-catalog-rest image]: https://img.shields.io/crates/v/iceberg-catalog-rest.svg [iceberg-catalog-rest link]: https://crates.io/crates/iceberg-catalog-rest [iceberg-catalog-rest release docs]: https://docs.rs/iceberg-catalog-rest [iceberg-catalog-rest dev docs]: https://rust.iceberg.apache.org/api/iceberg_catalog_rest/ +[iceberg-catalog-sql]: crates/catalog/sql +[iceberg-catalog-sql image]: https://img.shields.io/crates/v/iceberg-catalog-sql.svg +[iceberg-catalog-sql link]: https://crates.io/crates/iceberg-catalog-sql +[iceberg-catalog-sql release docs]: https://docs.rs/iceberg-catalog-sql +[iceberg-catalog-sql dev docs]: https://rust.iceberg.apache.org/api/iceberg_catalog_sql/ + +[iceberg-catalog-s3tables]: crates/catalog/s3tables/README.md +[iceberg-catalog-s3tables image]: https://img.shields.io/crates/v/iceberg-catalog-s3tables.svg +[iceberg-catalog-s3tables link]: https://crates.io/crates/iceberg-catalog-s3tables +[iceberg-catalog-s3tables release docs]: https://docs.rs/iceberg-catalog-s3tables +[iceberg-catalog-s3tables dev docs]: https://rust.iceberg.apache.org/api/iceberg_catalog_s3tables/ + +[iceberg-storage-opendal]: crates/storage/opendal/README.md +[iceberg-storage-opendal image]: https://img.shields.io/crates/v/iceberg-storage-opendal.svg +[iceberg-storage-opendal link]: https://crates.io/crates/iceberg-storage-opendal +[iceberg-storage-opendal release docs]: https://docs.rs/iceberg-storage-opendal +[iceberg-storage-opendal dev docs]: https://rust.iceberg.apache.org/api/iceberg_storage_opendal/ + +[iceberg-catalog-loader]: crates/catalog/loader +[iceberg-catalog-loader image]: https://img.shields.io/crates/v/iceberg-catalog-loader.svg +[iceberg-catalog-loader link]: https://crates.io/crates/iceberg-catalog-loader +[iceberg-catalog-loader release docs]: https://docs.rs/iceberg-catalog-loader +[iceberg-catalog-loader dev docs]: https://rust.iceberg.apache.org/api/iceberg_catalog_loader/ + +[iceberg-cache-moka]: crates/integrations/cache-moka +[iceberg-cache-moka image]: https://img.shields.io/crates/v/iceberg-cache-moka.svg +[iceberg-cache-moka link]: https://crates.io/crates/iceberg-cache-moka +[iceberg-cache-moka release docs]: https://docs.rs/iceberg-cache-moka +[iceberg-cache-moka dev docs]: https://rust.iceberg.apache.org/api/iceberg_cache_moka/ + ## Iceberg Rust Implementation Status The features that Iceberg Rust currently supports can be found [here](https://iceberg.apache.org/status/). @@ -99,6 +133,7 @@ The Apache Iceberg community is built on the principles described in the [Apache - [RisingWave](https://github.com/risingwavelabs/risingwave): A Postgres-compatible SQL database designed for real-time event streaming data processing, analysis, and management. - [Wrappers](https://github.com/supabase/wrappers): Postgres Foreign Data Wrapper development framework in Rust. - [ETL](https://github.com/supabase/etl): Stream your Postgres data anywhere in real-time. +- [Apache DataFusion Comet](https://github.com/apache/datafusion-comet): High-performance accelerator for Apache Spark, built on top of the powerful Apache DataFusion query engine. ## License diff --git a/bindings/python/Cargo.lock b/bindings/python/Cargo.lock index 7667a0b1a3..1b5c06f492 100644 --- a/bindings/python/Cargo.lock +++ b/bindings/python/Cargo.lock @@ -56,6 +56,41 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.8.12" @@ -72,9 +107,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] @@ -111,9 +146,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "apache-avro" @@ -133,8 +168,8 @@ dependencies = [ "serde", "serde_bytes", "serde_json", - "strum 0.27.2", - "strum_macros 0.27.2", + "strum", + "strum_macros", "thiserror", "uuid", "zstd", @@ -142,9 +177,9 @@ dependencies = [ [[package]] name = "ar_archive_writer" -version = "0.2.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c269894b6fe5e9d7ada0cf69b5bf847ff35bc25fc271f08e1d080fce80339a" +checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" dependencies = [ "object", ] @@ -169,9 +204,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb372a7cbcac02a35d3fb7b3fc1f969ec078e871f9bb899bf00a2e1809bec8a3" +checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c" dependencies = [ "arrow-arith", "arrow-array", @@ -191,9 +226,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f377dcd19e440174596d83deb49cd724886d91060c07fec4f67014ef9d54049" +checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59" dependencies = [ "arrow-array", "arrow-buffer", @@ -205,9 +240,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eaff85a44e9fa914660fb0d0bb00b79c4a3d888b5334adb3ea4330c84f002" +checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" dependencies = [ "ahash", "arrow-buffer", @@ -224,9 +259,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2819d893750cb3380ab31ebdc8c68874dd4429f90fd09180f3c93538bd21626" +checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" dependencies = [ "bytes", "half", @@ -236,9 +271,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d131abb183f80c450d4591dc784f8d7750c50c6e2bc3fcaad148afc8361271" +checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d" dependencies = [ "arrow-array", "arrow-buffer", @@ -258,9 +293,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2275877a0e5e7e7c76954669366c2aa1a829e340ab1f612e647507860906fb6b" +checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe" dependencies = [ "arrow-array", "arrow-cast", @@ -273,9 +308,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05738f3d42cb922b9096f7786f606fcb8669260c2640df8490533bb2fa38c9d3" +checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" dependencies = [ "arrow-buffer", "arrow-schema", @@ -286,9 +321,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d09446e8076c4b3f235603d9ea7c5494e73d441b01cd61fb33d7254c11964b3" +checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0" dependencies = [ "arrow-array", "arrow-buffer", @@ -302,9 +337,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "371ffd66fa77f71d7628c63f209c9ca5341081051aa32f9c8020feb0def787c0" +checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8" dependencies = [ "arrow-array", "arrow-buffer", @@ -313,7 +348,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.12.1", + "indexmap 2.13.0", "itoa", "lexical-core", "memchr", @@ -326,9 +361,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc94fc7adec5d1ba9e8cd1b1e8d6f72423b33fe978bf1f46d970fafab787521" +checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5" dependencies = [ "arrow-array", "arrow-buffer", @@ -339,9 +374,9 @@ dependencies = [ [[package]] name = "arrow-pyarrow" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbd810e3997bae72f58cda57231ccb0a2fda07911ca1b0a5718cbf9379abb297" +checksum = "e63351dc11981a316c828a6032a5021345bba882f68bc4a36c36825a50725089" dependencies = [ "arrow-array", "arrow-data", @@ -351,9 +386,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "169676f317157dc079cc5def6354d16db63d8861d61046d2f3883268ced6f99f" +checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e" dependencies = [ "arrow-array", "arrow-buffer", @@ -364,9 +399,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27609cd7dd45f006abae27995c2729ef6f4b9361cde1ddd019dc31a5aa017e0" +checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" dependencies = [ "bitflags", "serde_core", @@ -375,9 +410,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae980d021879ea119dd6e2a13912d81e64abed372d53163e804dfe84639d8010" +checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" dependencies = [ "ahash", "arrow-array", @@ -389,9 +424,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf35e8ef49dcf0c5f6d175edee6b8af7b45611805333129c541a8b89a0fc0534" +checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e" dependencies = [ "arrow-array", "arrow-buffer", @@ -424,13 +459,12 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.36" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98ec5f6c2f8bc326c994cb9e241cc257ddaba9afa8555a43cffbb5dd86efaa37" +checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1" dependencies = [ "compression-codecs", "compression-core", - "futures-core", "pin-project-lite", "tokio", ] @@ -446,9 +480,9 @@ dependencies = [ [[package]] name = "async-lock" -version = "3.4.1" +version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd03604047cee9b6ce9de9f70c6cd540a0520c813cbd49bae61f33ab80ed1dc" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" dependencies = [ "event-listener", "event-listener-strategy", @@ -463,7 +497,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -504,11 +538,17 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bigdecimal" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "560f42649de9fa436b73517378a147ec21f6c997a546581df4b4b31677828934" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" dependencies = [ "autocfg", "libm", @@ -526,9 +566,9 @@ checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" [[package]] name = "bitflags" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" [[package]] name = "blake2" @@ -541,15 +581,16 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.2" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" +checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", + "cpufeatures", ] [[package]] @@ -561,6 +602,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "bnum" version = "0.12.1" @@ -573,9 +623,9 @@ dependencies = [ [[package]] name = "bon" -version = "3.8.1" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebeb9aaf9329dff6ceb65c689ca3db33dbf15f324909c60e4e5eef5701ce31b1" +checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" dependencies = [ "bon-macros", "rustversion", @@ -583,17 +633,17 @@ dependencies = [ [[package]] name = "bon-macros" -version = "3.8.1" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645" +checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" dependencies = [ - "darling 0.21.3", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", "quote", "rustversion", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -619,15 +669,15 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.19.0" +version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" [[package]] name = "bytemuck" -version = "1.24.0" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" [[package]] name = "byteorder" @@ -637,9 +687,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "bzip2" @@ -650,11 +700,20 @@ dependencies = [ "libbz2-rs-sys", ] +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" -version = "1.2.43" +version = "1.2.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "739eb0f94557554b3ca9a86d2d37bebd49c5e6d0c1d2bda35ba5bdac830befc2" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" dependencies = [ "find-msvc-tools", "jobserver", @@ -676,9 +735,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.42" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" dependencies = [ "iana-time-zone", "js-sys", @@ -698,22 +757,31 @@ dependencies = [ "phf", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "comfy-table" -version = "7.1.2" +version = "7.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" dependencies = [ - "strum 0.26.3", - "strum_macros 0.26.4", + "unicode-segmentation", "unicode-width", ] [[package]] name = "compression-codecs" -version = "0.4.35" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0f7ac3e5b97fdce45e8922fb05cae2c37f7bbd63d30dd94821dacfd8f3f2bf2" +checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" dependencies = [ "bzip2", "compression-core", @@ -760,7 +828,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "once_cell", "tiny-keccak", ] @@ -776,9 +844,9 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.3.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" [[package]] name = "core-foundation-sys" @@ -860,11 +928,12 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -889,6 +958,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "darling" version = "0.20.11" @@ -901,12 +979,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core 0.21.3", - "darling_macro 0.21.3", + "darling_core 0.23.0", + "darling_macro 0.23.0", ] [[package]] @@ -920,21 +998,20 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] name = "darling_core" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ - "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -945,18 +1022,18 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] name = "darling_macro" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core 0.21.3", + "darling_core 0.23.0", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -975,9 +1052,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503f1f4a9060ae6e650d3dff5dc7a21266fea1302d890768d45b4b28586e830f" +checksum = "de9f8117889ba9503440f1dd79ebab32ba52ccf1720bb83cd718a29d4edc0d16" dependencies = [ "arrow", "arrow-schema", @@ -1030,9 +1107,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14417a3ee4ae3d092b56cd6c1d32e8ff3e2c9ec130ecb2276ec91c89fd599399" +checksum = "be893b73a13671f310ffcc8da2c546b81efcc54c22e0382c0a28aa3537017137" dependencies = [ "arrow", "async-trait", @@ -1055,9 +1132,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d0eba824adb45a4b3ac6f0251d40df3f6a9382371cad136f4f14ac9ebc6bc10" +checksum = "830487b51ed83807d6b32d6325f349c3144ae0c9bf772cf2a712db180c31d5e6" dependencies = [ "arrow", "async-trait", @@ -1078,9 +1155,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0039deefbd00c56adf5168b7ca58568fb058e4ba4c5a03b09f8be371b4e434b6" +checksum = "0d7663f3af955292f8004e74bcaf8f7ea3d66cc38438749615bb84815b61a293" dependencies = [ "ahash", "arrow", @@ -1088,7 +1165,8 @@ dependencies = [ "chrono", "half", "hashbrown 0.16.1", - "indexmap 2.12.1", + "indexmap 2.13.0", + "itertools 0.14.0", "libc", "log", "object_store", @@ -1102,9 +1180,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec7e3e60b813048331f8fb9673583173e5d2dd8fef862834ee871fc98b57ca7" +checksum = "5f590205c7e32fe1fea48dd53ffb406e56ae0e7a062213a3ac848db8771641bd" dependencies = [ "futures", "log", @@ -1113,9 +1191,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "802068957f620302ecf05f84ff4019601aeafd36f5f3f1334984af2e34265129" +checksum = "fde1e030a9dc87b743c806fbd631f5ecfa2ccaa4ffb61fa19144a07fea406b79" dependencies = [ "arrow", "async-compression", @@ -1148,9 +1226,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fc387d5067c62d494a6647d29c5ad4fcdd5a6e50ab4ea1d2568caa2d66f2cc" +checksum = "331ebae7055dc108f9b54994b93dff91f3a17445539efe5b74e89264f7b36e15" dependencies = [ "arrow", "arrow-ipc", @@ -1172,9 +1250,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd5e20579bb6c8bd4e6c620253972fb723822030c280dd6aa047f660d09eeba" +checksum = "9e0d475088325e2986876aa27bb30d0574f72a22955a527d202f454681d55c5c" dependencies = [ "arrow", "async-trait", @@ -1195,9 +1273,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0788b0d48fcef31880a02013ea3cc18e5a4e0eacc3b0abdd2cd0597b99dc96e" +checksum = "ea1520d81f31770f3ad6ee98b391e75e87a68a5bb90de70064ace5e0a7182fe8" dependencies = [ "arrow", "async-trait", @@ -1212,14 +1290,16 @@ dependencies = [ "datafusion-session", "futures", "object_store", + "serde_json", "tokio", + "tokio-stream", ] [[package]] name = "datafusion-datasource-parquet" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66639b70f1f363f5f0950733170100e588f1acfacac90c1894e231194aa35957" +checksum = "95be805d0742ab129720f4c51ad9242cd872599cdb076098b03f061fcdc7f946" dependencies = [ "arrow", "async-trait", @@ -1247,22 +1327,24 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e44b41f3e8267c6cf3eec982d63f34db9f1dd5f30abfd2e1f124f0871708952e" +checksum = "5c93ad9e37730d2c7196e68616f3f2dd3b04c892e03acd3a8eeca6e177f3c06a" [[package]] name = "datafusion-execution" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e456f60e5d38db45335e84617006d90af14a8c8c5b8e959add708b2daaa0e2c" +checksum = "9437d3cd5d363f9319f8122182d4d233427de79c7eb748f23054c9aaa0fdd8df" dependencies = [ "arrow", + "arrow-buffer", "async-trait", "chrono", "dashmap", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr-common", "futures", "log", "object_store", @@ -1274,9 +1356,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6507c719804265a58043134580c1c20767e7c23ba450724393f03ec982769ad9" +checksum = "67164333342b86521d6d93fa54081ee39839894fb10f7a700c099af96d7552cf" dependencies = [ "arrow", "async-trait", @@ -1287,7 +1369,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "paste", "recursive", @@ -1297,22 +1379,22 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a413caa9c5885072b539337aed68488f0291653e8edd7d676c92df2480f6cab0" +checksum = "ab05fdd00e05d5a6ee362882546d29d6d3df43a6c55355164a7fbee12d163bc9" dependencies = [ "arrow", "datafusion-common", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "paste", ] [[package]] name = "datafusion-ffi" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30f57f7f63a25a0b78b3f2a5e18c0ecbd54851b64064ac0d5a9eb05efd5586d2" +checksum = "4b8250f7cdf463a0ad145f41d7508bcfa54c9b9f027317e599f0331097e3cc38" dependencies = [ "abi_stable", "arrow", @@ -1340,9 +1422,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "189256495dc9cbbb8e20dbcf161f60422e628d201a78df8207e44bd4baefadb6" +checksum = "04fb863482d987cf938db2079e07ab0d3bb64595f28907a6c2f8671ad71cca7e" dependencies = [ "arrow", "arrow-buffer", @@ -1361,6 +1443,7 @@ dependencies = [ "itertools 0.14.0", "log", "md-5", + "memchr", "num-traits", "rand 0.9.2", "regex", @@ -1371,9 +1454,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12e73dfee4cd67c4a507ffff4c5a711d39983adf544adbc09c09bf06f789f413" +checksum = "829856f4e14275fb376c104f27cbf3c3b57a9cfe24885d98677525f5e43ce8d6" dependencies = [ "ahash", "arrow", @@ -1387,14 +1470,15 @@ dependencies = [ "datafusion-physical-expr-common", "half", "log", + "num-traits", "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87727bd9e65f4f9ac6d608c9810b7da9eaa3b18b26a4a4b76520592d49020acf" +checksum = "08af79cc3d2aa874a362fb97decfcbd73d687190cb096f16a6c85a7780cce311" dependencies = [ "ahash", "arrow", @@ -1405,9 +1489,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5ef761359224b7c2b5a1bfad6296ac63225f8583d08ad18af9ba1a89ac3887" +checksum = "465ae3368146d49c2eda3e2c0ef114424c87e8a6b509ab34c1026ace6497e790" dependencies = [ "arrow", "arrow-ord", @@ -1421,16 +1505,18 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", + "hashbrown 0.16.1", "itertools 0.14.0", + "itoa", "log", "paste", ] [[package]] name = "datafusion-functions-table" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b17dac25dfda2d2a90ff0ad1c054a11fb1523766226bec6e9bd8c410daee2ae" +checksum = "6156e6b22fcf1784112fc0173f3ae6e78c8fdb4d3ed0eace9543873b437e2af6" dependencies = [ "arrow", "async-trait", @@ -1444,9 +1530,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c594a29ddb22cbdbce500e4d99b5b2392c5cecb4c1086298b41d1ffec14dbb77" +checksum = "ca7baec14f866729012efb89011a6973f3a346dc8090c567bfcd328deff551c1" dependencies = [ "arrow", "datafusion-common", @@ -1462,9 +1548,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aa1b15ed81c7543f62264a30dd49dec4b1b0b698053b968f53be32dfba4f729" +checksum = "159228c3280d342658466bb556dc24de30047fe1d7e559dc5d16ccc5324166f9" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1472,20 +1558,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00c31c4795597aa25b74cab5174ac07a53051f27ce1e011ecaffa9eaeecef81" +checksum = "e5427e5da5edca4d21ea1c7f50e1c9421775fe33d7d5726e5641a833566e7578" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] name = "datafusion-optimizer" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80ccf60767c09302b2e0fc3afebb3761a6d508d07316fab8c5e93312728a21bb" +checksum = "89099eefcd5b223ec685c36a41d35c69239236310d71d339f2af0fa4383f3f46" dependencies = [ "arrow", "chrono", @@ -1493,7 +1579,7 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-physical-expr", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "log", "recursive", @@ -1503,9 +1589,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64b7f277556944e4edd3558da01d9e9ff9f5416f1c0aa7fee088e57bd141a7e" +checksum = "0f222df5195d605d79098ef37bdd5323bff0131c9d877a24da6ec98dfca9fe36" dependencies = [ "ahash", "arrow", @@ -1516,7 +1602,7 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.16.1", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "parking_lot", "paste", @@ -1527,9 +1613,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7abaee372ea2d19c016ee9ef8629c4415257d291cdd152bc7f0b75f28af1b63" +checksum = "40838625d63d9c12549d81979db3dd675d159055eb9135009ba272ab0e8d0f64" dependencies = [ "arrow", "datafusion-common", @@ -1542,9 +1628,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42237efe621f92adc22d111b531fdbc2cc38ca9b5e02327535628fb103ae2157" +checksum = "eacbcc4cfd502558184ed58fa3c72e775ec65bf077eef5fd2b3453db676f893c" dependencies = [ "ahash", "arrow", @@ -1552,16 +1638,16 @@ dependencies = [ "datafusion-common", "datafusion-expr-common", "hashbrown 0.16.1", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "parking_lot", ] [[package]] name = "datafusion-physical-optimizer" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd093498bd1319c6e5c76e9dfa905e78486f01b34579ce97f2e3a49f84c37fac" +checksum = "d501d0e1d0910f015677121601ac177ec59272ef5c9324d1147b394988f40941" dependencies = [ "arrow", "datafusion-common", @@ -1578,9 +1664,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cbe61b12daf81a9f20ba03bd3541165d51f86e004ef37426b11881330eed261" +checksum = "463c88ad6f1ecab1810f4c9f046898bee035b370137eb79b2b2db925e270631d" dependencies = [ "ahash", "arrow", @@ -1599,9 +1685,10 @@ dependencies = [ "futures", "half", "hashbrown 0.16.1", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "log", + "num-traits", "parking_lot", "pin-project-lite", "tokio", @@ -1609,9 +1696,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cf75daf56aa6b1c6867cc33ff0fb035d517d6d06737fd355a3e1ef67cba6e7a" +checksum = "677ee4448a010ed5faeff8d73ff78972c2ace59eff3cd7bd15833a1dafa00492" dependencies = [ "arrow", "chrono", @@ -1632,13 +1719,14 @@ dependencies = [ "datafusion-proto-common", "object_store", "prost", + "rand 0.9.2", ] [[package]] name = "datafusion-proto-common" -version = "52.1.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12a0cb3cce232a3de0d14ef44b58a6537aeb1362cfb6cf4d808691ddbb918956" +checksum = "965eca01edc8259edbbd95883a00b6d81e329fd44a019cfac3a03b026a83eade" dependencies = [ "arrow", "datafusion-common", @@ -1647,9 +1735,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0124331116db7f79df92ebfd2c3b11a8f90240f253555c9bb084f10b6fecf1dd" +checksum = "2857618a0ecbd8cd0cf29826889edd3a25774ec26b2995fc3862095c95d88fc6" dependencies = [ "arrow", "datafusion-common", @@ -1664,9 +1752,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1673e3c58ba618a6ea0568672f00664087b8982c581e9afd5aa6c3c79c9b431f" +checksum = "ef8637e35022c5c775003b3ab1debc6b4a8f0eb41b069bdd5475dd3aa93f6eba" dependencies = [ "async-trait", "datafusion-common", @@ -1678,27 +1766,39 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5272d256dab5347bb39d2040589f45d8c6b715b27edcb5fffe88cc8b9c3909cb" +checksum = "12d9e9f16a1692a11c94bcc418191fa15fd2b4d72a0c1a0c607db93c0b84dd81" dependencies = [ "arrow", "bigdecimal", "chrono", "datafusion-common", "datafusion-expr", - "indexmap 2.12.1", + "datafusion-functions-nested", + "indexmap 2.13.0", "log", "recursive", "regex", "sqlparser", ] +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" -version = "0.5.5" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ "powerfmt", "serde_core", @@ -1722,7 +1822,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -1732,7 +1832,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -1755,14 +1855,14 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] name = "dissimilar" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8975ffdaa0ef3661bfe02dbdcc06c9f829dfafe6a3c474de366a8d5e44276921" +checksum = "aeda16ab4059c5fd2a83f2b9c9e9c981327b18aa8e3b313f7e6563799d4f093e" [[package]] name = "dlv-list" @@ -1793,9 +1893,9 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "erased-serde" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e8918065695684b2b0702da20382d5ae6065cf3327bc2d6436bd49a71ce9f3" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" dependencies = [ "serde", "serde_core", @@ -1863,9 +1963,9 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "find-msvc-tools" -version = "0.1.4" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "fixedbitset" @@ -1875,9 +1975,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flatbuffers" -version = "25.9.23" +version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b6620799e7340ebd9968d2e0708eb82cf1971e9a16821e2091b6d6e475eed5" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ "bitflags", "rustc_version", @@ -1885,13 +1985,13 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.5" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", - "libz-rs-sys", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -1923,9 +2023,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" dependencies = [ "futures-channel", "futures-core", @@ -1938,9 +2038,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", "futures-sink", @@ -1948,15 +2048,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] name = "futures-executor" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" dependencies = [ "futures-core", "futures-task", @@ -1965,38 +2065,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] name = "futures-macro" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] name = "futures-sink" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" [[package]] name = "futures-task" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-util" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-channel", "futures-core", @@ -2006,7 +2106,6 @@ dependencies = [ "futures-task", "memchr", "pin-project-lite", - "pin-utils", "slab", ] @@ -2021,9 +2120,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.9" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -2031,9 +2130,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", "js-sys", @@ -2051,11 +2150,34 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "glob" version = "0.3.3" @@ -2141,21 +2263,20 @@ dependencies = [ [[package]] name = "home" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "http" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" dependencies = [ "bytes", - "fnv", "itoa", ] @@ -2196,9 +2317,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ "atomic-waker", "bytes", @@ -2234,14 +2355,13 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.17" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ "base64", "bytes", "futures-channel", - "futures-core", "futures-util", "http", "http-body", @@ -2258,9 +2378,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.64" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2284,6 +2404,7 @@ dependencies = [ name = "iceberg" version = "0.9.0" dependencies = [ + "aes-gcm", "anyhow", "apache-avro", "array-init", @@ -2323,12 +2444,13 @@ dependencies = [ "serde_json", "serde_repr", "serde_with", - "strum 0.27.2", + "strum", "tokio", "typed-builder", "typetag", "url", "uuid", + "zeroize", "zstd", ] @@ -2355,6 +2477,7 @@ dependencies = [ "async-trait", "bytes", "cfg-if", + "futures", "iceberg", "opendal", "reqsign", @@ -2366,9 +2489,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f578a71f2bfaf7ceb30b519a645ae48024b45f9eecbe060a31a004d7b4ba9462" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" dependencies = [ "displaydoc", "potential_utf", @@ -2379,9 +2502,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c219b62bf5a06801012446193fdfcbd7970e876823aba4c62def2ce957dcb44" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" dependencies = [ "displaydoc", "litemap", @@ -2392,9 +2515,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33747cecc725eebb47ac503fab725e395d50cb7889ae490a1359f130611d4cc5" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -2406,15 +2529,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6ce2d23e1b3c45624ba6a23e2c767e01c9680e0c0800b39c7abfff9565175d8" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" [[package]] name = "icu_properties" -version = "2.1.0" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d70f9b6574c79f7a83ea5ce72cc88d271a3e77355c5f7748a107e751d8617fb" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" dependencies = [ "icu_collections", "icu_locale_core", @@ -2426,15 +2549,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.0" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17fa55bf868e28e638ed132bcee1e5c21ba2c1e52c15e7c78b781858e7b54342" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" [[package]] name = "icu_provider" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f64958e359123591ae1f17a27b5fc9ebdb50c98b04e0401146154de1d8fe3e44" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" dependencies = [ "displaydoc", "icu_locale_core", @@ -2445,6 +2568,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "ident_case" version = "1.0.1" @@ -2485,9 +2614,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", "hashbrown 0.16.1", @@ -2496,12 +2625,13 @@ dependencies = [ ] [[package]] -name = "indoc" -version = "2.0.7" +name = "inout" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ - "rustversion", + "block-padding", + "generic-array", ] [[package]] @@ -2521,15 +2651,15 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.11.0" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "iri-string" -version = "0.7.8" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb" dependencies = [ "memchr", "serde", @@ -2555,15 +2685,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.15" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.16" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35" +checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -2576,20 +2706,20 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.16" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69" +checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] name = "jiff-tzdb" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68971ebff725b9e2ca27a601c5eb38a4c5d64422c4cbab0c535f248087eda5c2" +checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" [[package]] name = "jiff-tzdb-platform" @@ -2612,14 +2742,44 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.82" +version = "0.3.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" dependencies = [ "once_cell", "wasm-bindgen", ] +[[package]] +name = "jsonwebtoken" +version = "9.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" +dependencies = [ + "base64", + "js-sys", + "pem", + "ring", + "serde", + "serde_json", + "simple_asn1", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "lexical-core" version = "1.0.6" @@ -2685,9 +2845,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.177" +version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "libloading" @@ -2701,18 +2861,18 @@ dependencies = [ [[package]] name = "liblzma" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73c36d08cad03a3fbe2c4e7bb3a9e84c57e4ee4135ed0b065cade3d98480c648" +checksum = "b6033b77c21d1f56deeae8014eb9fbe7bdf1765185a6c508b5ca82eeaed7f899" dependencies = [ "liblzma-sys", ] [[package]] name = "liblzma-sys" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b9596486f6d60c3bbe644c0e1be1aa6ccc472ad630fe8927b456973d7cb736" +checksum = "9f2db66f3268487b5033077f266da6777d057949b8f93c8ad82e441df25e6186" dependencies = [ "cc", "libc", @@ -2721,24 +2881,15 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" - -[[package]] -name = "libz-rs-sys" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "840db8cf39d9ec4dd794376f38acc40d0fc65eec2a8f484f7fd375b84602becd" -dependencies = [ - "zlib-rs", -] +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" @@ -2757,9 +2908,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.28" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lru-slab" @@ -2769,9 +2920,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" dependencies = [ "twox-hash", ] @@ -2788,18 +2939,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.6" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" - -[[package]] -name = "memoffset" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "miniz_oxide" @@ -2813,9 +2955,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", "wasi", @@ -2824,9 +2966,9 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.11" +version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8261cd88c312e0004c1d51baad2980c66528dfdb2bee62003e643a4d8f86b077" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" dependencies = [ "async-lock", "crossbeam-channel", @@ -2837,7 +2979,6 @@ dependencies = [ "futures-util", "parking_lot", "portable-atomic", - "rustc_version", "smallvec", "tagptr", "uuid", @@ -2860,6 +3001,22 @@ dependencies = [ "serde", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -2871,9 +3028,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" [[package]] name = "num-integer" @@ -2884,6 +3041,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2896,23 +3064,25 @@ dependencies = [ [[package]] name = "object" -version = "0.32.2" +version = "0.37.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" dependencies = [ "memchr", ] [[package]] name = "object_store" -version = "0.12.4" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c1be0c6c22ec0817cdc77d3842f721a17fd30ab6965001415b5402a74e6b740" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", "bytes", "chrono", - "futures", + "futures-channel", + "futures-core", + "futures-util", "http", "humantime", "itertools 0.14.0", @@ -2929,9 +3099,15 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "opendal" @@ -2945,14 +3121,14 @@ dependencies = [ "bytes", "crc32c", "futures", - "getrandom 0.2.16", + "getrandom 0.2.17", "http", "http-body", "jiff", "log", "md-5", "percent-encoding", - "quick-xml 0.38.3", + "quick-xml 0.38.4", "reqsign", "reqwest", "serde", @@ -3021,14 +3197,13 @@ dependencies = [ [[package]] name = "parquet" -version = "57.1.0" +version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be3e4f6d320dd92bfa7d612e265d7d08bba0a240bab86af3425e1d255a511d89" +checksum = "7d3f9f2205199603564127932b89695f52b62322f541d0fc7179d57c2e1c9877" dependencies = [ "ahash", "arrow-array", "arrow-buffer", - "arrow-cast", "arrow-data", "arrow-ipc", "arrow-schema", @@ -3062,6 +3237,35 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -3076,7 +3280,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap 2.12.1", + "indexmap 2.13.0", "serde", ] @@ -3100,9 +3304,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pin-utils" @@ -3110,23 +3314,73 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "pkcs5", + "rand_core 0.6.4", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" -version = "1.11.1" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" dependencies = [ "portable-atomic", ] @@ -3162,23 +3416,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] name = "proc-macro2" -version = "1.0.103" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" dependencies = [ "bytes", "prost-derive", @@ -3186,22 +3440,22 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] name = "psm" -version = "0.1.28" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d11f2fedc3b7dafdc2851bc52f277377c5473d378859be234bc7ebb593144d01" +checksum = "3852766467df634d74f0b2d7819bf8dc483a0eb2e3b0f50f756f9cfe8b0d18d8" dependencies = [ "ar_archive_writer", "cc", @@ -3222,35 +3476,32 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.26.0" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba0117f4212101ee6544044dae45abe1083d30ce7b29c4b5cbdfa2354e07383" +checksum = "cf85e27e86080aafd5a22eae58a162e133a589551542b3e5cee4beb27e54f8e1" dependencies = [ - "indoc", "libc", - "memoffset", "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", - "unindent", ] [[package]] name = "pyo3-build-config" -version = "0.26.0" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fc6ddaf24947d12a9aa31ac65431fb1b851b8f4365426e182901eabfb87df5f" +checksum = "8bf94ee265674bf76c09fa430b0e99c26e319c945d96ca0d5a8215f31bf81cf7" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.26.0" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "025474d3928738efb38ac36d4744a74a400c901c7596199e20e45d98eb194105" +checksum = "491aa5fc66d8059dd44a75f4580a2962c1862a1c2945359db36f6c2818b748dc" dependencies = [ "libc", "pyo3-build-config", @@ -3258,27 +3509,27 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.26.0" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e64eb489f22fe1c95911b77c44cc41e7c19f3082fc81cce90f657cdc42ffded" +checksum = "f5d671734e9d7a43449f8480f8b38115df67bef8d21f76837fa75ee7aaa5e52e" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] name = "pyo3-macros-backend" -version = "0.26.0" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "100246c0ecf400b475341b8455a9213344569af29a3c841d29270e53102e0fcf" +checksum = "22faaa1ce6c430a1f71658760497291065e6450d7b5dc2bcf254d49f66ee700a" dependencies = [ "heck", "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -3299,9 +3550,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.38.3" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42a232e7487fc2ef313d96dde7948e7a3c05101870d8985e4fd8d26aedd27b89" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" dependencies = [ "memchr", "serde", @@ -3329,9 +3580,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.13" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "bytes", "getrandom 0.3.4", @@ -3364,9 +3615,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.41" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -3377,6 +3628,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.8.5" @@ -3395,7 +3652,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.3", + "rand_core 0.9.5", ] [[package]] @@ -3415,7 +3672,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.3", + "rand_core 0.9.5", ] [[package]] @@ -3424,14 +3681,14 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", ] [[package]] name = "rand_core" -version = "0.9.3" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" dependencies = [ "getrandom 0.3.4", ] @@ -3453,7 +3710,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -3482,14 +3739,14 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] name = "regex" -version = "1.12.2" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" dependencies = [ "aho-corasick", "memchr", @@ -3499,9 +3756,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -3510,15 +3767,15 @@ dependencies = [ [[package]] name = "regex-lite" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "repr_offset" @@ -3540,16 +3797,19 @@ dependencies = [ "base64", "chrono", "form_urlencoded", - "getrandom 0.2.16", + "getrandom 0.2.17", "hex", "hmac", "home", "http", + "jsonwebtoken", "log", + "once_cell", "percent-encoding", "quick-xml 0.37.5", "rand 0.8.5", "reqwest", + "rsa", "rust-ini", "serde", "serde_json", @@ -3560,9 +3820,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.24" +version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64", "bytes", @@ -3607,7 +3867,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.16", + "getrandom 0.2.17", "libc", "untrusted", "windows-sys 0.52.0", @@ -3615,14 +3875,35 @@ dependencies = [ [[package]] name = "roaring" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f08d6a905edb32d74a5d5737a0c9d7e950c312f3c46cb0ca0a2ca09ea11878a0" +checksum = "8ba9ce64a8f45d7fc86358410bb1a82e8c987504c0d4900e9141d69a9f26c885" dependencies = [ "bytemuck", "byteorder", ] +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "sha2", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rust-ini" version = "0.21.3" @@ -3650,9 +3931,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.2" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ "bitflags", "errno", @@ -3663,9 +3944,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.34" +version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9586e9ee2b4f8fab52a0048ca7334d7024eef48e2cb9407e3497bb7cab7fa7" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ "once_cell", "ring", @@ -3677,9 +3958,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.13.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ "web-time", "zeroize", @@ -3687,9 +3968,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.7" +version = "0.103.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10b3f4191e8a80e6b43eebabfac91e5dcecebb27a71f04e820c47ec41d314bf" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ "ring", "rustls-pki-types", @@ -3704,9 +3985,18 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" -version = "1.0.20" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] [[package]] name = "same-file" @@ -3731,9 +4021,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.0.4" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82d20c4491bc164fa2f6c5d44565947a52ad80b9505d8e36f8d54c27c739fcd0" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" dependencies = [ "dyn-clone", "ref-cast", @@ -3747,6 +4037,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2", +] + [[package]] name = "semver" version = "1.0.27" @@ -3805,20 +4106,20 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] name = "serde_json" -version = "1.0.145" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "itoa", "memchr", - "ryu", "serde", "serde_core", + "zmij", ] [[package]] @@ -3829,7 +4130,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -3846,17 +4147,17 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.15.1" +version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa66c845eee442168b2c8134fec70ac50dc20e760769c8ba0ad1319ca1959b04" +checksum = "dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f" dependencies = [ "base64", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.12.1", + "indexmap 2.13.0", "schemars 0.9.0", - "schemars 1.0.4", + "schemars 1.2.1", "serde_core", "serde_json", "serde_with_macros", @@ -3865,14 +4166,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.15.1" +version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91a903660542fced4e99881aa481bdbaec1634568ee02e0b8bd57c64cb38955" +checksum = "d3db8978e608f1fe7357e211969fd9abdcae80bac1ba7a3369bb7eb6b404eb65" dependencies = [ - "darling 0.21.3", + "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -3903,11 +4204,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "simd-adler32" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" [[package]] name = "simdutf8" @@ -3915,17 +4226,29 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "simple_asn1" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" +dependencies = [ + "num-bigint", + "num-traits", + "thiserror", + "time", +] + [[package]] name = "siphasher" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" [[package]] name = "slab" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" @@ -3941,19 +4264,35 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", ] [[package]] name = "sqlparser" -version = "0.59.0" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f" +checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" dependencies = [ "log", "recursive", @@ -3962,13 +4301,13 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.3.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -3979,9 +4318,9 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1f8b29fb42aafcea4edeeb6b2f2d7ecd0d969c48b4cf0d2e64aafc471dd6e59" +checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" dependencies = [ "cc", "cfg-if", @@ -3996,32 +4335,13 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" - [[package]] name = "strum" version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "strum_macros 0.27.2", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.113", + "strum_macros", ] [[package]] @@ -4033,7 +4353,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -4055,9 +4375,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.113" +version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678faa00651c9eb72dd2020cbdf275d92eccb2400d568e419efdd64838145cb4" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" dependencies = [ "proc-macro2", "quote", @@ -4081,7 +4401,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -4092,18 +4412,18 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "target-lexicon" -version = "0.13.3" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "tempfile" -version = "3.23.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix", "windows-sys 0.61.2", @@ -4111,22 +4431,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -4142,30 +4462,30 @@ dependencies = [ [[package]] name = "time" -version = "0.3.44" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", "num-conv", "powerfmt", - "serde", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.24" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", @@ -4192,9 +4512,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -4207,9 +4527,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.48.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" dependencies = [ "bytes", "libc", @@ -4222,13 +4542,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -4241,6 +4561,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -4256,9 +4588,9 @@ dependencies = [ [[package]] name = "tower" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", @@ -4271,9 +4603,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "bitflags", "bytes", @@ -4301,9 +4633,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.41" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "pin-project-lite", "tracing-attributes", @@ -4312,20 +4644,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] name = "tracing-core" -version = "0.1.34" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", ] @@ -4380,7 +4712,7 @@ checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -4416,7 +4748,7 @@ checksum = "27a7a9b72ba121f6f1f6c3632b85604cac41aedb5ddc70accbebb6cac83de846" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -4427,15 +4759,15 @@ checksum = "f8c1ae7cc0fdb8b842d65d127cb981574b0d2b249b74d1c7a2986863dc134f71" [[package]] name = "unicode-ident" -version = "1.0.20" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "462eeb75aeb73aea900253ce739c8e18a67423fadf006037cd3ff27e82748a06" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "a559e63b5d8004e12f9bce88af5c6d939c58de839b7532cfe9653846cedd2a9e" [[package]] name = "unicode-width" @@ -4444,10 +4776,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] -name = "unindent" -version = "0.2.4" +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "universal-hash" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] [[package]] name = "untrusted" @@ -4457,9 +4799,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.7" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", @@ -4475,11 +4817,11 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.19.0" +version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" +checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.2", "js-sys", "serde_core", "wasm-bindgen", @@ -4518,18 +4860,27 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.1+wasi-0.2.4" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.105" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" dependencies = [ "cfg-if", "once_cell", @@ -4540,11 +4891,12 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.55" +version = "0.4.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0" +checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" dependencies = [ "cfg-if", + "futures-util", "js-sys", "once_cell", "wasm-bindgen", @@ -4553,9 +4905,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.105" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4563,26 +4915,48 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.105" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.105" +version = "0.2.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap 2.13.0", + "wasm-encoder", + "wasmparser", +] + [[package]] name = "wasm-streams" version = "0.4.2" @@ -4596,11 +4970,23 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap 2.13.0", + "semver", +] + [[package]] name = "web-sys" -version = "0.3.82" +version = "0.3.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1" +checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" dependencies = [ "js-sys", "wasm-bindgen", @@ -4618,9 +5004,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.3" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b130c0d2d49f8b6889abc456e795e82525204f27c42cf767cf0d7734e089b8" +checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" dependencies = [ "rustls-pki-types", ] @@ -4677,7 +5063,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -4688,7 +5074,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -4882,9 +5268,91 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "wit-bindgen" -version = "0.46.0" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap 2.13.0", + "prettyplease", + "syn 2.0.117", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.117", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap 2.13.0", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.13.0", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] [[package]] name = "writeable" @@ -4911,28 +5379,28 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.27" +version = "0.8.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.27" +version = "0.8.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] @@ -4952,7 +5420,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", "synstructure", ] @@ -4992,14 +5460,20 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.113", + "syn 2.0.117", ] [[package]] name = "zlib-rs" -version = "0.5.2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + +[[package]] +name = "zmij" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" [[package]] name = "zstd" diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 4813c72db7..251f96d169 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -31,12 +31,12 @@ license = "Apache-2.0" crate-type = ["cdylib"] [dependencies] -arrow = { version = "57.1", features = ["pyarrow", "chrono-tz"] } +arrow = { version = "58", features = ["pyarrow", "chrono-tz"] } iceberg = { path = "../../crates/iceberg" } -iceberg-storage-opendal = { path = "../../crates/storage/opendal", features = ["opendal-s3", "opendal-fs", "opendal-memory"] } -pyo3 = { version = "0.26", features = ["extension-module", "abi3-py310"] } +iceberg-storage-opendal = { path = "../../crates/storage/opendal", features = ["opendal-all"] } +pyo3 = { version = "0.28", features = ["extension-module", "abi3-py310"] } iceberg-datafusion = { path = "../../crates/integrations/datafusion" } -datafusion-ffi = { version = "52.1" } +datafusion-ffi = "53.0.0" tokio = { version = "1.46.1", default-features = false } [profile.release] diff --git a/bindings/python/src/datafusion_table_provider.rs b/bindings/python/src/datafusion_table_provider.rs index 7fa9f53dbd..2f4745fa7a 100644 --- a/bindings/python/src/datafusion_table_provider.rs +++ b/bindings/python/src/datafusion_table_provider.rs @@ -16,62 +16,35 @@ // under the License. use std::collections::HashMap; -use std::ffi::CString; +use std::ffi::{CStr, CString}; use std::sync::Arc; use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec; use datafusion_ffi::table_provider::FFI_TableProvider; use iceberg::TableIdent; -use iceberg::io::{FileIOBuilder, StorageFactory}; +use iceberg::io::FileIOBuilder; use iceberg::table::StaticTable; use iceberg_datafusion::table::IcebergStaticTableProvider; -use iceberg_storage_opendal::OpenDalStorageFactory; +use iceberg_storage_opendal::OpenDalResolvingStorageFactory; use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::{PyAnyMethods, PyCapsuleMethods, *}; +use pyo3::prelude::*; use pyo3::types::{PyAny, PyCapsule}; use crate::runtime::runtime; -/// Parse the scheme from a URL and return the appropriate StorageFactory. -fn storage_factory_from_path(path: &str) -> PyResult> { - let scheme = path - .split("://") - .next() - .ok_or_else(|| PyRuntimeError::new_err(format!("Invalid path, missing scheme: {path}")))?; - - let factory: Arc = match scheme { - "file" | "" => Arc::new(OpenDalStorageFactory::Fs), - "s3" | "s3a" => Arc::new(OpenDalStorageFactory::S3 { - configured_scheme: scheme.to_string(), - customized_credential_load: None, - }), - "memory" => Arc::new(OpenDalStorageFactory::Memory), - _ => { - return Err(PyRuntimeError::new_err(format!( - "Unsupported storage scheme: {scheme}" - ))); - } - }; - - Ok(factory) -} - -pub(crate) fn validate_pycapsule(capsule: &Bound, name: &str) -> PyResult<()> { +// pyo3 0.28's CapsuleName only exposes `unsafe fn as_cstr() -> &CStr`, +// so we accept &CStr to allow direct comparison without UTF-8 validation. +pub(crate) fn validate_pycapsule(capsule: &Bound, name: &CStr) -> PyResult<()> { let capsule_name = capsule.name()?; - if capsule_name.is_none() { - return Err(PyValueError::new_err(format!( - "Expected {name} PyCapsule to have name set." - ))); - } - - let capsule_name = capsule_name.unwrap().to_str()?; - if capsule_name != name { - return Err(PyValueError::new_err(format!( - "Expected name '{name}' in PyCapsule, instead got '{capsule_name}'" - ))); + match capsule_name { + None => Err(PyValueError::new_err( + "Expected PyCapsule to have name set.", + )), + Some(capsule_name) if unsafe { capsule_name.as_cstr() } != name => { + Err(PyValueError::new_err("PyCapsule name mismatch")) + } + _ => Ok(()), } - - Ok(()) } pub(crate) fn ffi_logical_codec_from_pycapsule( @@ -84,10 +57,12 @@ pub(crate) fn ffi_logical_codec_from_pycapsule( obj }; - let capsule = capsule.downcast::()?; - validate_pycapsule(capsule, "datafusion_logical_extension_codec")?; + let capsule_name = c"datafusion_logical_extension_codec"; + let capsule = capsule.cast::()?; + validate_pycapsule(capsule, capsule_name)?; - let codec = unsafe { capsule.reference::() }; + let ptr = capsule.pointer_checked(Some(capsule_name))?; + let codec = unsafe { &*(ptr.as_ptr() as *const FFI_LogicalExtensionCodec) }; Ok(codec.clone()) } @@ -110,7 +85,7 @@ impl PyIcebergDataFusionTable { let table_ident = TableIdent::from_strs(identifier) .map_err(|e| PyRuntimeError::new_err(format!("Invalid table identifier: {e}")))?; - let factory = storage_factory_from_path(&metadata_location)?; + let factory = Arc::new(OpenDalResolvingStorageFactory::new()); let mut builder = FileIOBuilder::new(factory); diff --git a/bindings/python/src/transform.rs b/bindings/python/src/transform.rs index c159d573fc..b50a0fa84d 100644 --- a/bindings/python/src/transform.rs +++ b/bindings/python/src/transform.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use arrow::array::{Array, ArrayData, make_array}; +use arrow::array::{ArrayData, make_array}; use arrow::pyarrow::{FromPyArrow, ToPyArrow}; use iceberg::spec::Transform; use iceberg::transform::create_transform_function; @@ -70,7 +70,7 @@ fn apply(py: Python, array: Py, transform: Transform) -> PyResult=52 for FFI compatibility", + "Iceberg table provider requires datafusion>=53 for FFI compatibility", allow_module_level=True, ) diff --git a/crates/catalog/glue/Cargo.toml b/crates/catalog/glue/Cargo.toml index e41253de36..d8d0927a90 100644 --- a/crates/catalog/glue/Cargo.toml +++ b/crates/catalog/glue/Cargo.toml @@ -37,7 +37,6 @@ iceberg = { workspace = true } iceberg-storage-opendal = { workspace = true, features = ["opendal-s3"] } serde_json = { workspace = true } tokio = { workspace = true } -tracing = { workspace = true } [dev-dependencies] iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } diff --git a/crates/catalog/glue/src/catalog.rs b/crates/catalog/glue/src/catalog.rs index bf2f392330..a7e0171337 100644 --- a/crates/catalog/glue/src/catalog.rs +++ b/crates/catalog/glue/src/catalog.rs @@ -17,6 +17,7 @@ use std::collections::HashMap; use std::fmt::Debug; +use std::str::FromStr; use std::sync::Arc; use anyhow::anyhow; @@ -338,6 +339,13 @@ impl Catalog for GlueCatalog { namespace: &NamespaceIdent, properties: HashMap, ) -> Result { + if self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceAlreadyExists, + format!("Namespace {namespace:?} already exists"), + )); + } + let db_input = convert_to_database(namespace, &properties)?; let builder = self.client.0.create_database().database_input(db_input); @@ -364,7 +372,19 @@ impl Catalog for GlueCatalog { let builder = self.client.0.get_database().name(&db_name); let builder = with_catalog_id!(builder, self.config); - let resp = builder.send().await.map_err(from_aws_sdk_error)?; + let resp = builder.send().await.map_err(|err| { + if err + .as_service_error() + .map(|e| e.is_entity_not_found_exception()) + == Some(true) + { + return Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {namespace:?} does not exist"), + ); + } + from_aws_sdk_error(err) + })?; match resp.database() { Some(db) => { @@ -372,7 +392,7 @@ impl Catalog for GlueCatalog { Ok(namespace) } None => Err(Error::new( - ErrorKind::DataInvalid, + ErrorKind::NamespaceNotFound, format!("Database with name: {db_name} does not exist"), )), } @@ -428,6 +448,13 @@ impl Catalog for GlueCatalog { namespace: &NamespaceIdent, properties: HashMap, ) -> Result<()> { + if !self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {namespace:?} does not exist"), + )); + } + let db_name = validate_namespace(namespace)?; let db_input = convert_to_database(namespace, &properties)?; @@ -455,6 +482,13 @@ impl Catalog for GlueCatalog { /// - `Err(...)` signifies failure to drop the namespace due to validation /// errors, connectivity issues, or Glue Catalog constraints. async fn drop_namespace(&self, namespace: &NamespaceIdent) -> Result<()> { + if !self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {namespace:?} does not exist"), + )); + } + let db_name = validate_namespace(namespace)?; let table_list = self.list_tables(namespace).await?; @@ -550,14 +584,14 @@ impl Catalog for GlueCatalog { let metadata = TableMetadataBuilder::from_table_creation(creation)? .build()? .metadata; - let metadata_location = - MetadataLocation::new_with_table_location(location.clone()).to_string(); + let metadata_location = MetadataLocation::new_with_metadata(location.clone(), &metadata); metadata.write_to(&self.file_io, &metadata_location).await?; + let metadata_location_str = metadata_location.to_string(); let glue_table = convert_to_glue_table( &table_name, - metadata_location.clone(), + metadata_location_str.clone(), &metadata, metadata.properties(), None, @@ -575,7 +609,7 @@ impl Catalog for GlueCatalog { Table::builder() .file_io(self.file_io()) - .metadata_location(metadata_location) + .metadata_location(metadata_location_str) .metadata(metadata) .identifier(TableIdent::new(NamespaceIdent::new(db_name), table_name)) .build() @@ -625,6 +659,17 @@ impl Catalog for GlueCatalog { Ok(()) } + async fn purge_table(&self, table: &TableIdent) -> Result<()> { + let table_info = self.load_table(table).await?; + self.drop_table(table).await?; + iceberg::drop_table_data( + table_info.file_io(), + table_info.metadata(), + table_info.metadata_location(), + ) + .await + } + /// Asynchronously checks the existence of a specified table /// in the database. /// @@ -813,12 +858,13 @@ impl Catalog for GlueCatalog { let current_metadata_location = current_table.metadata_location_result()?.to_string(); let staged_table = commit.apply(current_table)?; - let staged_metadata_location = staged_table.metadata_location_result()?; + let staged_metadata_location_str = staged_table.metadata_location_result()?; + let staged_metadata_location = MetadataLocation::from_str(staged_metadata_location_str)?; // Write new metadata staged_table .metadata() - .write_to(staged_table.file_io(), staged_metadata_location) + .write_to(staged_table.file_io(), &staged_metadata_location) .await?; // Persist staged table to Glue with optimistic locking diff --git a/crates/catalog/glue/src/utils.rs b/crates/catalog/glue/src/utils.rs index f3be58381a..906e6fcc18 100644 --- a/crates/catalog/glue/src/utils.rs +++ b/crates/catalog/glue/src/utils.rs @@ -306,8 +306,6 @@ mod tests { fn test_convert_to_glue_table() -> Result<()> { let table_name = "my_table".to_string(); let location = "s3a://warehouse/hive".to_string(); - let metadata_location = MetadataLocation::new_with_table_location(location).to_string(); - let properties = HashMap::new(); let schema = Schema::builder() .with_schema_id(1) .with_fields(vec![ @@ -316,6 +314,8 @@ mod tests { .build()?; let metadata = create_metadata(schema)?; + let metadata_location = + MetadataLocation::new_with_metadata(location, &metadata).to_string(); let parameters = HashMap::from([ (ICEBERG_FIELD_ID.to_string(), "1".to_string()), @@ -336,8 +336,13 @@ mod tests { .location(metadata.location()) .build(); - let result = - convert_to_glue_table(&table_name, metadata_location, &metadata, &properties, None)?; + let result = convert_to_glue_table( + &table_name, + metadata_location, + &metadata, + metadata.properties(), + None, + )?; assert_eq!(result.name(), &table_name); assert_eq!(result.description(), None); diff --git a/crates/catalog/glue/tests/glue_catalog_test.rs b/crates/catalog/glue/tests/glue_catalog_test.rs deleted file mode 100644 index 0b7dbe9f23..0000000000 --- a/crates/catalog/glue/tests/glue_catalog_test.rs +++ /dev/null @@ -1,499 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Integration tests for glue catalog. -//! -//! These tests assume Docker containers are started externally via `make docker-up`. -//! Each test uses unique namespaces based on module path to avoid conflicts. - -use std::collections::HashMap; -use std::sync::Arc; - -use iceberg::io::{FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY}; -use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; -use iceberg::transaction::{ApplyTransactionAction, Transaction}; -use iceberg::{ - Catalog, CatalogBuilder, Namespace, NamespaceIdent, Result, TableCreation, TableIdent, -}; -use iceberg_catalog_glue::{ - AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, GLUE_CATALOG_PROP_URI, - GLUE_CATALOG_PROP_WAREHOUSE, GlueCatalog, GlueCatalogBuilder, -}; -use iceberg_storage_opendal::OpenDalStorageFactory; -use iceberg_test_utils::{ - cleanup_namespace, get_glue_endpoint, get_minio_endpoint, normalize_test_name_with_parts, - set_up, -}; -use tokio::time::sleep; -use tracing::info; - -async fn get_catalog() -> GlueCatalog { - set_up(); - - let glue_endpoint = get_glue_endpoint(); - let minio_endpoint = get_minio_endpoint(); - - let props = HashMap::from([ - (AWS_ACCESS_KEY_ID.to_string(), "my_access_id".to_string()), - ( - AWS_SECRET_ACCESS_KEY.to_string(), - "my_secret_key".to_string(), - ), - (AWS_REGION_NAME.to_string(), "us-east-1".to_string()), - (S3_ENDPOINT.to_string(), minio_endpoint), - (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), - (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), - (S3_REGION.to_string(), "us-east-1".to_string()), - ]); - - // Wait for bucket to actually exist - let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { - configured_scheme: "s3a".to_string(), - customized_credential_load: None, - })) - .with_props(props.clone()) - .build(); - - let mut retries = 0; - while retries < 30 { - if file_io.exists("s3a://warehouse/").await.unwrap_or(false) { - info!("S3 bucket 'warehouse' is ready"); - break; - } - info!("Waiting for bucket creation... (attempt {})", retries + 1); - sleep(std::time::Duration::from_millis(1000)).await; - retries += 1; - } - - let mut glue_props = HashMap::from([ - (GLUE_CATALOG_PROP_URI.to_string(), glue_endpoint), - ( - GLUE_CATALOG_PROP_WAREHOUSE.to_string(), - "s3a://warehouse/hive".to_string(), - ), - ]); - glue_props.extend(props.clone()); - - GlueCatalogBuilder::default() - .load("glue", glue_props) - .await - .unwrap() -} - -async fn set_test_namespace(catalog: &GlueCatalog, namespace: &NamespaceIdent) -> Result<()> { - let properties = HashMap::new(); - catalog.create_namespace(namespace, properties).await?; - - Ok(()) -} - -fn set_table_creation(location: Option, name: impl ToString) -> Result { - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "foo", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "bar", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - - let builder = TableCreation::builder() - .name(name.to_string()) - .properties(HashMap::new()) - .location_opt(location) - .schema(schema); - - Ok(builder.build()) -} - -#[tokio::test] -async fn test_rename_table() -> Result<()> { - let catalog = get_catalog().await; - let creation = set_table_creation(None, "my_table")?; - // Use unique namespace to avoid conflicts - let namespace = Namespace::new(NamespaceIdent::new(normalize_test_name_with_parts!( - "test_rename_table" - ))); - cleanup_namespace(&catalog, namespace.name()).await; - - catalog - .create_namespace(namespace.name(), HashMap::new()) - .await?; - - let table = catalog.create_table(namespace.name(), creation).await?; - - let dest = TableIdent::new(namespace.name().clone(), "my_table_rename".to_string()); - - catalog.rename_table(table.identifier(), &dest).await?; - - let table = catalog.load_table(&dest).await?; - assert_eq!(table.identifier(), &dest); - - let src = TableIdent::new(namespace.name().clone(), "my_table".to_string()); - - let src_table_exists = catalog.table_exists(&src).await?; - assert!(!src_table_exists); - - Ok(()) -} - -#[tokio::test] -async fn test_table_exists() -> Result<()> { - let catalog = get_catalog().await; - let creation = set_table_creation(None, "my_table")?; - // Use unique namespace to avoid conflicts - let namespace = Namespace::new(NamespaceIdent::new(normalize_test_name_with_parts!( - "test_table_exists" - ))); - cleanup_namespace(&catalog, namespace.name()).await; - - catalog - .create_namespace(namespace.name(), HashMap::new()) - .await?; - - let ident = TableIdent::new(namespace.name().clone(), "my_table".to_string()); - - let exists = catalog.table_exists(&ident).await?; - assert!(!exists); - - let table = catalog.create_table(namespace.name(), creation).await?; - - let exists = catalog.table_exists(table.identifier()).await?; - - assert!(exists); - - Ok(()) -} - -#[tokio::test] -async fn test_drop_table() -> Result<()> { - let catalog = get_catalog().await; - let creation = set_table_creation(None, "my_table")?; - // Use unique namespace to avoid conflicts - let namespace = Namespace::new(NamespaceIdent::new(normalize_test_name_with_parts!( - "test_drop_table" - ))); - cleanup_namespace(&catalog, namespace.name()).await; - - catalog - .create_namespace(namespace.name(), HashMap::new()) - .await?; - - let table = catalog.create_table(namespace.name(), creation).await?; - - catalog.drop_table(table.identifier()).await?; - - let result = catalog.table_exists(table.identifier()).await?; - - assert!(!result); - - Ok(()) -} - -#[tokio::test] -async fn test_load_table() -> Result<()> { - let catalog = get_catalog().await; - let creation = set_table_creation(None, "my_table")?; - // Use unique namespace to avoid conflicts - let namespace = Namespace::new(NamespaceIdent::new(normalize_test_name_with_parts!( - "test_load_table" - ))); - cleanup_namespace(&catalog, namespace.name()).await; - - catalog - .create_namespace(namespace.name(), HashMap::new()) - .await?; - - let expected = catalog.create_table(namespace.name(), creation).await?; - - let result = catalog - .load_table(&TableIdent::new( - namespace.name().clone(), - "my_table".to_string(), - )) - .await?; - - assert_eq!(result.identifier(), expected.identifier()); - assert_eq!(result.metadata_location(), expected.metadata_location()); - assert_eq!(result.metadata(), expected.metadata()); - - Ok(()) -} - -#[tokio::test] -async fn test_create_table() -> Result<()> { - let catalog = get_catalog().await; - // Use unique namespace to avoid conflicts - let namespace = NamespaceIdent::new(normalize_test_name_with_parts!("test_create_table")); - cleanup_namespace(&catalog, &namespace).await; - set_test_namespace(&catalog, &namespace).await?; - // inject custom location, ignore the namespace prefix - let creation = set_table_creation(Some("s3a://warehouse/hive".into()), "my_table")?; - let result = catalog.create_table(&namespace, creation).await?; - - assert_eq!(result.identifier().name(), "my_table"); - assert!( - result - .metadata_location() - .is_some_and(|location| location.starts_with("s3a://warehouse/hive/metadata/00000-")) - ); - assert!( - catalog - .file_io() - .exists("s3a://warehouse/hive/metadata/") - .await? - ); - - Ok(()) -} - -#[tokio::test] -async fn test_list_tables() -> Result<()> { - let catalog = get_catalog().await; - // Use unique namespace to avoid conflicts - let namespace = NamespaceIdent::new(normalize_test_name_with_parts!("test_list_tables")); - cleanup_namespace(&catalog, &namespace).await; - set_test_namespace(&catalog, &namespace).await?; - - let expected = vec![]; - let result = catalog.list_tables(&namespace).await?; - - assert_eq!(result, expected); - - Ok(()) -} - -#[tokio::test] -async fn test_drop_namespace() -> Result<()> { - let catalog = get_catalog().await; - // Use unique namespace to avoid conflicts - let namespace = NamespaceIdent::new(normalize_test_name_with_parts!("test_drop_namespace")); - cleanup_namespace(&catalog, &namespace).await; - set_test_namespace(&catalog, &namespace).await?; - - let exists = catalog.namespace_exists(&namespace).await?; - assert!(exists); - - catalog.drop_namespace(&namespace).await?; - - let exists = catalog.namespace_exists(&namespace).await?; - assert!(!exists); - - Ok(()) -} - -#[tokio::test] -async fn test_update_namespace() -> Result<()> { - let catalog = get_catalog().await; - // Use unique namespace to avoid conflicts - let namespace = NamespaceIdent::new(normalize_test_name_with_parts!("test_update_namespace")); - cleanup_namespace(&catalog, &namespace).await; - set_test_namespace(&catalog, &namespace).await?; - - let before_update = catalog.get_namespace(&namespace).await?; - let before_update = before_update.properties().get("description"); - - assert_eq!(before_update, None); - - let properties = HashMap::from([("description".to_string(), "my_update".to_string())]); - - catalog.update_namespace(&namespace, properties).await?; - - let after_update = catalog.get_namespace(&namespace).await?; - let after_update = after_update.properties().get("description"); - - assert_eq!(after_update, Some("my_update".to_string()).as_ref()); - - Ok(()) -} - -#[tokio::test] -async fn test_namespace_exists() -> Result<()> { - let catalog = get_catalog().await; - - // Use unique namespace to avoid conflicts - let namespace = NamespaceIdent::new(normalize_test_name_with_parts!("test_namespace_exists")); - cleanup_namespace(&catalog, &namespace).await; - - let exists = catalog.namespace_exists(&namespace).await?; - assert!(!exists); - - set_test_namespace(&catalog, &namespace).await?; - - let exists = catalog.namespace_exists(&namespace).await?; - assert!(exists); - - Ok(()) -} - -#[tokio::test] -async fn test_get_namespace() -> Result<()> { - let catalog = get_catalog().await; - - // Use unique namespace to avoid conflicts - let namespace = NamespaceIdent::new(normalize_test_name_with_parts!("test_get_namespace")); - cleanup_namespace(&catalog, &namespace).await; - - let does_not_exist = catalog.get_namespace(&namespace).await; - assert!(does_not_exist.is_err()); - - set_test_namespace(&catalog, &namespace).await?; - - let result = catalog.get_namespace(&namespace).await?; - let expected = Namespace::new(namespace); - - assert_eq!(result, expected); - - Ok(()) -} - -#[tokio::test] -async fn test_create_namespace() -> Result<()> { - let catalog = get_catalog().await; - - let properties = HashMap::new(); - // Use unique namespace to avoid conflicts - let namespace = NamespaceIdent::new(normalize_test_name_with_parts!("test_create_namespace")); - cleanup_namespace(&catalog, &namespace).await; - - let expected = Namespace::new(namespace.clone()); - - let result = catalog.create_namespace(&namespace, properties).await?; - - assert_eq!(result, expected); - - Ok(()) -} - -#[tokio::test] -async fn test_list_namespace() -> Result<()> { - let catalog = get_catalog().await; - - // Use unique namespace to avoid conflicts - let namespace = NamespaceIdent::new(normalize_test_name_with_parts!("test_list_namespace")); - cleanup_namespace(&catalog, &namespace).await; - set_test_namespace(&catalog, &namespace).await?; - - let result = catalog.list_namespaces(None).await?; - assert!(result.contains(&namespace)); - - let empty_result = catalog.list_namespaces(Some(&namespace)).await?; - assert!(empty_result.is_empty()); - - Ok(()) -} - -#[tokio::test] -async fn test_update_table() -> Result<()> { - let catalog = get_catalog().await; - let creation = set_table_creation(None, "my_table")?; - // Use unique namespace to avoid conflicts - let namespace = Namespace::new(NamespaceIdent::new(normalize_test_name_with_parts!( - "test_update_table" - ))); - cleanup_namespace(&catalog, namespace.name()).await; - - catalog - .create_namespace(namespace.name(), HashMap::new()) - .await?; - - let expected = catalog.create_table(namespace.name(), creation).await?; - - let table = catalog - .load_table(&TableIdent::new( - namespace.name().clone(), - "my_table".to_string(), - )) - .await?; - - assert_eq!(table.identifier(), expected.identifier()); - assert_eq!(table.metadata_location(), expected.metadata_location()); - assert_eq!(table.metadata(), expected.metadata()); - - // Store the original metadata location for comparison - let original_metadata_location = table.metadata_location(); - - // Update table properties using the transaction - let tx = Transaction::new(&table); - let tx = tx - .update_table_properties() - .set("test_property".to_string(), "test_value".to_string()) - .apply(tx)?; - - // Commit the transaction to the catalog - let updated_table = tx.commit(&catalog).await?; - - // Verify the update was successful - assert_eq!( - updated_table.metadata().properties().get("test_property"), - Some(&"test_value".to_string()) - ); - - // Verify the metadata location has been updated - assert_ne!( - updated_table.metadata_location(), - original_metadata_location, - "Metadata location should be updated after commit" - ); - - // Load the table again from the catalog to verify changes were persisted - let reloaded_table = catalog.load_table(table.identifier()).await?; - - // Verify the reloaded table matches the updated table - assert_eq!( - reloaded_table.metadata().properties().get("test_property"), - Some(&"test_value".to_string()) - ); - assert_eq!( - reloaded_table.metadata_location(), - updated_table.metadata_location(), - "Reloaded table should have the same metadata location as the updated table" - ); - - Ok(()) -} - -#[tokio::test] -async fn test_register_table() -> Result<()> { - let catalog = get_catalog().await; - // Use unique namespace to avoid conflicts - let namespace = NamespaceIdent::new(normalize_test_name_with_parts!("test_register_table")); - cleanup_namespace(&catalog, &namespace).await; - set_test_namespace(&catalog, &namespace).await?; - - let location = format!("s3a://warehouse/hive/{namespace}"); - let creation = set_table_creation(Some(location), "my_table")?; - let table = catalog.create_table(&namespace, creation).await?; - let metadata_location = table - .metadata_location() - .expect("Expected metadata location to be set") - .to_string(); - - catalog.drop_table(table.identifier()).await?; - let ident = TableIdent::new(namespace.clone(), "my_table".to_string()); - - let registered = catalog - .register_table(&ident, metadata_location.clone()) - .await?; - - assert_eq!(registered.identifier(), &ident); - assert_eq!( - registered.metadata_location(), - Some(metadata_location.as_str()) - ); - - Ok(()) -} diff --git a/crates/catalog/hms/src/catalog.rs b/crates/catalog/hms/src/catalog.rs index c508b9b1c7..4a030c1104 100644 --- a/crates/catalog/hms/src/catalog.rs +++ b/crates/catalog/hms/src/catalog.rs @@ -279,6 +279,12 @@ impl Catalog for HmsCatalog { namespace: &NamespaceIdent, properties: HashMap, ) -> Result { + if self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceAlreadyExists, + format!("Namespace {namespace:?} already exists"), + )); + } let database = convert_to_database(namespace, &properties)?; self.client @@ -303,13 +309,29 @@ impl Catalog for HmsCatalog { async fn get_namespace(&self, namespace: &NamespaceIdent) -> Result { let name = validate_namespace(namespace)?; - let db = self + let resp = self .client .0 .get_database(name.into()) .await - .map(from_thrift_exception) - .map_err(from_thrift_error)??; + .map_err(from_thrift_error)?; + + let db = match resp { + MaybeException::Ok(db) => db, + MaybeException::Exception(ThriftHiveMetastoreGetDatabaseException::O1(_)) => { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {namespace:?} not found"), + )); + } + MaybeException::Exception(exception) => { + return Err(Error::new( + ErrorKind::Unexpected, + "Operation failed for hitting thrift error".to_string(), + ) + .with_source(anyhow!("thrift error: {exception:?}"))); + } + }; let ns = convert_to_namespace(&db)?; @@ -362,6 +384,12 @@ impl Catalog for HmsCatalog { namespace: &NamespaceIdent, properties: HashMap, ) -> Result<()> { + if !self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {namespace:?} does not exist"), + )); + } let db = convert_to_database(namespace, &properties)?; let name = match &db.name { @@ -393,6 +421,13 @@ impl Catalog for HmsCatalog { async fn drop_namespace(&self, namespace: &NamespaceIdent) -> Result<()> { let name = validate_namespace(namespace)?; + if !self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {namespace:?} does not exist"), + )); + } + self.client .0 .drop_database(name.into(), false, false) @@ -413,6 +448,12 @@ impl Catalog for HmsCatalog { /// querying the database. async fn list_tables(&self, namespace: &NamespaceIdent) -> Result> { let name = validate_namespace(namespace)?; + if !self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {namespace:?} does not exist"), + )); + } let tables = self .client @@ -463,17 +504,17 @@ impl Catalog for HmsCatalog { .build()? .metadata; - let metadata_location = - MetadataLocation::new_with_table_location(location.clone()).to_string(); + let metadata_location = MetadataLocation::new_with_metadata(location.clone(), &metadata); metadata.write_to(&self.file_io, &metadata_location).await?; + let metadata_location_str = metadata_location.to_string(); let hive_table = convert_to_hive_table( db_name.clone(), metadata.current_schema(), table_name.clone(), location, - metadata_location.clone(), + metadata_location_str.clone(), metadata.properties(), )?; @@ -485,7 +526,7 @@ impl Catalog for HmsCatalog { Table::builder() .file_io(self.file_io()) - .metadata_location(metadata_location) + .metadata_location(metadata_location_str) .metadata(metadata) .identifier(TableIdent::new(NamespaceIdent::new(db_name), table_name)) .build() @@ -541,6 +582,18 @@ impl Catalog for HmsCatalog { /// - Any network or communication error occurs with the database backend. async fn drop_table(&self, table: &TableIdent) -> Result<()> { let db_name = validate_namespace(table.namespace())?; + if !self.namespace_exists(table.namespace()).await? { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {:?} does not exist", table.namespace()), + )); + } + if !self.table_exists(table).await? { + return Err(Error::new( + ErrorKind::TableNotFound, + format!("Table {table:?} does not exist"), + )); + } self.client .0 @@ -551,6 +604,17 @@ impl Catalog for HmsCatalog { Ok(()) } + async fn purge_table(&self, table: &TableIdent) -> Result<()> { + let table_info = self.load_table(table).await?; + self.drop_table(table).await?; + iceberg::drop_table_data( + table_info.file_io(), + table_info.metadata(), + table_info.metadata_location(), + ) + .await + } + /// Asynchronously checks the existence of a specified table /// in the database. /// @@ -589,6 +653,12 @@ impl Catalog for HmsCatalog { async fn rename_table(&self, src: &TableIdent, dest: &TableIdent) -> Result<()> { let src_dbname = validate_namespace(src.namespace())?; let dest_dbname = validate_namespace(dest.namespace())?; + if self.table_exists(dest).await? { + return Err(Error::new( + ErrorKind::TableAlreadyExists, + format!("Destination table {dest:?} already exists"), + )); + } let src_tbl_name = src.name.clone(); let dest_tbl_name = dest.name.clone(); diff --git a/crates/catalog/hms/src/utils.rs b/crates/catalog/hms/src/utils.rs index 096e792f61..cd9b557397 100644 --- a/crates/catalog/hms/src/utils.rs +++ b/crates/catalog/hms/src/utils.rs @@ -311,8 +311,8 @@ fn get_current_time() -> Result { #[cfg(test)] mod tests { - use iceberg::spec::{NestedField, PrimitiveType, Type}; - use iceberg::{MetadataLocation, Namespace, NamespaceIdent}; + use iceberg::spec::{NestedField, PrimitiveType, TableMetadataBuilder, Type}; + use iceberg::{MetadataLocation, Namespace, NamespaceIdent, TableCreation}; use super::*; @@ -343,8 +343,6 @@ mod tests { let db_name = "my_db".to_string(); let table_name = "my_table".to_string(); let location = "s3a://warehouse/hms".to_string(); - let metadata_location = - MetadataLocation::new_with_table_location(location.clone()).to_string(); let properties = HashMap::new(); let schema = Schema::builder() .with_schema_id(1) @@ -354,6 +352,18 @@ mod tests { ]) .build()?; + let table_creation = TableCreation::builder() + .name(table_name.clone()) + .location(location.clone()) + .schema(schema.clone()) + .properties(properties.clone()) + .build(); + let metadata = TableMetadataBuilder::from_table_creation(table_creation)? + .build()? + .metadata; + let metadata_location = + MetadataLocation::new_with_metadata(location.clone(), &metadata).to_string(); + let result = convert_to_hive_table( db_name.clone(), &schema, diff --git a/crates/catalog/hms/tests/hms_catalog_test.rs b/crates/catalog/hms/tests/hms_catalog_test.rs index 74c9e52e92..f19cf7bff4 100644 --- a/crates/catalog/hms/tests/hms_catalog_test.rs +++ b/crates/catalog/hms/tests/hms_catalog_test.rs @@ -24,16 +24,13 @@ use std::collections::HashMap; use std::sync::Arc; use iceberg::io::{FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY}; -use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; -use iceberg::{Catalog, CatalogBuilder, Namespace, NamespaceIdent, TableCreation, TableIdent}; +use iceberg::{Catalog, CatalogBuilder, Namespace, NamespaceIdent}; use iceberg_catalog_hms::{ HMS_CATALOG_PROP_THRIFT_TRANSPORT, HMS_CATALOG_PROP_URI, HMS_CATALOG_PROP_WAREHOUSE, HmsCatalog, HmsCatalogBuilder, THRIFT_TRANSPORT_BUFFERED, }; use iceberg_storage_opendal::OpenDalStorageFactory; -use iceberg_test_utils::{ - cleanup_namespace, get_hms_endpoint, get_minio_endpoint, normalize_test_name_with_parts, set_up, -}; +use iceberg_test_utils::{get_hms_endpoint, get_minio_endpoint, set_up}; use tokio::time::sleep; use tracing::info; @@ -90,230 +87,6 @@ async fn get_catalog() -> HmsCatalog { .unwrap() } -async fn set_test_namespace(catalog: &HmsCatalog, namespace: &NamespaceIdent) -> Result<()> { - let properties = HashMap::new(); - - catalog.create_namespace(namespace, properties).await?; - - Ok(()) -} - -fn set_table_creation(location: Option, name: impl ToString) -> Result { - let schema = Schema::builder() - .with_schema_id(0) - .with_fields(vec![ - NestedField::required(1, "foo", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::required(2, "bar", Type::Primitive(PrimitiveType::String)).into(), - ]) - .build()?; - - let builder = TableCreation::builder() - .name(name.to_string()) - .properties(HashMap::new()) - .location_opt(location) - .schema(schema); - - Ok(builder.build()) -} - -#[tokio::test] -async fn test_rename_table() -> Result<()> { - let catalog = get_catalog().await; - let creation: TableCreation = set_table_creation(None, "my_table")?; - // Use unique namespace to avoid conflicts - let namespace = Namespace::new(NamespaceIdent::new(normalize_test_name_with_parts!( - "test_rename_table" - ))); - // Clean up from any previous test runs - cleanup_namespace(&catalog, namespace.name()).await; - set_test_namespace(&catalog, namespace.name()).await?; - - let table: iceberg::table::Table = catalog.create_table(namespace.name(), creation).await?; - - let dest = TableIdent::new(namespace.name().clone(), "my_table_rename".to_string()); - - catalog.rename_table(table.identifier(), &dest).await?; - - let result = catalog.table_exists(&dest).await?; - - assert!(result); - - Ok(()) -} - -#[tokio::test] -async fn test_table_exists() -> Result<()> { - let catalog = get_catalog().await; - let creation = set_table_creation(None, "my_table")?; - // Use unique namespace to avoid conflicts - let namespace = Namespace::new(NamespaceIdent::new(normalize_test_name_with_parts!( - "test_table_exists" - ))); - cleanup_namespace(&catalog, namespace.name()).await; - set_test_namespace(&catalog, namespace.name()).await?; - - let table = catalog.create_table(namespace.name(), creation).await?; - - let result = catalog.table_exists(table.identifier()).await?; - - assert!(result); - - Ok(()) -} - -#[tokio::test] -async fn test_drop_table() -> Result<()> { - let catalog = get_catalog().await; - let creation = set_table_creation(None, "my_table")?; - // Use unique namespace to avoid conflicts - let namespace = Namespace::new(NamespaceIdent::new(normalize_test_name_with_parts!( - "test_drop_table" - ))); - cleanup_namespace(&catalog, namespace.name()).await; - set_test_namespace(&catalog, namespace.name()).await?; - - let table = catalog.create_table(namespace.name(), creation).await?; - - catalog.drop_table(table.identifier()).await?; - - let result = catalog.table_exists(table.identifier()).await?; - - assert!(!result); - - Ok(()) -} - -#[tokio::test] -async fn test_load_table() -> Result<()> { - let catalog = get_catalog().await; - let creation = set_table_creation(None, "my_table")?; - // Use unique namespace to avoid conflicts - let namespace = Namespace::new(NamespaceIdent::new(normalize_test_name_with_parts!( - "test_load_table" - ))); - cleanup_namespace(&catalog, namespace.name()).await; - set_test_namespace(&catalog, namespace.name()).await?; - - let expected = catalog.create_table(namespace.name(), creation).await?; - - let result = catalog - .load_table(&TableIdent::new( - namespace.name().clone(), - "my_table".to_string(), - )) - .await?; - - assert_eq!(result.identifier(), expected.identifier()); - assert_eq!(result.metadata_location(), expected.metadata_location()); - assert_eq!(result.metadata(), expected.metadata()); - - Ok(()) -} - -#[tokio::test] -async fn test_create_table() -> Result<()> { - let catalog = get_catalog().await; - // inject custom location, ignore the namespace prefix - let creation = set_table_creation(Some("s3a://warehouse/hive".into()), "my_table")?; - // Use unique namespace to avoid conflicts - let namespace = Namespace::new(NamespaceIdent::new(normalize_test_name_with_parts!( - "test_create_table" - ))); - cleanup_namespace(&catalog, namespace.name()).await; - set_test_namespace(&catalog, namespace.name()).await?; - - let result = catalog.create_table(namespace.name(), creation).await?; - - assert_eq!(result.identifier().name(), "my_table"); - assert!( - result - .metadata_location() - .is_some_and(|location| location.starts_with("s3a://warehouse/hive/metadata/00000-")) - ); - assert!( - catalog - .file_io() - .exists("s3a://warehouse/hive/metadata/") - .await? - ); - - Ok(()) -} - -#[tokio::test] -async fn test_list_tables() -> Result<()> { - let catalog = get_catalog().await; - // Use unique namespace to avoid conflicts - let ns = Namespace::new(NamespaceIdent::new(normalize_test_name_with_parts!( - "test_list_tables" - ))); - // Clean up and create namespace, then verify it's empty - cleanup_namespace(&catalog, ns.name()).await; - set_test_namespace(&catalog, ns.name()).await?; - let result = catalog.list_tables(ns.name()).await?; - - assert_eq!(result, vec![]); - - let creation = set_table_creation(None, "my_table")?; - catalog.create_table(ns.name(), creation).await?; - let result = catalog.list_tables(ns.name()).await?; - - assert_eq!(result, vec![TableIdent::new( - ns.name().clone(), - "my_table".to_string() - )]); - - Ok(()) -} - -#[tokio::test] -async fn test_list_namespace() -> Result<()> { - let catalog = get_catalog().await; - - let result_no_parent = catalog.list_namespaces(None).await?; - - let result_with_parent = catalog - .list_namespaces(Some(&NamespaceIdent::new("parent".into()))) - .await?; - - assert!(result_no_parent.contains(&NamespaceIdent::new("default".into()))); - assert!(result_with_parent.is_empty()); - - Ok(()) -} - -#[tokio::test] -async fn test_create_namespace() -> Result<()> { - let catalog = get_catalog().await; - - let properties = HashMap::from([ - ("comment".to_string(), "my_description".to_string()), - ("location".to_string(), "my_location".to_string()), - ( - "hive.metastore.database.owner".to_string(), - "apache".to_string(), - ), - ( - "hive.metastore.database.owner-type".to_string(), - "user".to_string(), - ), - ("key1".to_string(), "value1".to_string()), - ]); - - // Use unique namespace to avoid conflicts - let ns = Namespace::with_properties( - NamespaceIdent::new(normalize_test_name_with_parts!("test_create_namespace")), - properties.clone(), - ); - cleanup_namespace(&catalog, ns.name()).await; - - let result = catalog.create_namespace(ns.name(), properties).await?; - - assert_eq!(result, ns); - - Ok(()) -} - #[tokio::test] async fn test_get_default_namespace() -> Result<()> { let catalog = get_catalog().await; @@ -340,68 +113,3 @@ async fn test_get_default_namespace() -> Result<()> { Ok(()) } - -#[tokio::test] -async fn test_namespace_exists() -> Result<()> { - let catalog = get_catalog().await; - - let ns_exists = Namespace::new(NamespaceIdent::new("default".into())); - // Use unique namespace to ensure it doesn't exist - let ns_not_exists = Namespace::new(NamespaceIdent::new(normalize_test_name_with_parts!( - "test_namespace_exists" - ))); - cleanup_namespace(&catalog, ns_not_exists.name()).await; - - let result_exists = catalog.namespace_exists(ns_exists.name()).await?; - let result_not_exists = catalog.namespace_exists(ns_not_exists.name()).await?; - - assert!(result_exists); - assert!(!result_not_exists); - - Ok(()) -} - -#[tokio::test] -async fn test_update_namespace() -> Result<()> { - let catalog = get_catalog().await; - - // Use unique namespace to avoid conflicts - let ns = NamespaceIdent::new(normalize_test_name_with_parts!("test_update_namespace")); - cleanup_namespace(&catalog, &ns).await; - set_test_namespace(&catalog, &ns).await?; - let properties = HashMap::from([("comment".to_string(), "my_update".to_string())]); - - catalog.update_namespace(&ns, properties).await?; - - let db = catalog.get_namespace(&ns).await?; - - assert_eq!( - db.properties().get("comment"), - Some(&"my_update".to_string()) - ); - - Ok(()) -} - -#[tokio::test] -async fn test_drop_namespace() -> Result<()> { - let catalog = get_catalog().await; - - // Use unique namespace to avoid conflicts - let ns = Namespace::new(NamespaceIdent::new(normalize_test_name_with_parts!( - "test_drop_namespace" - ))); - cleanup_namespace(&catalog, ns.name()).await; - - catalog.create_namespace(ns.name(), HashMap::new()).await?; - - let result = catalog.namespace_exists(ns.name()).await?; - assert!(result); - - catalog.drop_namespace(ns.name()).await?; - - let result = catalog.namespace_exists(ns.name()).await?; - assert!(!result); - - Ok(()) -} diff --git a/crates/catalog/loader/Cargo.toml b/crates/catalog/loader/Cargo.toml index d4b925fb94..20daf84f50 100644 --- a/crates/catalog/loader/Cargo.toml +++ b/crates/catalog/loader/Cargo.toml @@ -39,5 +39,9 @@ tokio = { workspace = true } async-trait = { workspace = true } [dev-dependencies] +iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } +iceberg-storage-opendal = { workspace = true } +reqwest = { workspace = true } +rstest = { workspace = true } sqlx = { workspace = true, features = ["runtime-tokio", "sqlite", "migrate"] } tempfile = { workspace = true } diff --git a/crates/catalog/loader/tests/common/mod.rs b/crates/catalog/loader/tests/common/mod.rs new file mode 100644 index 0000000000..6524d56339 --- /dev/null +++ b/crates/catalog/loader/tests/common/mod.rs @@ -0,0 +1,355 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared helpers for catalog integration suites. + +#![allow(dead_code)] + +use std::collections::HashMap; +use std::fmt; +use std::sync::Arc; + +use iceberg::io::{ + FileIOBuilder, LocalFsStorageFactory, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, + S3_SECRET_ACCESS_KEY, +}; +use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; +use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; +use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableCreation}; +use iceberg_catalog_glue::{ + AWS_ACCESS_KEY_ID, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY, GLUE_CATALOG_PROP_URI, + GLUE_CATALOG_PROP_WAREHOUSE, GlueCatalog, GlueCatalogBuilder, +}; +use iceberg_catalog_hms::{ + HMS_CATALOG_PROP_THRIFT_TRANSPORT, HMS_CATALOG_PROP_URI, HMS_CATALOG_PROP_WAREHOUSE, + HmsCatalog, HmsCatalogBuilder, THRIFT_TRANSPORT_BUFFERED, +}; +use iceberg_catalog_rest::{ + CustomAuthenticator, REST_CATALOG_PROP_URI, RestCatalog, RestCatalogBuilder, +}; +use iceberg_catalog_s3tables::{ + S3TABLES_CATALOG_PROP_ENDPOINT_URL, S3TABLES_CATALOG_PROP_TABLE_BUCKET_ARN, + S3TablesCatalogBuilder, +}; +use iceberg_catalog_sql::{ + SQL_CATALOG_PROP_BIND_STYLE, SQL_CATALOG_PROP_URI, SQL_CATALOG_PROP_WAREHOUSE, SqlBindStyle, + SqlCatalogBuilder, +}; +use iceberg_storage_opendal::OpenDalStorageFactory; +use iceberg_test_utils::{ + get_glue_endpoint, get_hms_endpoint, get_minio_endpoint, get_rest_catalog_endpoint, set_up, +}; +use sqlx::migrate::MigrateDatabase; +use tempfile::TempDir; +use tokio::time::sleep; + +#[derive(Debug, Clone, Copy)] +pub enum CatalogKind { + Rest, + Glue, + Hms, + Sql, + S3Tables, + Memory, +} + +impl fmt::Display for CatalogKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let name = match self { + CatalogKind::Rest => "rest_catalog", + CatalogKind::Glue => "glue_catalog", + CatalogKind::Hms => "hms_catalog", + CatalogKind::Sql => "sql_catalog", + CatalogKind::S3Tables => "s3tables_catalog", + CatalogKind::Memory => "memory_catalog", + }; + f.write_str(name) + } +} + +pub struct CatalogHarness { + pub catalog: Arc, + pub label: &'static str, + _tempdirs: Vec, +} + +// Shared setup for each catalog implementation so the suites exercise +// the same behavior against all backends. +pub async fn load_catalog(kind: CatalogKind) -> Option { + set_up(); + match kind { + CatalogKind::Rest => Some(CatalogHarness { + catalog: Arc::new(rest_catalog().await) as Arc, + label: "rest", + _tempdirs: Vec::new(), + }), + CatalogKind::Glue => Some(CatalogHarness { + catalog: Arc::new(glue_catalog().await) as Arc, + label: "glue", + _tempdirs: Vec::new(), + }), + CatalogKind::Hms => Some(CatalogHarness { + catalog: Arc::new(hms_catalog().await) as Arc, + label: "hms", + _tempdirs: Vec::new(), + }), + CatalogKind::Sql => { + let warehouse_dir = TempDir::new().unwrap(); + let db_dir = TempDir::new().unwrap(); + let db_path = db_dir.path().join("catalog.db"); + let db_uri = format!("sqlite:{}", db_path.to_str().unwrap()); + sqlx::Sqlite::create_database(&db_uri).await.unwrap(); + + let catalog = SqlCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "sql", + HashMap::from([ + (SQL_CATALOG_PROP_URI.to_string(), db_uri), + ( + SQL_CATALOG_PROP_WAREHOUSE.to_string(), + warehouse_dir.path().to_str().unwrap().to_string(), + ), + ( + SQL_CATALOG_PROP_BIND_STYLE.to_string(), + SqlBindStyle::QMark.to_string(), + ), + ]), + ) + .await + .unwrap(); + + Some(CatalogHarness { + catalog: Arc::new(catalog) as Arc, + label: "sql", + _tempdirs: vec![warehouse_dir, db_dir], + }) + } + CatalogKind::S3Tables => { + let table_bucket_arn = match std::env::var("TABLE_BUCKET_ARN").ok() { + Some(value) => value, + None => return None, + }; + + let mut props = HashMap::from([( + S3TABLES_CATALOG_PROP_TABLE_BUCKET_ARN.to_string(), + table_bucket_arn, + )]); + + if let Ok(endpoint_url) = std::env::var("S3TABLES_ENDPOINT_URL") { + props.insert(S3TABLES_CATALOG_PROP_ENDPOINT_URL.to_string(), endpoint_url); + } + + let catalog = S3TablesCatalogBuilder::default() + .load("s3tables", props) + .await + .unwrap(); + + Some(CatalogHarness { + catalog: Arc::new(catalog) as Arc, + label: "s3tables", + _tempdirs: Vec::new(), + }) + } + CatalogKind::Memory => { + let warehouse_dir = TempDir::new().unwrap(); + let props = HashMap::from([( + MEMORY_CATALOG_WAREHOUSE.to_string(), + warehouse_dir.path().to_str().unwrap().to_string(), + )]); + let catalog = MemoryCatalogBuilder::default() + .load("memory", props) + .await + .unwrap(); + + Some(CatalogHarness { + catalog: Arc::new(catalog) as Arc, + label: "memory", + _tempdirs: vec![warehouse_dir], + }) + } + } +} + +// Catalog-specific setup is intentionally isolated here so the suites +// remain implementation-agnostic. +async fn rest_catalog() -> RestCatalog { + rest_catalog_with_auth(None).await +} + +pub async fn rest_catalog_with_auth( + authenticator: Option>, +) -> RestCatalog { + let rest_endpoint = get_rest_catalog_endpoint(); + + let client = reqwest::Client::new(); + let mut retries = 0; + while retries < 30 { + if client + .get(format!("{rest_endpoint}/v1/config")) + .send() + .await + .map(|resp| resp.status().is_success()) + .unwrap_or(false) + { + break; + } + sleep(std::time::Duration::from_millis(1000)).await; + retries += 1; + } + + let mut builder = + RestCatalogBuilder::default().with_storage_factory(Arc::new(LocalFsStorageFactory)); + if let Some(auth) = authenticator { + builder = builder.with_token_authenticator(auth); + } + + builder + .load( + "rest", + HashMap::from([(REST_CATALOG_PROP_URI.to_string(), rest_endpoint)]), + ) + .await + .unwrap() +} + +async fn glue_catalog() -> GlueCatalog { + let glue_endpoint = get_glue_endpoint(); + let minio_endpoint = get_minio_endpoint(); + + let props = HashMap::from([ + (AWS_ACCESS_KEY_ID.to_string(), "my_access_id".to_string()), + ( + AWS_SECRET_ACCESS_KEY.to_string(), + "my_secret_key".to_string(), + ), + (AWS_REGION_NAME.to_string(), "us-east-1".to_string()), + (S3_ENDPOINT.to_string(), minio_endpoint), + (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), + (S3_REGION.to_string(), "us-east-1".to_string()), + ]); + + let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { + configured_scheme: "s3a".to_string(), + customized_credential_load: None, + })) + .with_props(props.clone()) + .build(); + + let mut retries = 0; + while retries < 30 { + if file_io.exists("s3a://warehouse/").await.unwrap_or(false) { + break; + } + sleep(std::time::Duration::from_millis(1000)).await; + retries += 1; + } + + let mut glue_props = HashMap::from([ + (GLUE_CATALOG_PROP_URI.to_string(), glue_endpoint), + ( + GLUE_CATALOG_PROP_WAREHOUSE.to_string(), + "s3a://warehouse/hive".to_string(), + ), + ]); + glue_props.extend(props); + + GlueCatalogBuilder::default() + .load("glue", glue_props) + .await + .unwrap() +} + +async fn hms_catalog() -> HmsCatalog { + let hms_endpoint = get_hms_endpoint(); + let minio_endpoint = get_minio_endpoint(); + + let props = HashMap::from([ + (HMS_CATALOG_PROP_URI.to_string(), hms_endpoint), + ( + HMS_CATALOG_PROP_THRIFT_TRANSPORT.to_string(), + THRIFT_TRANSPORT_BUFFERED.to_string(), + ), + ( + HMS_CATALOG_PROP_WAREHOUSE.to_string(), + "s3a://warehouse/hive".to_string(), + ), + (S3_ENDPOINT.to_string(), minio_endpoint), + (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), + (S3_REGION.to_string(), "us-east-1".to_string()), + ]); + + let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { + configured_scheme: "s3a".to_string(), + customized_credential_load: None, + })) + .with_props(props.clone()) + .build(); + + let mut retries = 0; + while retries < 30 { + if file_io.exists("s3a://warehouse/").await.unwrap_or(false) { + break; + } + sleep(std::time::Duration::from_millis(1000)).await; + retries += 1; + } + + HmsCatalogBuilder::default() + .with_storage_factory(Arc::new(OpenDalStorageFactory::S3 { + configured_scheme: "s3a".to_string(), + customized_credential_load: None, + })) + .load("hms", props) + .await + .unwrap() +} + +// Common table schema used across suites to validate shared behavior. +pub fn table_creation(name: impl ToString) -> TableCreation { + let schema = Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "foo", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::required(2, "bar", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .unwrap(); + + TableCreation::builder() + .name(name.to_string()) + .properties(HashMap::new()) + .schema(schema) + .build() +} + +pub fn assert_map_contains(expected: &HashMap, actual: &HashMap) { + for (key, value) in expected { + assert_eq!(actual.get(key), Some(value)); + } +} + +pub async fn cleanup_namespace_dyn(catalog: &dyn Catalog, namespace: &NamespaceIdent) { + if let Ok(tables) = catalog.list_tables(namespace).await { + for table in tables { + let _ = catalog.purge_table(&table).await; + } + } + let _ = catalog.drop_namespace(namespace).await; +} diff --git a/crates/catalog/loader/tests/namespace_suite.rs b/crates/catalog/loader/tests/namespace_suite.rs new file mode 100644 index 0000000000..024e8a62b8 --- /dev/null +++ b/crates/catalog/loader/tests/namespace_suite.rs @@ -0,0 +1,355 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Common namespace behavior across catalogs. +//! +//! These tests assume Docker containers are started externally via `make docker-up`. + +mod common; + +use std::collections::HashMap; + +use common::{CatalogKind, assert_map_contains, cleanup_namespace_dyn, load_catalog}; +use iceberg::{ErrorKind, NamespaceIdent, Result}; +use iceberg_test_utils::normalize_test_name_with_parts; +use rstest::rstest; + +// Common behavior: querying a missing namespace should error. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_namespace_missing_returns_error(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_namespace_missing_returns_error", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + + let err = catalog.get_namespace(&namespace).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::NamespaceNotFound); + + Ok(()) +} + +// Common behavior: namespace lifecycle CRUD. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_namespace_lifecycle(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_namespace_lifecycle", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + + assert!(!catalog.namespace_exists(&namespace).await?); + + let props = HashMap::from([ + ("owner".to_string(), "rust".to_string()), + ("purpose".to_string(), "catalog_suite".to_string()), + ]); + let created = catalog.create_namespace(&namespace, props.clone()).await?; + assert_eq!(created.name(), &namespace); + assert_map_contains(&props, created.properties()); + + let fetched = catalog.get_namespace(&namespace).await?; + assert_eq!(fetched.name(), &namespace); + assert_map_contains(&props, fetched.properties()); + + let namespaces = catalog.list_namespaces(None).await?; + assert!(namespaces.contains(&namespace)); + + catalog.drop_namespace(&namespace).await?; + assert!(!catalog.namespace_exists(&namespace).await?); + + Ok(()) +} + +// Common behavior: update_namespace persists changes when supported. +#[rstest] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_update_namespace_supported(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_update_namespace_supported", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let updated_props = HashMap::from([("owner".to_string(), "updated".to_string())]); + catalog + .update_namespace(&namespace, updated_props.clone()) + .await?; + + let updated = catalog.get_namespace(&namespace).await?; + assert_map_contains(&updated_props, updated.properties()); + + Ok(()) +} + +// Common behavior: update_namespace returns FeatureUnsupported when not implemented. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[tokio::test] +async fn test_catalog_update_namespace_unsupported(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_update_namespace_unsupported", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let err = catalog + .update_namespace( + &namespace, + HashMap::from([("key".to_string(), "value".to_string())]), + ) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); + + Ok(()) +} + +// Common behavior: listing namespaces under a parent returns its children. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_namespace_listing_with_parent(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let parent_name = + normalize_test_name_with_parts!("catalog_namespace_listing_with_parent", harness.label); + let parent = NamespaceIdent::new(parent_name.clone()); + let child1 = NamespaceIdent::from_strs([&parent_name, "child1"]).unwrap(); + let child2 = NamespaceIdent::from_strs([&parent_name, "child2"]).unwrap(); + + cleanup_namespace_dyn(catalog.as_ref(), &child1).await; + cleanup_namespace_dyn(catalog.as_ref(), &child2).await; + cleanup_namespace_dyn(catalog.as_ref(), &parent).await; + + catalog.create_namespace(&parent, HashMap::new()).await?; + + catalog.create_namespace(&child1, HashMap::new()).await?; + catalog.create_namespace(&child2, HashMap::new()).await?; + + let top_level = catalog.list_namespaces(None).await?; + assert!(top_level.contains(&parent)); + + let children = catalog.list_namespaces(Some(&parent)).await?; + assert!(children.contains(&child1)); + assert!(children.contains(&child2)); + + Ok(()) +} + +// Common behavior: hierarchical namespaces are rejected when unsupported. +#[rstest] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[tokio::test] +async fn test_catalog_namespace_listing_with_parent_unsupported( + #[case] kind: CatalogKind, +) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let parent_name = normalize_test_name_with_parts!( + "catalog_namespace_listing_with_parent_unsupported", + harness.label + ); + let parent = NamespaceIdent::new(parent_name.clone()); + let child = NamespaceIdent::from_strs([&parent_name, "child"]).unwrap(); + + cleanup_namespace_dyn(catalog.as_ref(), &child).await; + cleanup_namespace_dyn(catalog.as_ref(), &parent).await; + + catalog.create_namespace(&parent, HashMap::new()).await?; + + let err = catalog + .create_namespace(&child, HashMap::new()) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::DataInvalid); + + Ok(()) +} + +// Common behavior: listing top-level namespaces includes created namespaces. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_list_namespaces_contains_created(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let ns_one = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_list_namespaces_contains_created", + harness.label, + "one" + )); + let ns_two = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_list_namespaces_contains_created", + harness.label, + "two" + )); + + cleanup_namespace_dyn(catalog.as_ref(), &ns_one).await; + cleanup_namespace_dyn(catalog.as_ref(), &ns_two).await; + + catalog.create_namespace(&ns_one, HashMap::new()).await?; + catalog.create_namespace(&ns_two, HashMap::new()).await?; + + let namespaces = catalog.list_namespaces(None).await?; + assert!(namespaces.contains(&ns_one)); + assert!(namespaces.contains(&ns_two)); + + Ok(()) +} + +// Common behavior: creating an existing namespace should error. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_create_namespace_duplicate_fails(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_create_namespace_duplicate_fails", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let err = catalog + .create_namespace(&namespace, HashMap::new()) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::NamespaceAlreadyExists); + Ok(()) +} + +// Common behavior: update on a missing namespace should return NamespaceNotFound. +#[rstest] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_update_namespace_missing_errors(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_update_namespace_missing_errors", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + + let err = catalog + .update_namespace( + &namespace, + HashMap::from([("key".to_string(), "value".to_string())]), + ) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::NamespaceNotFound); + + Ok(()) +} + +// Common behavior: dropping a missing namespace should error. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_drop_namespace_missing_errors(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_drop_namespace_missing_errors", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + + let err = catalog.drop_namespace(&namespace).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::NamespaceNotFound); + Ok(()) +} diff --git a/crates/catalog/loader/tests/rest_auth_suite.rs b/crates/catalog/loader/tests/rest_auth_suite.rs new file mode 100644 index 0000000000..1fb09d71ab --- /dev/null +++ b/crates/catalog/loader/tests/rest_auth_suite.rs @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! REST-catalog-specific integration tests for the custom authenticator. +//! +//! These tests assume Docker containers are started externally via `make docker-up`. + +mod common; + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use common::{cleanup_namespace_dyn, rest_catalog_with_auth}; +use iceberg::{Catalog, NamespaceIdent, Result as IcebergResult}; +use iceberg_catalog_rest::CustomAuthenticator; +use iceberg_test_utils::set_up; + +#[derive(Debug)] +struct CountingAuthenticator { + count: Arc>, +} + +#[async_trait] +impl CustomAuthenticator for CountingAuthenticator { + async fn get_token(&self) -> IcebergResult { + let mut c = self.count.lock().unwrap(); + *c += 1; + Ok(format!("token_{}", *c)) + } +} + +#[tokio::test] +async fn test_authenticator_token_refresh() { + set_up(); + + let token_request_count = Arc::new(Mutex::new(0)); + let authenticator = Arc::new(CountingAuthenticator { + count: token_request_count.clone(), + }); + + let catalog = rest_catalog_with_auth(Some(authenticator)).await; + + let ns1_ident = NamespaceIdent::from_strs(["test_refresh_1"]).unwrap(); + let ns2_ident = NamespaceIdent::from_strs(["test_refresh_2"]).unwrap(); + cleanup_namespace_dyn(&catalog, &ns1_ident).await; + cleanup_namespace_dyn(&catalog, &ns2_ident).await; + + catalog + .create_namespace(&ns1_ident, HashMap::new()) + .await + .unwrap(); + catalog + .create_namespace(&ns2_ident, HashMap::new()) + .await + .unwrap(); + + // With lazy authentication, the token is fetched once and cached for reuse + // across multiple operations, rather than being called on every request + let count = *token_request_count.lock().unwrap(); + assert_eq!( + count, 1, + "Authenticator should have been called once for lazy token caching, but was called {count} times" + ); + + // Test that token is refreshed when invalidated + catalog.invalidate_token().await.unwrap(); + + let ns3_ident = NamespaceIdent::from_strs(["test_refresh_3"]).unwrap(); + cleanup_namespace_dyn(&catalog, &ns3_ident).await; + catalog + .create_namespace(&ns3_ident, HashMap::new()) + .await + .unwrap(); + + // After invalidating and making another request, authenticator should be called again + let count = *token_request_count.lock().unwrap(); + assert_eq!( + count, 2, + "Authenticator should have been called twice (once initial, once after invalidation), but was called {count} times" + ); + + cleanup_namespace_dyn(&catalog, &ns1_ident).await; + cleanup_namespace_dyn(&catalog, &ns2_ident).await; + cleanup_namespace_dyn(&catalog, &ns3_ident).await; +} + +#[tokio::test] +async fn test_authenticator_persists_across_operations() { + set_up(); + + let operation_count = Arc::new(Mutex::new(0)); + let authenticator = Arc::new(CountingAuthenticator { + count: operation_count.clone(), + }); + + let catalog = rest_catalog_with_auth(Some(authenticator)).await; + + let ns_ident = NamespaceIdent::from_strs(["test_persist", "auth"]).unwrap(); + let parent_ident = NamespaceIdent::from_strs(["test_persist"]).unwrap(); + cleanup_namespace_dyn(&catalog, &ns_ident).await; + + catalog + .create_namespace(&ns_ident, HashMap::new()) + .await + .unwrap(); + + let count_after_create = *operation_count.lock().unwrap(); + + let list_result = catalog.list_namespaces(Some(&parent_ident)).await.unwrap(); + assert!( + list_result.contains(&ns_ident), + "Namespace {:?} not found in list {:?}", + ns_ident, + list_result + ); + + let count_after_list = *operation_count.lock().unwrap(); + + // With lazy authentication, the token is fetched once on the first operation + // and then reused for subsequent operations without calling the authenticator again + assert_eq!( + count_after_create, 1, + "Authenticator should be called once for the create operation" + ); + assert_eq!( + count_after_list, 1, + "Authenticator should still have been called only once (token is cached and reused for list)" + ); + + cleanup_namespace_dyn(&catalog, &ns_ident).await; +} diff --git a/crates/catalog/loader/tests/table_register_suite.rs b/crates/catalog/loader/tests/table_register_suite.rs new file mode 100644 index 0000000000..b43054b833 --- /dev/null +++ b/crates/catalog/loader/tests/table_register_suite.rs @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Common register-table behavior across catalogs. +//! +//! These tests assume Docker containers are started externally via `make docker-up`. + +mod common; + +use std::collections::HashMap; + +use common::{CatalogKind, cleanup_namespace_dyn, load_catalog, table_creation}; +use iceberg::{ErrorKind, NamespaceIdent, Result, TableIdent}; +use iceberg_test_utils::normalize_test_name_with_parts; +use rstest::rstest; + +// Common behavior: register_table rehydrates a dropped table. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_register_table_roundtrip(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_register_table_roundtrip", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table = catalog + .create_table( + &namespace, + table_creation(normalize_test_name_with_parts!( + "catalog_register_table_roundtrip", + harness.label, + "table" + )), + ) + .await?; + let table_ident = table.identifier().clone(); + let metadata_location = table + .metadata_location() + .ok_or_else(|| iceberg::Error::new(ErrorKind::Unexpected, "Missing metadata location"))? + .to_string(); + + catalog.drop_table(&table_ident).await?; + + let registered = catalog + .register_table(&table_ident, metadata_location.clone()) + .await?; + assert_eq!(registered.identifier(), &table_ident); + assert_eq!( + registered.metadata_location(), + Some(metadata_location.as_str()) + ); + + Ok(()) +} + +// HMS and S3Tables do not support register_table yet. +#[rstest] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[tokio::test] +async fn test_catalog_register_table_unsupported(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_register_table_unsupported", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table = catalog + .create_table( + &namespace, + table_creation(normalize_test_name_with_parts!( + "catalog_register_table_unsupported", + harness.label, + "table" + )), + ) + .await?; + let table_ident = table.identifier().clone(); + let metadata_location = table + .metadata_location() + .ok_or_else(|| iceberg::Error::new(ErrorKind::Unexpected, "Missing metadata location"))? + .to_string(); + + let err = catalog + .register_table(&table_ident, metadata_location) + .await + .unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); + + Ok(()) +} + +// Common behavior: registering a table with an existing name should error. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_register_table_conflict_errors(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_register_table_conflict_errors", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_ident = TableIdent::new( + namespace.clone(), + normalize_test_name_with_parts!( + "catalog_register_table_conflict_errors", + harness.label, + "table" + ), + ); + let table = catalog + .create_table(&namespace, table_creation(table_ident.name.clone())) + .await?; + let metadata_location = table + .metadata_location() + .ok_or_else(|| iceberg::Error::new(ErrorKind::Unexpected, "Missing metadata location"))? + .to_string(); + + assert!( + catalog + .register_table(&table_ident, metadata_location) + .await + .is_err() + ); + Ok(()) +} diff --git a/crates/catalog/loader/tests/table_rename_suite.rs b/crates/catalog/loader/tests/table_rename_suite.rs new file mode 100644 index 0000000000..757d1cd044 --- /dev/null +++ b/crates/catalog/loader/tests/table_rename_suite.rs @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Common rename behavior across catalogs. +//! +//! These tests assume Docker containers are started externally via `make docker-up`. + +mod common; + +use std::collections::HashMap; + +use common::{CatalogKind, cleanup_namespace_dyn, load_catalog, table_creation}; +use iceberg::{NamespaceIdent, Result, TableIdent}; +use iceberg_test_utils::normalize_test_name_with_parts; +use rstest::rstest; + +// Common behavior: renaming across namespaces moves the table. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_rename_table_across_namespaces(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let src_namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_rename_table_across_namespaces", + harness.label, + "src" + )); + let dst_namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_rename_table_across_namespaces", + harness.label, + "dst" + )); + + cleanup_namespace_dyn(catalog.as_ref(), &src_namespace).await; + cleanup_namespace_dyn(catalog.as_ref(), &dst_namespace).await; + catalog + .create_namespace(&src_namespace, HashMap::new()) + .await?; + catalog + .create_namespace(&dst_namespace, HashMap::new()) + .await?; + + let table = catalog + .create_table( + &src_namespace, + table_creation(normalize_test_name_with_parts!( + "catalog_rename_table_across_namespaces", + harness.label, + "table" + )), + ) + .await?; + let src_ident = table.identifier().clone(); + let dst_ident = TableIdent::new(dst_namespace.clone(), src_ident.name.clone()); + + catalog.rename_table(&src_ident, &dst_ident).await?; + assert!(catalog.table_exists(&dst_ident).await?); + assert!(!catalog.table_exists(&src_ident).await?); + + Ok(()) +} + +// Common behavior: renaming a missing table should error. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_rename_table_missing_source_errors(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_rename_table_missing_source_errors", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let src_ident = TableIdent::new(namespace.clone(), "missing".to_string()); + let dst_ident = TableIdent::new(namespace.clone(), "dest".to_string()); + + assert!(catalog.rename_table(&src_ident, &dst_ident).await.is_err()); + Ok(()) +} + +// Common behavior: renaming to an existing destination should error. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_rename_table_dest_exists_errors(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_rename_table_dest_exists_errors", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let src = catalog + .create_table( + &namespace, + table_creation(normalize_test_name_with_parts!( + "catalog_rename_table_dest_exists_errors", + harness.label, + "src" + )), + ) + .await? + .identifier() + .clone(); + let dst = catalog + .create_table( + &namespace, + table_creation(normalize_test_name_with_parts!( + "catalog_rename_table_dest_exists_errors", + harness.label, + "dst" + )), + ) + .await? + .identifier() + .clone(); + + assert!(catalog.rename_table(&src, &dst).await.is_err()); + Ok(()) +} diff --git a/crates/catalog/loader/tests/table_suite.rs b/crates/catalog/loader/tests/table_suite.rs new file mode 100644 index 0000000000..cdc9b11043 --- /dev/null +++ b/crates/catalog/loader/tests/table_suite.rs @@ -0,0 +1,353 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Common table behavior across catalogs. +//! +//! These tests assume Docker containers are started externally via `make docker-up`. + +mod common; + +use std::collections::HashMap; + +use common::{CatalogKind, cleanup_namespace_dyn, load_catalog, table_creation}; +use iceberg::transaction::{ApplyTransactionAction, Transaction}; +use iceberg::{ErrorKind, NamespaceIdent, Result, TableIdent}; +use iceberg_test_utils::normalize_test_name_with_parts; +use rstest::rstest; + +// Common behavior: table lifecycle CRUD. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_table_lifecycle(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_table_lifecycle", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_name = + normalize_test_name_with_parts!("catalog_table_lifecycle", harness.label, "table"); + let table = catalog + .create_table(&namespace, table_creation(table_name)) + .await?; + let ident = table.identifier().clone(); + + assert!(catalog.table_exists(&ident).await?); + let loaded = catalog.load_table(&ident).await?; + assert_eq!(loaded.identifier(), &ident); + + let tables = catalog.list_tables(&namespace).await?; + assert!(tables.contains(&ident)); + + let dest = TableIdent::new(ident.namespace.clone(), format!("{}_renamed", ident.name)); + catalog.rename_table(&ident, &dest).await?; + assert!(catalog.table_exists(&dest).await?); + assert!(!catalog.table_exists(&ident).await?); + + catalog.drop_table(&dest).await?; + assert!(!catalog.table_exists(&dest).await?); + + catalog.drop_namespace(&namespace).await?; + + Ok(()) +} + +// Common behavior: listing tables for a missing namespace should error. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_list_tables_missing_namespace_errors( + #[case] kind: CatalogKind, +) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_list_tables_missing_namespace_errors", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + + assert!(catalog.list_tables(&namespace).await.is_err()); + Ok(()) +} + +// Common behavior: listing tables in an empty namespace returns empty. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_list_tables_empty_namespace(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_list_tables_empty_namespace", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let tables = catalog.list_tables(&namespace).await?; + assert!(tables.is_empty()); + + Ok(()) +} + +// Common behavior: created tables expose the requested schema. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_create_table_schema(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_create_table_schema", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_name = + normalize_test_name_with_parts!("catalog_create_table_schema", harness.label, "table"); + let creation = table_creation(table_name); + let expected_schema = creation.schema.clone(); + + let table = catalog.create_table(&namespace, creation).await?; + assert_eq!(table.identifier().namespace, namespace); + assert_eq!(table.metadata().current_schema().as_ref(), &expected_schema); + + Ok(()) +} + +// Common behavior: updating table properties persists through the catalog. +// HMS is excluded because update_table is not supported yet. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_update_table_properties(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_update_table_properties", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_name = + normalize_test_name_with_parts!("catalog_update_table_properties", harness.label, "table"); + let table = catalog + .create_table(&namespace, table_creation(table_name)) + .await?; + + let tx = Transaction::new(&table); + let tx = tx + .update_table_properties() + .set("test_property".to_string(), "test_value".to_string()) + .apply(tx)?; + let updated = tx.commit(catalog.as_ref()).await?; + + assert_eq!( + updated.metadata().properties().get("test_property"), + Some(&"test_value".to_string()) + ); + + Ok(()) +} + +// Common behavior: update_table_properties is rejected when unsupported. +#[rstest] +#[case::hms_catalog(CatalogKind::Hms)] +#[tokio::test] +async fn test_catalog_update_table_properties_unsupported(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_update_table_properties_unsupported", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_name = normalize_test_name_with_parts!( + "catalog_update_table_properties_unsupported", + harness.label, + "table" + ); + let table = catalog + .create_table(&namespace, table_creation(table_name)) + .await?; + + let tx = Transaction::new(&table); + let tx = tx + .update_table_properties() + .set("test_property".to_string(), "test_value".to_string()) + .apply(tx)?; + + let err = tx.commit(catalog.as_ref()).await.unwrap_err(); + assert_eq!(err.kind(), ErrorKind::FeatureUnsupported); + + Ok(()) +} + +// Common behavior: dropping a missing table should error. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_drop_table_missing_errors(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_drop_table_missing_errors", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_ident = TableIdent::new(namespace.clone(), "missing".to_string()); + assert!(catalog.drop_table(&table_ident).await.is_err()); + Ok(()) +} + +// Common behavior: purge_table removes the table from the catalog. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_purge_table(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_purge_table", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_name = normalize_test_name_with_parts!("catalog_purge_table", harness.label, "table"); + let table = catalog + .create_table(&namespace, table_creation(table_name)) + .await?; + let ident = table.identifier().clone(); + + assert!(catalog.table_exists(&ident).await?); + + // Capture metadata location and file_io before purge so we can verify + // that the underlying files are actually deleted. + let metadata_location = table.metadata_location().map(|s| s.to_string()); + let file_io = table.file_io().clone(); + + catalog.purge_table(&ident).await?; + assert!(!catalog.table_exists(&ident).await?); + + if let Some(location) = &metadata_location { + assert!( + !file_io.exists(location).await?, + "Metadata file should have been deleted after purge" + ); + } + + catalog.drop_namespace(&namespace).await?; + + Ok(()) +} + +// Common behavior: purging a missing table should error. +#[rstest] +#[case::rest_catalog(CatalogKind::Rest)] +#[case::glue_catalog(CatalogKind::Glue)] +#[case::hms_catalog(CatalogKind::Hms)] +#[case::sql_catalog(CatalogKind::Sql)] +#[case::s3tables_catalog(CatalogKind::S3Tables)] +#[case::memory_catalog(CatalogKind::Memory)] +#[tokio::test] +async fn test_catalog_purge_table_missing_errors(#[case] kind: CatalogKind) -> Result<()> { + let Some(harness) = load_catalog(kind).await else { + return Ok(()); + }; + let catalog = harness.catalog; + let namespace = NamespaceIdent::new(normalize_test_name_with_parts!( + "catalog_purge_table_missing_errors", + harness.label + )); + + cleanup_namespace_dyn(catalog.as_ref(), &namespace).await; + catalog.create_namespace(&namespace, HashMap::new()).await?; + + let table_ident = TableIdent::new(namespace.clone(), "missing".to_string()); + assert!(catalog.purge_table(&table_ident).await.is_err()); + Ok(()) +} diff --git a/crates/catalog/rest/Cargo.toml b/crates/catalog/rest/Cargo.toml index e8a9562f8d..80cf83fdbc 100644 --- a/crates/catalog/rest/Cargo.toml +++ b/crates/catalog/rest/Cargo.toml @@ -39,7 +39,6 @@ serde = { workspace = true } serde_derive = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true, features = ["sync"] } -tracing = { workspace = true } typed-builder = { workspace = true } uuid = { workspace = true, features = ["v4"] } diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 8e07bbb21b..e01092f8b4 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -388,6 +388,35 @@ impl RestCatalog { } } + /// Sends a DELETE request for the given table, optionally requesting purge. + async fn delete_table(&self, table: &TableIdent, purge: bool) -> Result<()> { + let context = self.context().await?; + + let mut request_builder = context + .client + .request(Method::DELETE, context.config.table_endpoint(table)); + + if purge { + request_builder = request_builder.query(&[("purgeRequested", "true")]); + } + + let request = request_builder.build()?; + let http_response = context.client.query_catalog(request).await?; + + match http_response.status() { + StatusCode::NO_CONTENT | StatusCode::OK => Ok(()), + StatusCode::NOT_FOUND => Err(Error::new( + ErrorKind::TableNotFound, + "Tried to drop a table that does not exist", + )), + _ => Err(deserialize_unexpected_catalog_error( + http_response, + context.client.disable_header_redaction(), + ) + .await), + } + } + /// Gets the [`RestContext`] from the catalog. async fn context(&self) -> Result<&RestContext> { self.ctx @@ -541,7 +570,7 @@ impl RestCatalog { } StatusCode::NOT_FOUND => { return Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::TableNotFound, "Tried to load a table that does not exist", )); } @@ -665,13 +694,13 @@ impl RestCatalog { } StatusCode::NOT_FOUND => { return Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::NamespaceNotFound, "Tried to create a table under a namespace that does not exist", )); } StatusCode::CONFLICT => { return Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::TableAlreadyExists, "The table already exists", )); } @@ -781,7 +810,7 @@ impl Catalog for RestCatalog { } StatusCode::NOT_FOUND => { return Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::NamespaceNotFound, "The parent parameter of the namespace provided does not exist", )); } @@ -823,7 +852,7 @@ impl Catalog for RestCatalog { Ok(Namespace::from(response)) } StatusCode::CONFLICT => Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::NamespaceAlreadyExists, "Tried to create a namespace that already exists", )), _ => Err(deserialize_unexpected_catalog_error( @@ -851,7 +880,7 @@ impl Catalog for RestCatalog { Ok(Namespace::from(response)) } StatusCode::NOT_FOUND => Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::NamespaceNotFound, "Tried to get a namespace that does not exist", )), _ => Err(deserialize_unexpected_catalog_error( @@ -907,7 +936,7 @@ impl Catalog for RestCatalog { match http_response.status() { StatusCode::NO_CONTENT | StatusCode::OK => Ok(()), StatusCode::NOT_FOUND => Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::NamespaceNotFound, "Tried to drop a namespace that does not exist", )), _ => Err(deserialize_unexpected_catalog_error( @@ -947,7 +976,7 @@ impl Catalog for RestCatalog { } StatusCode::NOT_FOUND => { return Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::NamespaceNotFound, "Tried to list tables of a namespace that does not exist", )); } @@ -989,27 +1018,13 @@ impl Catalog for RestCatalog { /// Drop a table from the catalog. async fn drop_table(&self, table: &TableIdent) -> Result<()> { - let context = self.context().await?; - - let request = context - .client - .request(Method::DELETE, context.config.table_endpoint(table)) - .build()?; - - let http_response = context.client.query_catalog(request).await?; + self.delete_table(table, false).await + } - match http_response.status() { - StatusCode::NO_CONTENT | StatusCode::OK => Ok(()), - StatusCode::NOT_FOUND => Err(Error::new( - ErrorKind::Unexpected, - "Tried to drop a table that does not exist", - )), - _ => Err(deserialize_unexpected_catalog_error( - http_response, - context.client.disable_header_redaction(), - ) - .await), - } + /// Drop a table from the catalog and purge its data by sending + /// `purgeRequested=true` to the REST server. + async fn purge_table(&self, table: &TableIdent) -> Result<()> { + self.delete_table(table, true).await } /// Check if a table exists in the catalog. @@ -1052,11 +1067,11 @@ impl Catalog for RestCatalog { match http_response.status() { StatusCode::NO_CONTENT | StatusCode::OK => Ok(()), StatusCode::NOT_FOUND => Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::TableNotFound, "Tried to rename a table that does not exist (is the namespace correct?)", )), StatusCode::CONFLICT => Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::TableAlreadyExists, "Tried to rename a table to a name that already exists", )), _ => Err(deserialize_unexpected_catalog_error( diff --git a/crates/catalog/rest/tests/rest_catalog_test.rs b/crates/catalog/rest/tests/rest_catalog_test.rs deleted file mode 100644 index 8bf2b6727c..0000000000 --- a/crates/catalog/rest/tests/rest_catalog_test.rs +++ /dev/null @@ -1,642 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Integration tests for rest catalog. -//! -//! These tests assume Docker containers are started externally via `make docker-up`. -//! Each test uses unique namespaces based on module path to avoid conflicts. - -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; - -use async_trait::async_trait; -use iceberg::io::LocalFsStorageFactory; -use iceberg::spec::{FormatVersion, NestedField, PrimitiveType, Schema, Type}; -use iceberg::transaction::{ApplyTransactionAction, Transaction}; -use iceberg::{ - Catalog, CatalogBuilder, Namespace, NamespaceIdent, Result as IcebergResult, TableCreation, - TableIdent, -}; -use iceberg_catalog_rest::{ - CustomAuthenticator, REST_CATALOG_PROP_URI, RestCatalog, RestCatalogBuilder, -}; -use iceberg_test_utils::{ - cleanup_namespace, get_rest_catalog_endpoint, normalize_test_name_with_parts, set_up, -}; -use tokio::time::sleep; -use tracing::info; - -async fn get_catalog(authenticator: Option>) -> RestCatalog { - set_up(); - - let rest_endpoint = get_rest_catalog_endpoint(); - - // Wait for catalog to be ready - let client = reqwest::Client::new(); - let mut retries = 0; - while retries < 30 { - match client - .get(format!("{rest_endpoint}/v1/config")) - .send() - .await - { - Ok(resp) if resp.status().is_success() => { - info!("REST catalog is ready at {}", rest_endpoint); - break; - } - _ => { - info!( - "Waiting for REST catalog to be ready... (attempt {})", - retries + 1 - ); - sleep(std::time::Duration::from_millis(1000)).await; - retries += 1; - } - } - } - - let mut builder = - RestCatalogBuilder::default().with_storage_factory(Arc::new(LocalFsStorageFactory)); - if let Some(auth) = authenticator { - builder = builder.with_token_authenticator(auth); - } - - builder - .load( - "rest", - HashMap::from([(REST_CATALOG_PROP_URI.to_string(), rest_endpoint)]), - ) - .await - .unwrap() -} - -#[tokio::test] -async fn test_get_non_exist_namespace() { - let catalog = get_catalog(None).await; - - // Use unique namespace name to ensure it doesn't exist - let ns_ident = NamespaceIdent::new(normalize_test_name_with_parts!( - "test_get_non_exist_namespace" - )); - // Clean up from any previous test runs - cleanup_namespace(&catalog, &ns_ident).await; - - let result = catalog.get_namespace(&ns_ident).await; - - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("does not exist")); -} - -#[tokio::test] -async fn test_get_namespace() { - let catalog = get_catalog(None).await; - - // Use unique namespace to avoid conflicts with other tests - let ns = Namespace::with_properties( - NamespaceIdent::from_strs([ - "apple", - "ios", - &normalize_test_name_with_parts!("test_get_namespace"), - ]) - .unwrap(), - HashMap::from([ - ("owner".to_string(), "ray".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - // Clean up from any previous test runs - cleanup_namespace(&catalog, ns.name()).await; - - // Verify that namespace doesn't exist - assert!(catalog.get_namespace(ns.name()).await.is_err()); - - // Create this namespace - let created_ns = catalog - .create_namespace(ns.name(), ns.properties().clone()) - .await - .unwrap(); - - assert_eq!(ns.name(), created_ns.name()); - assert_map_contains(ns.properties(), created_ns.properties()); - - // Check that this namespace already exists - let get_ns = catalog.get_namespace(ns.name()).await.unwrap(); - assert_eq!(ns.name(), get_ns.name()); - assert_map_contains(ns.properties(), created_ns.properties()); -} - -#[tokio::test] -async fn test_list_namespace() { - let catalog = get_catalog(None).await; - - // Use unique parent namespace to avoid conflicts - let parent_ns_name = normalize_test_name_with_parts!("test_list_namespace"); - let parent_ident = NamespaceIdent::from_strs([&parent_ns_name]).unwrap(); - - let ns1 = Namespace::with_properties( - NamespaceIdent::from_strs([&parent_ns_name, "ios"]).unwrap(), - HashMap::from([ - ("owner".to_string(), "ray".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - let ns2 = Namespace::with_properties( - NamespaceIdent::from_strs([&parent_ns_name, "macos"]).unwrap(), - HashMap::from([ - ("owner".to_string(), "xuanwo".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - // Clean up from any previous test runs - cleanup_namespace(&catalog, ns1.name()).await; - cleanup_namespace(&catalog, ns2.name()).await; - cleanup_namespace(&catalog, &parent_ident).await; - - // Currently this namespace doesn't exist, so it should return error. - assert!(catalog.list_namespaces(Some(&parent_ident)).await.is_err()); - - // Create namespaces - catalog - .create_namespace(ns1.name(), ns1.properties().clone()) - .await - .unwrap(); - catalog - .create_namespace(ns2.name(), ns1.properties().clone()) - .await - .unwrap(); - - // List namespace - let nss = catalog.list_namespaces(Some(&parent_ident)).await.unwrap(); - - assert!(nss.contains(ns1.name())); - assert!(nss.contains(ns2.name())); -} - -#[tokio::test] -async fn test_list_empty_namespace() { - let catalog = get_catalog(None).await; - - // Use unique namespace to avoid conflicts - let ns_apple = Namespace::with_properties( - NamespaceIdent::from_strs([ - "list_empty", - "apple", - &normalize_test_name_with_parts!("test_list_empty_namespace"), - ]) - .unwrap(), - HashMap::from([ - ("owner".to_string(), "ray".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - // Clean up from any previous test runs - cleanup_namespace(&catalog, ns_apple.name()).await; - - // Currently this namespace doesn't exist, so it should return error. - assert!( - catalog - .list_namespaces(Some(ns_apple.name())) - .await - .is_err() - ); - - // Create namespaces - catalog - .create_namespace(ns_apple.name(), ns_apple.properties().clone()) - .await - .unwrap(); - - // List namespace - let nss = catalog - .list_namespaces(Some(ns_apple.name())) - .await - .unwrap(); - assert!(nss.is_empty()); -} - -#[tokio::test] -async fn test_list_root_namespace() { - let catalog = get_catalog(None).await; - - // Use unique root namespace to avoid conflicts - let root_ns_name = normalize_test_name_with_parts!("test_list_root_namespace"); - let root_ident = NamespaceIdent::from_strs([&root_ns_name]).unwrap(); - - let ns1 = Namespace::with_properties( - NamespaceIdent::from_strs([&root_ns_name, "apple", "ios"]).unwrap(), - HashMap::from([ - ("owner".to_string(), "ray".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - let ns2 = Namespace::with_properties( - NamespaceIdent::from_strs([&root_ns_name, "google", "android"]).unwrap(), - HashMap::from([ - ("owner".to_string(), "xuanwo".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - // Clean up from any previous test runs - cleanup_namespace(&catalog, ns1.name()).await; - cleanup_namespace(&catalog, ns2.name()).await; - cleanup_namespace(&catalog, &root_ident).await; - - // Currently this namespace doesn't exist, so it should return error. - assert!(catalog.list_namespaces(Some(&root_ident)).await.is_err()); - - // Create namespaces - catalog - .create_namespace(ns1.name(), ns1.properties().clone()) - .await - .unwrap(); - catalog - .create_namespace(ns2.name(), ns1.properties().clone()) - .await - .unwrap(); - - // List namespace - let nss = catalog.list_namespaces(None).await.unwrap(); - assert!(nss.contains(&root_ident)); -} - -#[tokio::test] -async fn test_create_table() { - let catalog = get_catalog(None).await; - - // Use unique namespace to avoid conflicts - let ns = Namespace::with_properties( - NamespaceIdent::from_strs([ - "create_table", - "apple", - "ios", - &normalize_test_name_with_parts!("test_create_table"), - ]) - .unwrap(), - HashMap::from([ - ("owner".to_string(), "ray".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - // Clean up from any previous test runs - cleanup_namespace(&catalog, ns.name()).await; - - // Create namespaces - catalog - .create_namespace(ns.name(), ns.properties().clone()) - .await - .unwrap(); - - let schema = Schema::builder() - .with_schema_id(1) - .with_identifier_field_ids(vec![2]) - .with_fields(vec![ - NestedField::optional(1, "foo", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(2, "bar", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(3, "baz", Type::Primitive(PrimitiveType::Boolean)).into(), - ]) - .build() - .unwrap(); - - let table_creation = TableCreation::builder() - .name("t1".to_string()) - .schema(schema.clone()) - .build(); - - let table = catalog - .create_table(ns.name(), table_creation) - .await - .unwrap(); - - assert_eq!( - table.identifier(), - &TableIdent::new(ns.name().clone(), "t1".to_string()) - ); - - assert_eq!( - table.metadata().current_schema().as_struct(), - schema.as_struct() - ); - assert_eq!(table.metadata().format_version(), FormatVersion::V2); - assert!(table.metadata().current_snapshot().is_none()); - assert!(table.metadata().history().is_empty()); - assert!(table.metadata().default_sort_order().is_unsorted()); - assert!(table.metadata().default_partition_spec().is_unpartitioned()); -} - -#[tokio::test] -async fn test_update_table() { - let catalog = get_catalog(None).await; - - // Use unique namespace to avoid conflicts - let ns = Namespace::with_properties( - NamespaceIdent::from_strs([ - "update_table", - "apple", - "ios", - &normalize_test_name_with_parts!("test_update_table"), - ]) - .unwrap(), - HashMap::from([ - ("owner".to_string(), "ray".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - // Clean up from any previous test runs - cleanup_namespace(&catalog, ns.name()).await; - - // Create namespaces - catalog - .create_namespace(ns.name(), ns.properties().clone()) - .await - .unwrap(); - - let schema = Schema::builder() - .with_schema_id(1) - .with_identifier_field_ids(vec![2]) - .with_fields(vec![ - NestedField::optional(1, "foo", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(2, "bar", Type::Primitive(PrimitiveType::Int)).into(), - NestedField::optional(3, "baz", Type::Primitive(PrimitiveType::Boolean)).into(), - ]) - .build() - .unwrap(); - - // Now we create a table - let table_creation = TableCreation::builder() - .name("t1".to_string()) - .schema(schema.clone()) - .build(); - - let table = catalog - .create_table(ns.name(), table_creation) - .await - .unwrap(); - - assert_eq!( - table.identifier(), - &TableIdent::new(ns.name().clone(), "t1".to_string()) - ); - - let tx = Transaction::new(&table); - // Update table by committing transaction - let table2 = tx - .update_table_properties() - .set("prop1".to_string(), "v1".to_string()) - .apply(tx) - .unwrap() - .commit(&catalog) - .await - .unwrap(); - - assert_map_contains( - &HashMap::from([("prop1".to_string(), "v1".to_string())]), - table2.metadata().properties(), - ); -} - -fn assert_map_contains(map1: &HashMap, map2: &HashMap) { - for (k, v) in map1 { - assert!(map2.contains_key(k)); - assert_eq!(map2.get(k).unwrap(), v); - } -} - -#[tokio::test] -async fn test_list_empty_multi_level_namespace() { - let catalog = get_catalog(None).await; - - // Use unique namespace to avoid conflicts - let ns_apple = Namespace::with_properties( - NamespaceIdent::from_strs([ - "multi_level", - "a_a", - "apple", - &normalize_test_name_with_parts!("test_list_empty_multi_level_namespace"), - ]) - .unwrap(), - HashMap::from([ - ("owner".to_string(), "ray".to_string()), - ("community".to_string(), "apache".to_string()), - ]), - ); - - // Clean up from any previous test runs - cleanup_namespace(&catalog, ns_apple.name()).await; - - // Currently this namespace doesn't exist, so it should return error. - assert!( - catalog - .list_namespaces(Some(ns_apple.name())) - .await - .is_err() - ); - - // Create namespaces - catalog - .create_namespace(ns_apple.name(), ns_apple.properties().clone()) - .await - .unwrap(); - - // List namespace - let nss = catalog - .list_namespaces(Some(ns_apple.name())) - .await - .unwrap(); - assert!(nss.is_empty()); -} - -#[tokio::test] -async fn test_register_table() { - let catalog = get_catalog(None).await; - - // Create unique namespace to avoid conflicts - let ns = NamespaceIdent::new(normalize_test_name_with_parts!("test_register_table")); - - // Clean up from any previous test runs - cleanup_namespace(&catalog, &ns).await; - - catalog.create_namespace(&ns, HashMap::new()).await.unwrap(); - - // Create the table, store the metadata location, drop the table - let empty_schema = Schema::builder().build().unwrap(); - let table_creation = TableCreation::builder() - .name("t1".to_string()) - .schema(empty_schema) - .build(); - - let table = catalog.create_table(&ns, table_creation).await.unwrap(); - - let metadata_location = table.metadata_location().unwrap(); - catalog.drop_table(table.identifier()).await.unwrap(); - - let new_table_identifier = TableIdent::new(ns.clone(), "t2".to_string()); - let table_registered = catalog - .register_table(&new_table_identifier, metadata_location.to_string()) - .await - .unwrap(); - - assert_eq!( - table.metadata_location(), - table_registered.metadata_location() - ); - assert_ne!( - table.identifier().to_string(), - table_registered.identifier().to_string() - ); -} - -#[derive(Debug)] -struct CountingAuthenticator { - count: Arc>, -} - -#[async_trait] -impl CustomAuthenticator for CountingAuthenticator { - async fn get_token(&self) -> IcebergResult { - let mut c = self.count.lock().unwrap(); - *c += 1; - // Return a unique token each time to ensure dynamic generation - Ok(format!("token_{}", *c)) - } -} - -#[tokio::test] -async fn test_authenticator_token_refresh() { - // Track how many times tokens were requested - let token_request_count = Arc::new(Mutex::new(0)); - let token_request_count_clone = token_request_count.clone(); - - let authenticator = Arc::new(CountingAuthenticator { - count: token_request_count_clone, - }); - - let catalog_with_auth = get_catalog(Some(authenticator.clone())).await; - - // Clean up from any previous test runs - let ns1_ident = NamespaceIdent::from_strs(["test_refresh_1"]).unwrap(); - let ns2_ident = NamespaceIdent::from_strs(["test_refresh_2"]).unwrap(); - cleanup_namespace(&catalog_with_auth, &ns1_ident).await; - cleanup_namespace(&catalog_with_auth, &ns2_ident).await; - - // Perform multiple operations that should reuse the cached token - let ns1 = Namespace::with_properties(ns1_ident.clone(), HashMap::new()); - catalog_with_auth - .create_namespace(ns1.name(), HashMap::new()) - .await - .unwrap(); - - let ns2 = Namespace::with_properties(ns2_ident.clone(), HashMap::new()); - catalog_with_auth - .create_namespace(ns2.name(), HashMap::new()) - .await - .unwrap(); - - // With lazy authentication, the token is fetched once and cached for reuse - // across multiple operations, rather than being called on every request - let count = *token_request_count.lock().unwrap(); - assert_eq!( - count, 1, - "Authenticator should have been called once for lazy token caching, but was called {count} times" - ); - - // Test that token is refreshed when invalidated - catalog_with_auth.invalidate_token().await.unwrap(); - - let ns3_ident = NamespaceIdent::from_strs(["test_refresh_3"]).unwrap(); - cleanup_namespace(&catalog_with_auth, &ns3_ident).await; - - let ns3 = Namespace::with_properties(ns3_ident.clone(), HashMap::new()); - catalog_with_auth - .create_namespace(ns3.name(), HashMap::new()) - .await - .unwrap(); - - // After invalidating and making another request, authenticator should be called again - let count = *token_request_count.lock().unwrap(); - assert_eq!( - count, 2, - "Authenticator should have been called twice (once initial, once after invalidation), but was called {count} times" - ); - - // Clean up - cleanup_namespace(&catalog_with_auth, &ns1_ident).await; - cleanup_namespace(&catalog_with_auth, &ns2_ident).await; - cleanup_namespace(&catalog_with_auth, &ns3_ident).await; -} - -#[tokio::test] -async fn test_authenticator_persists_across_operations() { - let operation_count = Arc::new(Mutex::new(0)); - let operation_count_clone = operation_count.clone(); - - let authenticator = Arc::new(CountingAuthenticator { - count: operation_count_clone, - }); - - let catalog_with_auth = get_catalog(Some(authenticator)).await; - - // Clean up from any previous test runs - let ns_ident = NamespaceIdent::from_strs(["test_persist", "auth"]).unwrap(); - let parent_ident = NamespaceIdent::from_strs(["test_persist"]).unwrap(); - cleanup_namespace(&catalog_with_auth, &ns_ident).await; - - // Create a namespace - let ns = Namespace::with_properties(ns_ident.clone(), HashMap::new()); - catalog_with_auth - .create_namespace(ns.name(), HashMap::new()) - .await - .unwrap(); - - let count_after_create = *operation_count.lock().unwrap(); - - // List the namespace children (should reuse the cached token from the create operation) - // We need to list children of "test_persist" to find "auth" - let list_result = catalog_with_auth - .list_namespaces(Some(&parent_ident)) - .await - .unwrap(); - assert!( - list_result.contains(&ns_ident), - "Namespace {:?} not found in list {:?}", - ns.name(), - list_result - ); - - let count_after_list = *operation_count.lock().unwrap(); - - // With lazy authentication, the token is fetched once on the first operation - // and then reused for subsequent operations without calling the authenticator again - assert_eq!( - count_after_create, 1, - "Authenticator should be called once for the create operation" - ); - assert_eq!( - count_after_list, 1, - "Authenticator should still have been called only once (token is cached and reused for list)" - ); - - // Clean up - cleanup_namespace(&catalog_with_auth, &ns_ident).await; -} diff --git a/crates/catalog/s3tables/src/catalog.rs b/crates/catalog/s3tables/src/catalog.rs index afe28ae453..b88bd77d29 100644 --- a/crates/catalog/s3tables/src/catalog.rs +++ b/crates/catalog/s3tables/src/catalog.rs @@ -17,6 +17,7 @@ use std::collections::HashMap; use std::future::Future; +use std::str::FromStr; use std::sync::Arc; use async_trait::async_trait; @@ -307,6 +308,13 @@ impl Catalog for S3TablesCatalog { namespace: &NamespaceIdent, _properties: HashMap, ) -> Result { + if self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceAlreadyExists, + format!("Namespace {namespace:?} already exists"), + )); + } + let req = self .s3tables_client .create_namespace() @@ -329,6 +337,13 @@ impl Catalog for S3TablesCatalog { /// - If there is an error querying the database, returned by /// `from_aws_sdk_error`. async fn get_namespace(&self, namespace: &NamespaceIdent) -> Result { + if !self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {namespace:?} does not exist"), + )); + } + let req = self .s3tables_client .get_namespace() @@ -396,6 +411,13 @@ impl Catalog for S3TablesCatalog { /// - Errors from the underlying database deletion process, converted using /// `from_aws_sdk_error`. async fn drop_namespace(&self, namespace: &NamespaceIdent) -> Result<()> { + if !self.namespace_exists(namespace).await? { + return Err(Error::new( + ErrorKind::NamespaceNotFound, + format!("Namespace {namespace:?} does not exist"), + )); + } + let req = self .s3tables_client .delete_namespace() @@ -501,17 +523,17 @@ impl Catalog for S3TablesCatalog { let metadata = TableMetadataBuilder::from_table_creation(creation)? .build()? .metadata; - let metadata_location = - MetadataLocation::new_with_table_location(table_location).to_string(); + let metadata_location = MetadataLocation::new_with_metadata(table_location, &metadata); metadata.write_to(&self.file_io, &metadata_location).await?; // update metadata location + let metadata_location_str = metadata_location.to_string(); self.s3tables_client .update_table_metadata_location() .table_bucket_arn(self.config.table_bucket_arn.clone()) .namespace(namespace.to_url_string()) .name(table_ident.name()) - .metadata_location(metadata_location.clone()) + .metadata_location(metadata_location_str.clone()) .version_token(create_resp.version_token()) .send() .await @@ -519,7 +541,7 @@ impl Catalog for S3TablesCatalog { let table = Table::builder() .identifier(table_ident) - .metadata_location(metadata_location) + .metadata_location(metadata_location_str) .metadata(metadata) .file_io(self.file_io.clone()) .build()?; @@ -540,15 +562,18 @@ impl Catalog for S3TablesCatalog { Ok(self.load_table_with_version_token(table_ident).await?.0) } - /// Drops an existing table from the s3tables catalog. - /// - /// Validates the table identifier and then deletes the corresponding - /// table from the s3tables catalog. + /// Not supported for S3Tables. Use `purge_table` instead. /// - /// This function can return an error in the following situations: - /// - Errors from the underlying database deletion process, converted using - /// `from_aws_sdk_error`. - async fn drop_table(&self, table: &TableIdent) -> Result<()> { + /// S3 Tables doesn't support soft delete, so dropping a table will permanently remove it from the catalog. + async fn drop_table(&self, _table: &TableIdent) -> Result<()> { + Err(Error::new( + ErrorKind::FeatureUnsupported, + "drop_table is not supported for S3Tables; use purge_table instead", + )) + } + + /// Purge a table from the S3 Tables catalog. + async fn purge_table(&self, table: &TableIdent) -> Result<()> { let req = self .s3tables_client .delete_table() @@ -630,11 +655,12 @@ impl Catalog for S3TablesCatalog { self.load_table_with_version_token(&table_ident).await?; let staged_table = commit.apply(current_table)?; - let staged_metadata_location = staged_table.metadata_location_result()?; + let staged_metadata_location_str = staged_table.metadata_location_result()?; + let staged_metadata_location = MetadataLocation::from_str(staged_metadata_location_str)?; staged_table .metadata() - .write_to(staged_table.file_io(), staged_metadata_location) + .write_to(staged_table.file_io(), &staged_metadata_location) .await?; let builder = self @@ -644,7 +670,7 @@ impl Catalog for S3TablesCatalog { .namespace(table_namespace.to_url_string()) .name(table_ident.name()) .version_token(version_token) - .metadata_location(staged_metadata_location); + .metadata_location(staged_metadata_location_str); let _ = builder.send().await.map_err(|e| { let error = e.into_service_error(); diff --git a/crates/catalog/sql/src/catalog.rs b/crates/catalog/sql/src/catalog.rs index 97a224aa9c..7e468e7e37 100644 --- a/crates/catalog/sql/src/catalog.rs +++ b/crates/catalog/sql/src/catalog.rs @@ -434,7 +434,7 @@ impl Catalog for SqlCatalog { if exists { return Err(Error::new( - iceberg::ErrorKind::Unexpected, + iceberg::ErrorKind::NamespaceAlreadyExists, format!("Namespace {namespace:?} already exists"), )); } @@ -757,6 +757,17 @@ impl Catalog for SqlCatalog { Ok(()) } + async fn purge_table(&self, table: &TableIdent) -> Result<()> { + let table_info = self.load_table(table).await?; + self.drop_table(table).await?; + iceberg::drop_table_data( + table_info.file_io(), + table_info.metadata(), + table_info.metadata_location(), + ) + .await + } + async fn load_table(&self, identifier: &TableIdent) -> Result { if !self.table_exists(identifier).await? { return no_such_table_err(identifier); @@ -851,21 +862,22 @@ impl Catalog for SqlCatalog { .build()? .metadata; let tbl_metadata_location = - MetadataLocation::new_with_table_location(location.clone()).to_string(); + MetadataLocation::new_with_metadata(location.clone(), &tbl_metadata); tbl_metadata .write_to(&self.fileio, &tbl_metadata_location) .await?; + let tbl_metadata_location_str = tbl_metadata_location.to_string(); self.execute(&format!( "INSERT INTO {CATALOG_TABLE_NAME} ({CATALOG_FIELD_CATALOG_NAME}, {CATALOG_FIELD_TABLE_NAMESPACE}, {CATALOG_FIELD_TABLE_NAME}, {CATALOG_FIELD_METADATA_LOCATION_PROP}, {CATALOG_FIELD_RECORD_TYPE}) VALUES (?, ?, ?, ?, ?) - "), vec![Some(&self.name), Some(&namespace.join(".")), Some(&tbl_name.clone()), Some(&tbl_metadata_location), Some(CATALOG_FIELD_TABLE_RECORD_TYPE)], None).await?; + "), vec![Some(&self.name), Some(&namespace.join(".")), Some(&tbl_name.clone()), Some(&tbl_metadata_location_str), Some(CATALOG_FIELD_TABLE_RECORD_TYPE)], None).await?; Ok(Table::builder() .file_io(self.fileio.clone()) - .metadata_location(tbl_metadata_location) + .metadata_location(tbl_metadata_location_str) .identifier(tbl_ident) .metadata(tbl_metadata) .build()?) @@ -949,13 +961,15 @@ impl Catalog for SqlCatalog { let current_metadata_location = current_table.metadata_location_result()?.to_string(); let staged_table = commit.apply(current_table)?; - let staged_metadata_location = staged_table.metadata_location_result()?; + let staged_metadata_location_str = staged_table.metadata_location_result()?; + let staged_metadata_location = MetadataLocation::from_str(staged_metadata_location_str)?; staged_table .metadata() .write_to(staged_table.file_io(), &staged_metadata_location) .await?; + let staged_metadata_location_str = staged_metadata_location.to_string(); let update_result = self .execute( &format!( @@ -971,7 +985,7 @@ impl Catalog for SqlCatalog { AND {CATALOG_FIELD_METADATA_LOCATION_PROP} = ?" ), vec![ - Some(staged_metadata_location), + Some(&staged_metadata_location_str), Some(current_metadata_location.as_str()), Some(&self.name), Some(table_ident.name()), @@ -1003,7 +1017,6 @@ mod tests { use iceberg::io::LocalFsStorageFactory; use iceberg::spec::{NestedField, PartitionSpec, PrimitiveType, Schema, SortOrder, Type}; use iceberg::table::Table; - use iceberg::transaction::{ApplyTransactionAction, Transaction}; use iceberg::{Catalog, CatalogBuilder, Namespace, NamespaceIdent, TableCreation, TableIdent}; use itertools::Itertools; use regex::Regex; @@ -1383,20 +1396,6 @@ mod tests { assert_eq!(catalog2.list_namespaces(None).await.unwrap(), vec![]); } - #[tokio::test] - async fn test_list_namespaces_returns_multiple_namespaces() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; - let namespace_ident_1 = NamespaceIdent::new("a".into()); - let namespace_ident_2 = NamespaceIdent::new("b".into()); - create_namespaces(&catalog, &vec![&namespace_ident_1, &namespace_ident_2]).await; - - assert_eq!( - to_set(catalog.list_namespaces(None).await.unwrap()), - to_set(vec![namespace_ident_1, namespace_ident_2]) - ); - } - #[tokio::test] async fn test_list_namespaces_returns_only_top_level_namespaces() { let warehouse_loc = temp_path(); @@ -1543,31 +1542,6 @@ mod tests { ); } - #[tokio::test] - async fn test_create_namespace_throws_error_if_namespace_already_exists() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; - let namespace_ident = NamespaceIdent::new("a".into()); - create_namespace(&catalog, &namespace_ident).await; - - assert_eq!( - catalog - .create_namespace(&namespace_ident, HashMap::new()) - .await - .unwrap_err() - .to_string(), - format!( - "Unexpected => Namespace {:?} already exists", - &namespace_ident - ) - ); - - assert_eq!( - catalog.get_namespace(&namespace_ident).await.unwrap(), - Namespace::with_properties(namespace_ident, default_properties()) - ); - } - #[tokio::test] async fn test_create_nested_namespace() { let warehouse_loc = temp_path(); @@ -1637,35 +1611,6 @@ mod tests { ) } - #[tokio::test] - async fn test_update_namespace() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; - let namespace_ident = NamespaceIdent::new("a".into()); - create_namespace(&catalog, &namespace_ident).await; - - let mut props = HashMap::from_iter([ - ("prop1".to_string(), "val1".to_string()), - ("prop2".into(), "val2".into()), - ]); - - catalog - .update_namespace(&namespace_ident, props.clone()) - .await - .unwrap(); - - props.insert("exists".into(), "true".into()); - - assert_eq!( - *catalog - .get_namespace(&namespace_ident) - .await - .unwrap() - .properties(), - props - ) - } - #[tokio::test] async fn test_update_nested_namespace() { let warehouse_loc = temp_path(); @@ -1695,28 +1640,6 @@ mod tests { ) } - #[tokio::test] - async fn test_update_namespace_errors_if_namespace_doesnt_exist() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; - let namespace_ident = NamespaceIdent::new("a".into()); - - let props = HashMap::from_iter([ - ("prop1".to_string(), "val1".to_string()), - ("prop2".into(), "val2".into()), - ]); - - let err = catalog - .update_namespace(&namespace_ident, props) - .await - .unwrap_err(); - - assert_eq!( - err.message(), - format!("No such namespace: {namespace_ident:?}") - ); - } - #[tokio::test] async fn test_update_namespace_errors_if_nested_namespace_doesnt_exist() { let warehouse_loc = temp_path(); @@ -1739,18 +1662,6 @@ mod tests { ); } - #[tokio::test] - async fn test_drop_namespace() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; - let namespace_ident = NamespaceIdent::new("abc".into()); - create_namespace(&catalog, &namespace_ident).await; - - catalog.drop_namespace(&namespace_ident).await.unwrap(); - - assert!(!catalog.namespace_exists(&namespace_ident).await.unwrap()) - } - #[tokio::test] async fn test_drop_nested_namespace() { let warehouse_loc = temp_path(); @@ -1807,22 +1718,6 @@ mod tests { assert!(catalog.namespace_exists(&namespace_ident_a).await.unwrap()); } - #[tokio::test] - async fn test_drop_namespace_throws_error_if_namespace_doesnt_exist() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; - - let non_existent_namespace_ident = NamespaceIdent::new("abc".into()); - assert_eq!( - catalog - .drop_namespace(&non_existent_namespace_ident) - .await - .unwrap_err() - .to_string(), - format!("Unexpected => No such namespace: {non_existent_namespace_ident:?}") - ) - } - #[tokio::test] async fn test_drop_namespace_throws_error_if_nested_namespace_doesnt_exist() { let warehouse_loc = temp_path(); @@ -1837,7 +1732,7 @@ mod tests { .await .unwrap_err() .to_string(), - format!("Unexpected => No such namespace: {non_existent_namespace_ident:?}") + format!("NamespaceNotFound => No such namespace: {non_existent_namespace_ident:?}") ) } @@ -1861,72 +1756,6 @@ mod tests { ); } - #[tokio::test] - async fn test_list_tables_returns_empty_vector() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; - let namespace_ident = NamespaceIdent::new("a".into()); - create_namespace(&catalog, &namespace_ident).await; - - assert_eq!(catalog.list_tables(&namespace_ident).await.unwrap(), vec![]); - } - - #[tokio::test] - async fn test_list_tables_throws_error_if_namespace_doesnt_exist() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; - - let non_existent_namespace_ident = NamespaceIdent::new("n1".into()); - - assert_eq!( - catalog - .list_tables(&non_existent_namespace_ident) - .await - .unwrap_err() - .to_string(), - format!("Unexpected => No such namespace: {non_existent_namespace_ident:?}"), - ); - } - - #[tokio::test] - async fn test_create_table_with_location() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await; - let namespace_ident = NamespaceIdent::new("a".into()); - create_namespace(&catalog, &namespace_ident).await; - - let table_name = "abc"; - let location = warehouse_loc.clone(); - let table_creation = TableCreation::builder() - .name(table_name.into()) - .location(location.clone()) - .schema(simple_table_schema()) - .build(); - - let expected_table_ident = TableIdent::new(namespace_ident.clone(), table_name.into()); - - assert_table_eq( - &catalog - .create_table(&namespace_ident, table_creation) - .await - .unwrap(), - &expected_table_ident, - &simple_table_schema(), - ); - - let table = catalog.load_table(&expected_table_ident).await.unwrap(); - - assert_table_eq(&table, &expected_table_ident, &simple_table_schema()); - - assert!( - table - .metadata_location() - .unwrap() - .to_string() - .starts_with(&location) - ) - } - #[tokio::test] async fn test_create_table_falls_back_to_namespace_location_if_table_location_is_missing() { let warehouse_loc = temp_path(); @@ -2125,54 +1954,10 @@ mod tests { .await .unwrap_err() .to_string(), - format!("Unexpected => Table {:?} already exists.", &table_ident) - ); - } - - #[tokio::test] - async fn test_rename_table_in_same_namespace() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; - let namespace_ident = NamespaceIdent::new("n1".into()); - create_namespace(&catalog, &namespace_ident).await; - let src_table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into()); - let dst_table_ident = TableIdent::new(namespace_ident.clone(), "tbl2".into()); - create_table(&catalog, &src_table_ident).await; - - catalog - .rename_table(&src_table_ident, &dst_table_ident) - .await - .unwrap(); - - assert_eq!(catalog.list_tables(&namespace_ident).await.unwrap(), vec![ - dst_table_ident - ],); - } - - #[tokio::test] - async fn test_rename_table_across_namespaces() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; - let src_namespace_ident = NamespaceIdent::new("a".into()); - let dst_namespace_ident = NamespaceIdent::new("b".into()); - create_namespaces(&catalog, &vec![&src_namespace_ident, &dst_namespace_ident]).await; - let src_table_ident = TableIdent::new(src_namespace_ident.clone(), "tbl1".into()); - let dst_table_ident = TableIdent::new(dst_namespace_ident.clone(), "tbl2".into()); - create_table(&catalog, &src_table_ident).await; - - catalog - .rename_table(&src_table_ident, &dst_table_ident) - .await - .unwrap(); - - assert_eq!( - catalog.list_tables(&src_namespace_ident).await.unwrap(), - vec![], - ); - - assert_eq!( - catalog.list_tables(&dst_namespace_ident).await.unwrap(), - vec![dst_table_ident], + format!( + "TableAlreadyExists => Table {:?} already exists.", + &table_ident + ) ); } @@ -2241,213 +2026,7 @@ mod tests { .await .unwrap_err() .to_string(), - format!("Unexpected => No such namespace: {non_existent_dst_namespace_ident:?}"), - ); - } - - #[tokio::test] - async fn test_rename_table_throws_error_if_src_table_doesnt_exist() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; - let namespace_ident = NamespaceIdent::new("n1".into()); - create_namespace(&catalog, &namespace_ident).await; - let src_table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into()); - let dst_table_ident = TableIdent::new(namespace_ident.clone(), "tbl2".into()); - - assert_eq!( - catalog - .rename_table(&src_table_ident, &dst_table_ident) - .await - .unwrap_err() - .to_string(), - format!("Unexpected => No such table: {src_table_ident:?}"), - ); - } - - #[tokio::test] - async fn test_rename_table_throws_error_if_dst_table_already_exists() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; - let namespace_ident = NamespaceIdent::new("n1".into()); - create_namespace(&catalog, &namespace_ident).await; - let src_table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into()); - let dst_table_ident = TableIdent::new(namespace_ident.clone(), "tbl2".into()); - create_tables(&catalog, vec![&src_table_ident, &dst_table_ident]).await; - - assert_eq!( - catalog - .rename_table(&src_table_ident, &dst_table_ident) - .await - .unwrap_err() - .to_string(), - format!("Unexpected => Table {:?} already exists.", &dst_table_ident), - ); - } - - #[tokio::test] - async fn test_drop_table_throws_error_if_table_not_exist() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await; - let namespace_ident = NamespaceIdent::new("a".into()); - let table_name = "tbl1"; - let table_ident = TableIdent::new(namespace_ident.clone(), table_name.into()); - create_namespace(&catalog, &namespace_ident).await; - - let err = catalog - .drop_table(&table_ident) - .await - .unwrap_err() - .to_string(); - assert_eq!( - err, - "Unexpected => No such table: TableIdent { namespace: NamespaceIdent([\"a\"]), name: \"tbl1\" }" - ); - } - - #[tokio::test] - async fn test_drop_table() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await; - let namespace_ident = NamespaceIdent::new("a".into()); - let table_name = "tbl1"; - let table_ident = TableIdent::new(namespace_ident.clone(), table_name.into()); - create_namespace(&catalog, &namespace_ident).await; - - let location = warehouse_loc.clone(); - let table_creation = TableCreation::builder() - .name(table_name.into()) - .location(location.clone()) - .schema(simple_table_schema()) - .build(); - - catalog - .create_table(&namespace_ident, table_creation) - .await - .unwrap(); - - let table = catalog.load_table(&table_ident).await.unwrap(); - assert_table_eq(&table, &table_ident, &simple_table_schema()); - - catalog.drop_table(&table_ident).await.unwrap(); - let err = catalog - .load_table(&table_ident) - .await - .unwrap_err() - .to_string(); - assert_eq!( - err, - "Unexpected => No such table: TableIdent { namespace: NamespaceIdent([\"a\"]), name: \"tbl1\" }" - ); - } - - #[tokio::test] - async fn test_register_table_throws_error_if_table_with_same_name_already_exists() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await; - let namespace_ident = NamespaceIdent::new("a".into()); - create_namespace(&catalog, &namespace_ident).await; - let table_name = "tbl1"; - let table_ident = TableIdent::new(namespace_ident.clone(), table_name.into()); - create_table(&catalog, &table_ident).await; - - assert_eq!( - catalog - .register_table(&table_ident, warehouse_loc) - .await - .unwrap_err() - .to_string(), - format!("Unexpected => Table {:?} already exists.", &table_ident) - ); - } - - #[tokio::test] - async fn test_register_table() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc.clone(), Some("iceberg")).await; - let namespace_ident = NamespaceIdent::new("a".into()); - create_namespace(&catalog, &namespace_ident).await; - - let table_name = "abc"; - let location = warehouse_loc.clone(); - let table_creation = TableCreation::builder() - .name(table_name.into()) - .location(location.clone()) - .schema(simple_table_schema()) - .build(); - - let table_ident = TableIdent::new(namespace_ident.clone(), table_name.into()); - let expected_table = catalog - .create_table(&namespace_ident, table_creation) - .await - .unwrap(); - - let metadata_location = expected_table - .metadata_location() - .expect("Expected metadata location to be set") - .to_string(); - - assert_table_eq(&expected_table, &table_ident, &simple_table_schema()); - - let _ = catalog.drop_table(&table_ident).await; - - let table = catalog - .register_table(&table_ident, metadata_location.clone()) - .await - .unwrap(); - - assert_eq!(table.identifier(), expected_table.identifier()); - assert_eq!(table.metadata_location(), Some(metadata_location.as_str())); - } - - #[tokio::test] - async fn test_update_table() { - let warehouse_loc = temp_path(); - let catalog = new_sql_catalog(warehouse_loc, Some("iceberg")).await; - - // Create a test namespace and table - let namespace_ident = NamespaceIdent::new("ns1".into()); - create_namespace(&catalog, &namespace_ident).await; - let table_ident = TableIdent::new(namespace_ident.clone(), "tbl1".into()); - create_table(&catalog, &table_ident).await; - - let table = catalog.load_table(&table_ident).await.unwrap(); - - // Store the original metadata location for comparison - let original_metadata_location = table.metadata_location().unwrap().to_string(); - - // Create a transaction to update the table - let tx = Transaction::new(&table); - let tx = tx - .update_table_properties() - .set("test_property".to_string(), "test_value".to_string()) - .apply(tx) - .unwrap(); - - // Commit the transaction to the catalog - let updated_table = tx.commit(&catalog).await.unwrap(); - - // Verify the update was successful - assert_eq!( - updated_table.metadata().properties().get("test_property"), - Some(&"test_value".to_string()) - ); - // Verify the metadata location has been updated - assert_ne!( - updated_table.metadata_location().unwrap(), - original_metadata_location.as_str() - ); - - // Load the table again from the catalog to verify changes were persisted - let reloaded = catalog.load_table(&table_ident).await.unwrap(); - - // Verify the reloaded table matches the updated table - assert_eq!( - reloaded.metadata().properties().get("test_property"), - Some(&"test_value".to_string()) - ); - assert_eq!( - reloaded.metadata_location(), - updated_table.metadata_location() + format!("NamespaceNotFound => No such namespace: {non_existent_dst_namespace_ident:?}"), ); } } diff --git a/crates/catalog/sql/src/error.rs b/crates/catalog/sql/src/error.rs index a08f755596..55e0e0a368 100644 --- a/crates/catalog/sql/src/error.rs +++ b/crates/catalog/sql/src/error.rs @@ -28,21 +28,21 @@ pub fn from_sqlx_error(error: sqlx::Error) -> Error { pub fn no_such_namespace_err(namespace: &NamespaceIdent) -> Result { Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::NamespaceNotFound, format!("No such namespace: {namespace:?}"), )) } pub fn no_such_table_err(table_ident: &TableIdent) -> Result { Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::TableNotFound, format!("No such table: {table_ident:?}"), )) } pub fn table_already_exists_err(table_ident: &TableIdent) -> Result { Err(Error::new( - ErrorKind::Unexpected, + ErrorKind::TableAlreadyExists, format!("Table {table_ident:?} already exists."), )) } diff --git a/crates/iceberg/Cargo.toml b/crates/iceberg/Cargo.toml index 78e5b7c64f..7e91050605 100644 --- a/crates/iceberg/Cargo.toml +++ b/crates/iceberg/Cargo.toml @@ -40,6 +40,7 @@ storage-s3 = ["opendal/services-s3", "reqsign"] [dependencies] +aes-gcm = { workspace = true } anyhow = { workspace = true } apache-avro = { workspace = true } array-init = { workspace = true } @@ -87,6 +88,7 @@ typed-builder = { workspace = true } typetag = { workspace = true } url = { workspace = true } uuid = { workspace = true } +zeroize = { workspace = true } zstd = { workspace = true } [dev-dependencies] diff --git a/crates/iceberg/src/arrow/reader.rs b/crates/iceberg/src/arrow/reader.rs index 93cc44dbcc..3696e88584 100644 --- a/crates/iceberg/src/arrow/reader.rs +++ b/crates/iceberg/src/arrow/reader.rs @@ -66,7 +66,7 @@ use crate::scan::{ArrowRecordBatchStream, FileScanTask, FileScanTaskStream}; use crate::spec::{ Datum, NameMapping, NestedField, PartitionSpec, PrimitiveType, Schema, SchemaRef, Struct, Type, }; -use crate::utils::available_parallelism; +use crate::util::available_parallelism; use crate::{Error, ErrorKind}; /// Default gap between byte ranges below which they are coalesced into a @@ -2737,7 +2737,7 @@ message schema { let props = WriterProperties::builder() .set_compression(Compression::SNAPPY) - .set_max_row_group_size(100) + .set_max_row_group_row_count(Some(100)) .build(); let file = File::create(&file_path).unwrap(); @@ -3052,7 +3052,7 @@ message schema { // Force each batch into its own row group let props = WriterProperties::builder() .set_compression(Compression::SNAPPY) - .set_max_row_group_size(100) + .set_max_row_group_row_count(Some(100)) .build(); let file = File::create(&data_file_path).unwrap(); @@ -3248,7 +3248,7 @@ message schema { // Force each batch into its own row group let props = WriterProperties::builder() .set_compression(Compression::SNAPPY) - .set_max_row_group_size(100) + .set_max_row_group_row_count(Some(100)) .build(); let file = File::create(&data_file_path).unwrap(); @@ -3472,7 +3472,7 @@ message schema { // Force each batch into its own row group let props = WriterProperties::builder() .set_compression(Compression::SNAPPY) - .set_max_row_group_size(100) + .set_max_row_group_row_count(Some(100)) .build(); let file = File::create(&data_file_path).unwrap(); @@ -3920,7 +3920,7 @@ message schema { let props = WriterProperties::builder() .set_compression(Compression::SNAPPY) .set_write_batch_size(2) - .set_max_row_group_size(2) + .set_max_row_group_row_count(Some(2)) .build(); let file = File::create(format!("{table_location}/1.parquet")).unwrap(); diff --git a/crates/iceberg/src/catalog/memory/catalog.rs b/crates/iceberg/src/catalog/memory/catalog.rs index e008de8050..8fa5c479c3 100644 --- a/crates/iceberg/src/catalog/memory/catalog.rs +++ b/crates/iceberg/src/catalog/memory/catalog.rs @@ -18,6 +18,7 @@ //! This module contains memory catalog implementation. use std::collections::HashMap; +use std::str::FromStr; use std::sync::Arc; use async_trait::async_trait; @@ -295,15 +296,15 @@ impl Catalog for MemoryCatalog { let metadata = TableMetadataBuilder::from_table_creation(table_creation)? .build()? .metadata; - let metadata_location = MetadataLocation::new_with_table_location(location).to_string(); + let metadata_location = MetadataLocation::new_with_metadata(location, &metadata); metadata.write_to(&self.file_io, &metadata_location).await?; - root_namespace_state.insert_new_table(&table_ident, metadata_location.clone())?; + root_namespace_state.insert_new_table(&table_ident, metadata_location.to_string())?; Table::builder() .file_io(self.file_io.clone()) - .metadata_location(metadata_location) + .metadata_location(metadata_location.to_string()) .metadata(metadata) .identifier(table_ident) .build() @@ -321,8 +322,19 @@ impl Catalog for MemoryCatalog { async fn drop_table(&self, table_ident: &TableIdent) -> Result<()> { let mut root_namespace_state = self.root_namespace_state.lock().await; - let metadata_location = root_namespace_state.remove_existing_table(table_ident)?; - self.file_io.delete(&metadata_location).await + root_namespace_state.remove_existing_table(table_ident)?; + Ok(()) + } + + async fn purge_table(&self, table_ident: &TableIdent) -> Result<()> { + let table_info = self.load_table(table_ident).await?; + self.drop_table(table_ident).await?; + crate::catalog::utils::drop_table_data( + table_info.file_io(), + table_info.metadata(), + table_info.metadata_location(), + ) + .await } /// Check if a table exists in the catalog. @@ -381,12 +393,11 @@ impl Catalog for MemoryCatalog { let staged_table = commit.apply(current_table)?; // Write table metadata to the new location + let metadata_location = + MetadataLocation::from_str(staged_table.metadata_location_result()?)?; staged_table .metadata() - .write_to( - staged_table.file_io(), - staged_table.metadata_location_result()?, - ) + .write_to(staged_table.file_io(), &metadata_location) .await?; // Flip the pointer to reference the new metadata file. diff --git a/crates/iceberg/src/catalog/metadata_location.rs b/crates/iceberg/src/catalog/metadata_location.rs index 3705ee42dc..acd041d5e1 100644 --- a/crates/iceberg/src/catalog/metadata_location.rs +++ b/crates/iceberg/src/catalog/metadata_location.rs @@ -15,41 +15,86 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; use std::fmt::Display; use std::str::FromStr; use uuid::Uuid; +use crate::compression::CompressionCodec; +use crate::spec::{TableMetadata, parse_metadata_file_compression}; use crate::{Error, ErrorKind, Result}; /// Helper for parsing a location of the format: `/metadata/-.metadata.json` +/// or with compression: `/metadata/-.gz.metadata.json` #[derive(Clone, Debug, PartialEq)] pub struct MetadataLocation { table_location: String, version: i32, id: Uuid, + compression_codec: CompressionCodec, } impl MetadataLocation { + /// Determines the compression codec from table properties. + /// Parse errors result in CompressionCodec::None. + fn compression_from_properties(properties: &HashMap) -> CompressionCodec { + parse_metadata_file_compression(properties).unwrap_or(CompressionCodec::None) + } + /// Creates a completely new metadata location starting at version 0. - /// Only used for creating a new table. For updates, see `with_next_version`. + /// Only used for creating a new table. For updates, see `next_version`. + #[deprecated( + since = "0.8.0", + note = "Use new_with_metadata instead to properly handle compression settings" + )] pub fn new_with_table_location(table_location: impl ToString) -> Self { Self { table_location: table_location.to_string(), version: 0, id: Uuid::new_v4(), + compression_codec: CompressionCodec::None, + } + } + + /// Creates a completely new metadata location starting at version 0, + /// with compression settings from the table metadata. + /// Only used for creating a new table. For updates, see `next_version`. + pub fn new_with_metadata(table_location: impl ToString, metadata: &TableMetadata) -> Self { + Self { + table_location: table_location.to_string(), + version: 0, + id: Uuid::new_v4(), + compression_codec: Self::compression_from_properties(metadata.properties()), } } /// Creates a new metadata location for an updated metadata file. + /// Increments the version number and generates a new UUID. pub fn with_next_version(&self) -> Self { Self { table_location: self.table_location.clone(), version: self.version + 1, id: Uuid::new_v4(), + compression_codec: self.compression_codec, + } + } + + /// Updates the metadata location with compression settings from the new metadata. + pub fn with_new_metadata(&self, new_metadata: &TableMetadata) -> Self { + Self { + table_location: self.table_location.clone(), + version: self.version, + id: self.id, + compression_codec: Self::compression_from_properties(new_metadata.properties()), } } + /// Returns the compression codec used for this metadata location. + pub fn compression_codec(&self) -> CompressionCodec { + self.compression_codec + } + fn parse_metadata_path_prefix(path: &str) -> Result { let prefix = path.strip_suffix("/metadata").ok_or(Error::new( ErrorKind::Unexpected, @@ -59,30 +104,43 @@ impl MetadataLocation { Ok(prefix.to_string()) } - /// Parses a file name of the format `-.metadata.json`. - fn parse_file_name(file_name: &str) -> Result<(i32, Uuid)> { - let (version, id) = file_name - .strip_suffix(".metadata.json") - .ok_or(Error::new( - ErrorKind::Unexpected, - format!("Invalid metadata file ending: {file_name}"), - ))? - .split_once('-') - .ok_or(Error::new( - ErrorKind::Unexpected, - format!("Invalid metadata file name format: {file_name}"), - ))?; - - Ok((version.parse::()?, Uuid::parse_str(id)?)) + /// Parses a file name of the format `-.metadata.json` + /// or with compression: `-.gz.metadata.json`. + /// Parse errors for compression codec result in CompressionCodec::None. + fn parse_file_name(file_name: &str) -> Result<(i32, Uuid, CompressionCodec)> { + let stripped = file_name.strip_suffix(".metadata.json").ok_or(Error::new( + ErrorKind::Unexpected, + format!("Invalid metadata file ending: {file_name}"), + ))?; + + // Check for compression suffix (e.g., .gz) + let gzip_suffix = CompressionCodec::gzip_default().suffix()?; + let (stripped, compression_codec) = if let Some(s) = stripped.strip_suffix(gzip_suffix) { + (s, CompressionCodec::gzip_default()) + } else { + (stripped, CompressionCodec::None) + }; + + let (version, id) = stripped.split_once('-').ok_or(Error::new( + ErrorKind::Unexpected, + format!("Invalid metadata file name format: {file_name}"), + ))?; + + Ok(( + version.parse::()?, + Uuid::parse_str(id)?, + compression_codec, + )) } } impl Display for MetadataLocation { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let suffix = self.compression_codec.suffix().unwrap_or(""); write!( f, - "{}/metadata/{:0>5}-{}.metadata.json", - self.table_location, self.version, self.id + "{}/metadata/{:0>5}-{}{}.metadata.json", + self.table_location, self.version, self.id, suffix ) } } @@ -97,23 +155,41 @@ impl FromStr for MetadataLocation { ))?; let prefix = Self::parse_metadata_path_prefix(path)?; - let (version, id) = Self::parse_file_name(file_name)?; + let (version, id, compression_codec) = Self::parse_file_name(file_name)?; Ok(MetadataLocation { table_location: prefix, version, id, + compression_codec, }) } } #[cfg(test)] mod test { + use std::collections::HashMap; use std::str::FromStr; use uuid::Uuid; - use crate::MetadataLocation; + use crate::compression::CompressionCodec; + use crate::spec::{Schema, TableMetadata, TableMetadataBuilder}; + use crate::{MetadataLocation, TableCreation}; + + fn create_test_metadata(properties: HashMap) -> TableMetadata { + let table_creation = TableCreation::builder() + .name("test_table".to_string()) + .location("/test/table".to_string()) + .schema(Schema::builder().build().unwrap()) + .properties(properties) + .build(); + TableMetadataBuilder::from_table_creation(table_creation) + .unwrap() + .build() + .unwrap() + .metadata + } #[test] fn test_metadata_location_from_string() { @@ -125,6 +201,7 @@ mod test { table_location: "".to_string(), version: 1234567, id: Uuid::from_str("2cd22b57-5127-4198-92ba-e4e67c79821b").unwrap(), + compression_codec: CompressionCodec::None, }), ), // Some prefix @@ -134,6 +211,7 @@ mod test { table_location: "/abc".to_string(), version: 1234567, id: Uuid::from_str("2cd22b57-5127-4198-92ba-e4e67c79821b").unwrap(), + compression_codec: CompressionCodec::None, }), ), // Longer prefix @@ -143,6 +221,7 @@ mod test { table_location: "/abc/def".to_string(), version: 1234567, id: Uuid::from_str("2cd22b57-5127-4198-92ba-e4e67c79821b").unwrap(), + compression_codec: CompressionCodec::None, }), ), // Prefix with special characters @@ -152,6 +231,7 @@ mod test { table_location: "https://127.0.0.1".to_string(), version: 1234567, id: Uuid::from_str("2cd22b57-5127-4198-92ba-e4e67c79821b").unwrap(), + compression_codec: CompressionCodec::None, }), ), // Another id @@ -161,6 +241,7 @@ mod test { table_location: "/abc".to_string(), version: 1234567, id: Uuid::from_str("81056704-ce5b-41c4-bb83-eb6408081af6").unwrap(), + compression_codec: CompressionCodec::None, }), ), // Version 0 @@ -170,6 +251,17 @@ mod test { table_location: "/abc".to_string(), version: 0, id: Uuid::from_str("2cd22b57-5127-4198-92ba-e4e67c79821b").unwrap(), + compression_codec: CompressionCodec::None, + }), + ), + // With gzip compression + ( + "/abc/metadata/1234567-2cd22b57-5127-4198-92ba-e4e67c79821b.gz.metadata.json", + Ok(MetadataLocation { + table_location: "/abc".to_string(), + version: 1234567, + id: Uuid::from_str("2cd22b57-5127-4198-92ba-e4e67c79821b").unwrap(), + compression_codec: CompressionCodec::gzip_default(), }), ), // Negative version @@ -216,8 +308,9 @@ mod test { #[test] fn test_metadata_location_with_next_version() { + let metadata = create_test_metadata(HashMap::new()); let test_cases = vec![ - MetadataLocation::new_with_table_location("/abc"), + MetadataLocation::new_with_metadata("/abc", &metadata), MetadataLocation::from_str( "/abc/def/metadata/1234567-2cd22b57-5127-4198-92ba-e4e67c79821b.metadata.json", ) @@ -233,4 +326,88 @@ mod test { assert_ne!(next.id, input.id); } } + + #[test] + fn test_with_next_version_preserves_compression() { + // Start from a parsed location with no compression + let location_none = MetadataLocation::from_str( + "/test/table/metadata/00000-2cd22b57-5127-4198-92ba-e4e67c79821b.metadata.json", + ) + .unwrap(); + assert_eq!(location_none.compression_codec, CompressionCodec::None); + + let next_none = location_none.with_next_version(); + assert_eq!(next_none.compression_codec, CompressionCodec::None); + assert_eq!(next_none.version, 1); + + // Start from a parsed location with gzip compression + let location_gzip = MetadataLocation::from_str( + "/test/table/metadata/00005-81056704-ce5b-41c4-bb83-eb6408081af6.gz.metadata.json", + ) + .unwrap(); + assert_eq!( + location_gzip.compression_codec, + CompressionCodec::gzip_default() + ); + + let next_gzip = location_gzip.with_next_version(); + assert_eq!( + next_gzip.compression_codec, + CompressionCodec::gzip_default() + ); + assert_eq!(next_gzip.version, 6); + } + + #[test] + fn test_with_new_metadata_updates_compression() { + // Start from a parsed location with no compression + let location = MetadataLocation::from_str( + "/test/table/metadata/00000-2cd22b57-5127-4198-92ba-e4e67c79821b.metadata.json", + ) + .unwrap(); + assert_eq!(location.compression_codec, CompressionCodec::None); + + // Update to gzip compression + let mut props_gzip = HashMap::new(); + props_gzip.insert( + "write.metadata.compression-codec".to_string(), + "gzip".to_string(), + ); + let metadata_gzip = create_test_metadata(props_gzip); + let updated_gzip = location.with_new_metadata(&metadata_gzip); + assert_eq!( + updated_gzip.compression_codec, + CompressionCodec::gzip_default() + ); + assert_eq!(updated_gzip.version, 0); + assert_eq!( + updated_gzip.to_string(), + "/test/table/metadata/00000-2cd22b57-5127-4198-92ba-e4e67c79821b.gz.metadata.json" + ); + + // Update back to no compression + let props_none = HashMap::new(); + let metadata_none = create_test_metadata(props_none); + let updated_none = updated_gzip.with_new_metadata(&metadata_none); + assert_eq!(updated_none.compression_codec, CompressionCodec::None); + assert_eq!(updated_none.version, 0); + assert_eq!( + updated_none.to_string(), + "/test/table/metadata/00000-2cd22b57-5127-4198-92ba-e4e67c79821b.metadata.json" + ); + + // Test explicit "none" codec + let mut props_explicit_none = HashMap::new(); + props_explicit_none.insert( + "write.metadata.compression-codec".to_string(), + "none".to_string(), + ); + let metadata_explicit_none = create_test_metadata(props_explicit_none); + let updated_explicit = updated_gzip.with_new_metadata(&metadata_explicit_none); + assert_eq!(updated_explicit.compression_codec, CompressionCodec::None); + assert_eq!( + updated_explicit.to_string(), + "/test/table/metadata/00000-2cd22b57-5127-4198-92ba-e4e67c79821b.metadata.json" + ); + } } diff --git a/crates/iceberg/src/catalog/mod.rs b/crates/iceberg/src/catalog/mod.rs index 8db9674ad9..f296cf2260 100644 --- a/crates/iceberg/src/catalog/mod.rs +++ b/crates/iceberg/src/catalog/mod.rs @@ -19,6 +19,7 @@ pub mod memory; mod metadata_location; +pub(crate) mod utils; use std::collections::HashMap; use std::fmt::{Debug, Display}; @@ -98,6 +99,14 @@ pub trait Catalog: Debug + Sync + Send { /// Drop a table from the catalog, or returns error if it doesn't exist. async fn drop_table(&self, table: &TableIdent) -> Result<()>; + /// Drop a table from the catalog and delete the underlying table data. + /// + /// Implementations should load the table metadata, drop the table + /// from the catalog, then delete all associated data and metadata files. + /// The [`drop_table_data`](utils::drop_table_data) utility function can + /// be used for the file cleanup step. + async fn purge_table(&self, table: &TableIdent) -> Result<()>; + /// Check if a table exists in the catalog. async fn table_exists(&self, table: &TableIdent) -> Result; @@ -382,13 +391,16 @@ impl TableCommit { metadata_builder = update.apply(metadata_builder)?; } - // Bump the version of metadata + // Build the new metadata + let new_metadata = metadata_builder.build()?.metadata; + let new_metadata_location = MetadataLocation::from_str(current_metadata_location)? .with_next_version() + .with_new_metadata(&new_metadata) .to_string(); Ok(table - .with_metadata(Arc::new(metadata_builder.build()?.metadata)) + .with_metadata(Arc::new(new_metadata)) .with_metadata_location(new_metadata_location)) } } diff --git a/crates/iceberg/src/catalog/utils.rs b/crates/iceberg/src/catalog/utils.rs new file mode 100644 index 0000000000..d450f9df80 --- /dev/null +++ b/crates/iceberg/src/catalog/utils.rs @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Utility functions for catalog operations. + +use std::collections::HashSet; + +use futures::{TryStreamExt, stream}; + +use crate::Result; +use crate::io::FileIO; +use crate::spec::TableMetadata; + +const DELETE_CONCURRENCY: usize = 10; + +/// Deletes all data and metadata files referenced by the given table metadata. +/// +/// This mirrors the Java implementation's `CatalogUtil.dropTableData`. +/// It collects all manifest files, manifest lists, previous metadata files, +/// statistics files, and partition statistics files, then deletes them. +/// +/// Data files within manifests are only deleted if the `gc.enabled` table +/// property is `true` (the default), to avoid corrupting other tables that +/// may share the same data files. +pub async fn drop_table_data( + io: &FileIO, + metadata: &TableMetadata, + metadata_location: Option<&str>, +) -> Result<()> { + let mut manifest_lists_to_delete: HashSet = HashSet::new(); + let mut manifests_to_delete: HashSet = HashSet::new(); + + // Load all manifest lists concurrently + let results: Vec<_> = + futures::future::try_join_all(metadata.snapshots().map(|snapshot| async { + let manifest_list = snapshot.load_manifest_list(io, metadata).await?; + Ok::<_, crate::Error>((snapshot.manifest_list().to_string(), manifest_list)) + })) + .await?; + + for (manifest_list_location, manifest_list) in results { + if !manifest_list_location.is_empty() { + manifest_lists_to_delete.insert(manifest_list_location); + } + for manifest_file in manifest_list.entries() { + manifests_to_delete.insert(manifest_file.manifest_path.clone()); + } + } + + // Delete data files only if gc.enabled is true, to avoid corrupting shared tables + if metadata.table_properties()?.gc_enabled { + delete_data_files(io, &manifests_to_delete).await?; + } + + // Delete manifest files + io.delete_stream(stream::iter(manifests_to_delete)).await?; + + // Delete manifest lists + io.delete_stream(stream::iter(manifest_lists_to_delete)) + .await?; + + // Delete previous metadata files + let prev_metadata_paths: Vec = metadata + .metadata_log() + .iter() + .map(|m| m.metadata_file.clone()) + .collect(); + io.delete_stream(stream::iter(prev_metadata_paths)).await?; + + // Delete statistics files + let stats_paths: Vec = metadata + .statistics_iter() + .map(|s| s.statistics_path.clone()) + .collect(); + io.delete_stream(stream::iter(stats_paths)).await?; + + // Delete partition statistics files + let partition_stats_paths: Vec = metadata + .partition_statistics_iter() + .map(|s| s.statistics_path.clone()) + .collect(); + io.delete_stream(stream::iter(partition_stats_paths)) + .await?; + + // Delete the current metadata file + if let Some(location) = metadata_location { + io.delete(location).await?; + } + + Ok(()) +} + +/// Reads manifests concurrently and deletes the data files referenced within. +async fn delete_data_files(io: &FileIO, manifest_paths: &HashSet) -> Result<()> { + stream::iter(manifest_paths.iter().map(Ok)) + .try_for_each_concurrent(DELETE_CONCURRENCY, |manifest_path| async move { + let input = io.new_input(manifest_path)?; + let manifest_content = input.read().await?; + let manifest = crate::spec::Manifest::parse_avro(&manifest_content)?; + + let data_file_paths = manifest + .entries() + .iter() + .map(|entry| entry.data_file.file_path().to_string()) + .collect::>(); + + io.delete_stream(stream::iter(data_file_paths)).await + }) + .await +} diff --git a/crates/iceberg/src/compression.rs b/crates/iceberg/src/compression.rs index 1218d81df6..929d9226e7 100644 --- a/crates/iceberg/src/compression.rs +++ b/crates/iceberg/src/compression.rs @@ -17,28 +17,101 @@ //! Compression codec support for data compression and decompression. +use std::fmt; use std::io::{Read, Write}; use flate2::Compression; use flate2::read::GzDecoder; use flate2::write::GzEncoder; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use crate::{Error, ErrorKind, Result}; +/// Default compression level for Zstandard (zstd). +const ZSTD_DEFAULT_LEVEL: u8 = 3; +/// Default compression level for Gzip. +const GZIP_DEFAULT_LEVEL: u8 = 6; +/// Maximum compression level for Gzip. +const GZIP_MAX_LEVEL: u8 = 9; + /// Data compression formats -#[derive(Debug, PartialEq, Eq, Clone, Copy, Default, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] +#[derive(Debug, PartialEq, Eq, Clone, Copy, Default)] pub enum CompressionCodec { #[default] /// No compression None, /// LZ4 single compression frame with content size present Lz4, - /// Zstandard single compression frame with content size present - Zstd, - /// Gzip compression - Gzip, + /// Zstandard single compression frame with content size present. + /// Level range is 0–22, where 0 means default compression level (not no compression). + /// Use [`CompressionCodec::zstd_default`] to construct with the default level. + Zstd(u8), + /// Gzip compression. Level range is 0–9, where 0 means no compression. + /// Use [`CompressionCodec::gzip_default`] to construct with the default level. + Gzip(u8), + /// Snappy compression + Snappy, +} + +impl CompressionCodec { + /// Returns a Zstd codec with the default compression level. + pub const fn zstd_default() -> Self { + CompressionCodec::Zstd(ZSTD_DEFAULT_LEVEL) + } + + /// Returns a Gzip codec with the default compression level. + pub const fn gzip_default() -> Self { + CompressionCodec::Gzip(GZIP_DEFAULT_LEVEL) + } + + /// Returns the codec name as used in serialization and error messages. + pub fn name(&self) -> &'static str { + match self { + CompressionCodec::None => "none", + CompressionCodec::Lz4 => "lz4", + CompressionCodec::Zstd(_) => "zstd", + CompressionCodec::Gzip(_) => "gzip", + CompressionCodec::Snappy => "snappy", + } + } +} + +// Note: serialize/deserialize do not round-trip the compression level. Iceberg configuration +// only the codec name (e.g. "zstd"), not the level, so deserialization always produces the +// default level. A `Zstd(5)` written to metadata will be read back as `Zstd(3)`. Some +// compression configuration (e.g. Avro metadata) has a separate level field alongside the codec name. +impl Serialize for CompressionCodec { + fn serialize(&self, serializer: S) -> std::result::Result { + serializer.serialize_str(self.name()) + } +} + +impl<'de> Deserialize<'de> for CompressionCodec { + fn deserialize>(deserializer: D) -> std::result::Result { + let s = String::deserialize(deserializer)?; + match s.to_lowercase().as_str() { + "none" => Ok(CompressionCodec::None), + "lz4" => Ok(CompressionCodec::Lz4), + "zstd" => Ok(CompressionCodec::zstd_default()), + "gzip" => Ok(CompressionCodec::gzip_default()), + "snappy" => Ok(CompressionCodec::Snappy), + other => Err(serde::de::Error::unknown_variant(other, &[ + "none", "lz4", "zstd", "gzip", "snappy", + ])), + } + } +} + +impl fmt::Display for CompressionCodec { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + CompressionCodec::None => write!(f, "None"), + CompressionCodec::Lz4 => write!(f, "Lz4"), + CompressionCodec::Zstd(level) => write!(f, "Zstd(level={level})"), + CompressionCodec::Gzip(level) => write!(f, "Gzip(level={level})"), + CompressionCodec::Snappy => write!(f, "Snappy"), + } + } } impl CompressionCodec { @@ -49,13 +122,17 @@ impl CompressionCodec { ErrorKind::FeatureUnsupported, "LZ4 decompression is not supported currently", )), - CompressionCodec::Zstd => Ok(zstd::stream::decode_all(&bytes[..])?), - CompressionCodec::Gzip => { + CompressionCodec::Zstd(_) => Ok(zstd::stream::decode_all(&bytes[..])?), + CompressionCodec::Gzip(_) => { let mut decoder = GzDecoder::new(&bytes[..]); let mut decompressed = Vec::new(); decoder.read_to_end(&mut decompressed)?; Ok(decompressed) } + CompressionCodec::Snappy => Err(Error::new( + ErrorKind::FeatureUnsupported, + "Snappy decompression is not supported currently", + )), } } @@ -66,25 +143,49 @@ impl CompressionCodec { ErrorKind::FeatureUnsupported, "LZ4 compression is not supported currently", )), - CompressionCodec::Zstd => { + CompressionCodec::Zstd(level) => { let writer = Vec::::new(); - let mut encoder = zstd::stream::Encoder::new(writer, 3)?; + let mut encoder = zstd::stream::Encoder::new(writer, *level as i32)?; encoder.include_checksum(true)?; encoder.set_pledged_src_size(Some(bytes.len().try_into()?))?; std::io::copy(&mut &bytes[..], &mut encoder)?; Ok(encoder.finish()?) } - CompressionCodec::Gzip => { - let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + CompressionCodec::Gzip(level) => { + let compression = Compression::new((*level).min(GZIP_MAX_LEVEL) as u32); + let mut encoder = GzEncoder::new(Vec::new(), compression); encoder.write_all(&bytes)?; Ok(encoder.finish()?) } + CompressionCodec::Snappy => Err(Error::new( + ErrorKind::FeatureUnsupported, + "Snappy compression is not supported currently", + )), } } pub(crate) fn is_none(&self) -> bool { matches!(self, CompressionCodec::None) } + + /// Returns the file extension suffix for this compression codec. + /// Returns empty string for None, ".gz" for Gzip. + /// + /// # Errors + /// + /// Returns an error for Lz4 and Zstd as they are not fully supported. + pub fn suffix(&self) -> Result<&'static str> { + match self { + CompressionCodec::None => Ok(""), + CompressionCodec::Gzip(_) => Ok(".gz"), + codec @ (CompressionCodec::Lz4 + | CompressionCodec::Zstd(_) + | CompressionCodec::Snappy) => Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("suffix not defined for {codec:?}"), + )), + } + } } #[cfg(test)] @@ -106,7 +207,10 @@ mod tests { async fn test_compression_codec_compress() { let bytes_vec = [0_u8; 100].to_vec(); - let compression_codecs = [CompressionCodec::Zstd, CompressionCodec::Gzip]; + let compression_codecs = [ + CompressionCodec::zstd_default(), + CompressionCodec::gzip_default(), + ]; for codec in compression_codecs { let compressed = codec.compress(bytes_vec.clone()).unwrap(); @@ -118,7 +222,10 @@ mod tests { #[tokio::test] async fn test_compression_codec_unsupported() { - let unsupported_codecs = [(CompressionCodec::Lz4, "LZ4")]; + let unsupported_codecs = [ + (CompressionCodec::Lz4, "LZ4"), + (CompressionCodec::Snappy, "Snappy"), + ]; let bytes_vec = [0_u8; 100].to_vec(); for (codec, name) in unsupported_codecs { @@ -133,4 +240,37 @@ mod tests { ); } } + + #[test] + fn test_suffix() { + assert_eq!(CompressionCodec::None.suffix().unwrap(), ""); + assert_eq!(CompressionCodec::gzip_default().suffix().unwrap(), ".gz"); + + assert!(CompressionCodec::Lz4.suffix().is_err()); + assert!(CompressionCodec::zstd_default().suffix().is_err()); + assert!(CompressionCodec::Snappy.suffix().is_err()); + + let lz4_err = CompressionCodec::Lz4.suffix().unwrap_err(); + assert!(lz4_err.to_string().contains("suffix not defined for Lz4")); + + let zstd_err = CompressionCodec::zstd_default().suffix().unwrap_err(); + assert!(zstd_err.to_string().contains("suffix not defined for Zstd")); + } + + #[test] + fn test_display() { + assert_eq!(CompressionCodec::None.to_string(), "None"); + assert_eq!(CompressionCodec::Lz4.to_string(), "Lz4"); + assert_eq!( + CompressionCodec::zstd_default().to_string(), + "Zstd(level=3)" + ); + assert_eq!(CompressionCodec::Zstd(5).to_string(), "Zstd(level=5)"); + assert_eq!( + CompressionCodec::gzip_default().to_string(), + "Gzip(level=6)" + ); + assert_eq!(CompressionCodec::Gzip(9).to_string(), "Gzip(level=9)"); + assert_eq!(CompressionCodec::Snappy.to_string(), "Snappy"); + } } diff --git a/crates/iceberg/src/encryption/crypto.rs b/crates/iceberg/src/encryption/crypto.rs new file mode 100644 index 0000000000..0b34580db8 --- /dev/null +++ b/crates/iceberg/src/encryption/crypto.rs @@ -0,0 +1,523 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Core cryptographic operations for Iceberg encryption. + +use std::fmt; +use std::str::FromStr; + +use aes_gcm::aead::generic_array::typenum::U12; +use aes_gcm::aead::rand_core::RngCore; +use aes_gcm::aead::{Aead, AeadCore, KeyInit, OsRng, Payload}; +use aes_gcm::{Aes128Gcm, Aes256Gcm, AesGcm, Nonce}; +use zeroize::Zeroizing; + +/// AES-192-GCM with 96-bit nonce. Not provided by `aes-gcm` but constructible +/// from the underlying primitives, same as `Aes128Gcm` and `Aes256Gcm`. +type Aes192Gcm = AesGcm; + +use crate::{Error, ErrorKind, Result}; + +/// Wrapper for sensitive byte data (encryption keys, DEKs, etc.) that: +/// - Zeroizes memory on drop +/// - Redacts content in [`Debug`] and [`Display`] output +/// - Provides only `&[u8]` access via [`as_bytes()`](Self::as_bytes) +/// - Uses `Box<[u8]>` (immutable boxed slice) since key bytes never grow +/// +/// Use this type for any struct field that holds plaintext key material. +/// Because its [`Debug`] impl always prints `[N bytes REDACTED]`, structs +/// containing `SensitiveBytes` can safely derive or implement `Debug` +/// without risk of leaking key material. +#[derive(Clone, PartialEq, Eq)] +struct SensitiveBytes(Zeroizing>); + +impl SensitiveBytes { + /// Wraps the given bytes as sensitive material. + pub fn new(bytes: impl Into>) -> Self { + Self(Zeroizing::new(bytes.into())) + } + + /// Returns the underlying bytes. + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } + + /// Returns the number of bytes. + #[allow(dead_code)] // Encryption work is ongoing so currently unused + pub fn len(&self) -> usize { + self.0.len() + } + + /// Returns `true` if the byte slice is empty. + #[allow(dead_code)] // Encryption work is ongoing so currently unused + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl fmt::Debug for SensitiveBytes { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "[{} bytes REDACTED]", self.0.len()) + } +} + +impl fmt::Display for SensitiveBytes { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "[{} bytes REDACTED]", self.0.len()) + } +} + +/// Supported AES key sizes for AES-GCM encryption. +/// +/// The Iceberg spec supports 128, 192, and 256-bit keys for AES-GCM. +/// See: +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AesKeySize { + /// 128-bit AES key (16 bytes) + Bits128 = 128, + /// 192-bit AES key (24 bytes) + Bits192 = 192, + /// 256-bit AES key (32 bytes) + Bits256 = 256, +} + +impl AesKeySize { + /// Returns the key length in bytes for this key size. + pub fn key_length(&self) -> usize { + match self { + Self::Bits128 => 16, + Self::Bits192 => 24, + Self::Bits256 => 32, + } + } + + /// Returns the key size for a given DEK length in bytes. + /// + /// Matches Java's `encryption.data-key-length` property semantics: + /// 16 → 128-bit, 24 → 192-bit, 32 → 256-bit. + pub fn from_key_length(len: usize) -> Result { + match len { + 16 => Ok(Self::Bits128), + 24 => Ok(Self::Bits192), + 32 => Ok(Self::Bits256), + _ => Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Unsupported data key length: {len} (must be 16, 24, or 32)"), + )), + } + } +} + +impl FromStr for AesKeySize { + type Err = Error; + + fn from_str(s: &str) -> Result { + match s { + "128" | "AES_GCM_128" | "AES128_GCM" => Ok(Self::Bits128), + "192" | "AES_GCM_192" | "AES192_GCM" => Ok(Self::Bits192), + "256" | "AES_GCM_256" | "AES256_GCM" => Ok(Self::Bits256), + _ => Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Unsupported AES key size: {s}"), + )), + } + } +} + +/// A secure encryption key that zeroes its memory on drop. +pub struct SecureKey { + key: SensitiveBytes, + key_size: AesKeySize, +} + +impl SecureKey { + /// Creates a new secure key with the specified key size. + /// + /// # Errors + /// Returns an error if the key length doesn't match the key size requirements. + pub fn new(key: &[u8]) -> Result { + let key_size = AesKeySize::from_key_length(key.len())?; + Ok(Self { + key: SensitiveBytes::new(key), + key_size, + }) + } + + /// Generates a new random key for the specified key size. + pub fn generate(key_size: AesKeySize) -> Self { + let mut key = vec![0u8; key_size.key_length()]; + OsRng.fill_bytes(&mut key); + Self { + key: SensitiveBytes::new(key), + key_size, + } + } + + /// Returns the AES key size. + pub fn key_size(&self) -> AesKeySize { + self.key_size + } + + /// Returns the key bytes. + pub fn as_bytes(&self) -> &[u8] { + self.key.as_bytes() + } +} + +/// AES-GCM cipher for encrypting and decrypting data. +pub struct AesGcmCipher { + key: SensitiveBytes, + key_size: AesKeySize, +} + +impl AesGcmCipher { + /// AES-GCM nonce length in bytes (96 bits). + pub const NONCE_LEN: usize = 12; + /// AES-GCM authentication tag length in bytes (128 bits). + pub const TAG_LEN: usize = 16; + + /// Creates a new cipher with the specified key. + pub fn new(key: SecureKey) -> Self { + Self { + key: SensitiveBytes::new(key.as_bytes()), + key_size: key.key_size(), + } + } + + /// Encrypts data using AES-GCM. + /// + /// # Arguments + /// * `plaintext` - The data to encrypt + /// * `aad` - Additional authenticated data (optional) + /// + /// # Returns + /// The encrypted data in the format: [12-byte nonce][ciphertext][16-byte auth tag] + /// This matches the Java implementation format for compatibility. + pub fn encrypt(&self, plaintext: &[u8], aad: Option<&[u8]>) -> Result> { + match self.key_size { + AesKeySize::Bits128 => { + encrypt_aes_gcm::(self.key.as_bytes(), plaintext, aad) + } + AesKeySize::Bits192 => { + encrypt_aes_gcm::(self.key.as_bytes(), plaintext, aad) + } + AesKeySize::Bits256 => { + encrypt_aes_gcm::(self.key.as_bytes(), plaintext, aad) + } + } + } + + /// Decrypts data using AES-GCM. + /// + /// # Arguments + /// * `ciphertext` - The encrypted data with format: [12-byte nonce][encrypted data][16-byte auth tag] + /// * `aad` - Additional authenticated data (must match encryption) + /// + /// # Returns + /// The decrypted plaintext. + pub fn decrypt(&self, ciphertext: &[u8], aad: Option<&[u8]>) -> Result> { + if ciphertext.len() < Self::NONCE_LEN + Self::TAG_LEN { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Ciphertext too short: expected at least {} bytes, got {}", + Self::NONCE_LEN + Self::TAG_LEN, + ciphertext.len() + ), + )); + } + + match self.key_size { + AesKeySize::Bits128 => { + decrypt_aes_gcm::(self.key.as_bytes(), ciphertext, aad) + } + AesKeySize::Bits192 => { + decrypt_aes_gcm::(self.key.as_bytes(), ciphertext, aad) + } + AesKeySize::Bits256 => { + decrypt_aes_gcm::(self.key.as_bytes(), ciphertext, aad) + } + } + } +} + +fn encrypt_aes_gcm(key_bytes: &[u8], plaintext: &[u8], aad: Option<&[u8]>) -> Result> +where C: Aead + AeadCore + KeyInit { + let cipher = C::new_from_slice(key_bytes).map_err(|e| { + Error::new(ErrorKind::DataInvalid, "Invalid AES key").with_source(anyhow::anyhow!(e)) + })?; + let nonce = C::generate_nonce(&mut OsRng); + + let ciphertext = if let Some(aad) = aad { + cipher.encrypt(&nonce, Payload { + msg: plaintext, + aad, + }) + } else { + cipher.encrypt(&nonce, plaintext.as_ref()) + } + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "AES-GCM encryption failed") + .with_source(anyhow::anyhow!(e)) + })?; + + // Prepend nonce to ciphertext (Java compatible format) + let mut result = Vec::with_capacity(nonce.len() + ciphertext.len()); + result.extend_from_slice(&nonce); + result.extend_from_slice(&ciphertext); + Ok(result) +} + +fn decrypt_aes_gcm(key_bytes: &[u8], ciphertext: &[u8], aad: Option<&[u8]>) -> Result> +where C: Aead + AeadCore + KeyInit { + let cipher = C::new_from_slice(key_bytes).map_err(|e| { + Error::new(ErrorKind::DataInvalid, "Invalid AES key").with_source(anyhow::anyhow!(e)) + })?; + + let nonce = Nonce::from_slice(&ciphertext[..AesGcmCipher::NONCE_LEN]); + let encrypted_data = &ciphertext[AesGcmCipher::NONCE_LEN..]; + + let plaintext = if let Some(aad) = aad { + cipher.decrypt(nonce, Payload { + msg: encrypted_data, + aad, + }) + } else { + cipher.decrypt(nonce, encrypted_data) + } + .map_err(|e| { + Error::new(ErrorKind::Unexpected, "AES-GCM decryption failed") + .with_source(anyhow::anyhow!(e)) + })?; + + Ok(plaintext) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_aes_key_size() { + assert_eq!(AesKeySize::Bits128.key_length(), 16); + assert_eq!(AesKeySize::Bits192.key_length(), 24); + assert_eq!(AesKeySize::Bits256.key_length(), 32); + + assert_eq!( + AesKeySize::from_key_length(16).unwrap(), + AesKeySize::Bits128 + ); + assert_eq!( + AesKeySize::from_key_length(24).unwrap(), + AesKeySize::Bits192 + ); + assert_eq!( + AesKeySize::from_key_length(32).unwrap(), + AesKeySize::Bits256 + ); + assert!(AesKeySize::from_key_length(8).is_err()); + + assert_eq!(AesKeySize::from_str("128").unwrap(), AesKeySize::Bits128); + assert_eq!( + AesKeySize::from_str("AES_GCM_128").unwrap(), + AesKeySize::Bits128 + ); + assert_eq!( + AesKeySize::from_str("AES_GCM_256").unwrap(), + AesKeySize::Bits256 + ); + assert!(AesKeySize::from_str("INVALID").is_err()); + } + + #[test] + fn test_secure_key() { + // Test key generation + let key1 = SecureKey::generate(AesKeySize::Bits128); + assert_eq!(key1.as_bytes().len(), 16); + assert_eq!(key1.key_size(), AesKeySize::Bits128); + + // Test key creation with validation + let valid_key = [0u8; 16]; + assert!(SecureKey::new(valid_key.as_slice()).is_ok()); + + let invalid_key = [0u8; 33]; + assert!(SecureKey::new(invalid_key.as_slice()).is_err()); + } + + #[test] + fn test_aes128_gcm_encryption_roundtrip() { + let key = SecureKey::generate(AesKeySize::Bits128); + let cipher = AesGcmCipher::new(key); + + let plaintext = b"Hello, Iceberg encryption!"; + let aad = b"additional authenticated data"; + + // Test without AAD + let ciphertext = cipher.encrypt(plaintext, None).unwrap(); + assert!(ciphertext.len() > plaintext.len() + 12); // nonce + tag + assert_ne!(&ciphertext[12..], plaintext); // encrypted portion differs + + let decrypted = cipher.decrypt(&ciphertext, None).unwrap(); + assert_eq!(decrypted, plaintext); + + // Test with AAD + let ciphertext = cipher.encrypt(plaintext, Some(aad)).unwrap(); + let decrypted = cipher.decrypt(&ciphertext, Some(aad)).unwrap(); + assert_eq!(decrypted, plaintext); + + // Test with wrong AAD fails + assert!(cipher.decrypt(&ciphertext, Some(b"wrong aad")).is_err()); + } + + #[test] + fn test_aes192_gcm_encryption_roundtrip() { + let key = SecureKey::generate(AesKeySize::Bits192); + let cipher = AesGcmCipher::new(key); + + let plaintext = b"Hello, Iceberg encryption!"; + let aad = b"additional authenticated data"; + + // Test without AAD + let ciphertext = cipher.encrypt(plaintext, None).unwrap(); + let decrypted = cipher.decrypt(&ciphertext, None).unwrap(); + assert_eq!(decrypted, plaintext); + + // Test with AAD + let ciphertext = cipher.encrypt(plaintext, Some(aad)).unwrap(); + let decrypted = cipher.decrypt(&ciphertext, Some(aad)).unwrap(); + assert_eq!(decrypted, plaintext); + + // Test with wrong AAD fails + assert!(cipher.decrypt(&ciphertext, Some(b"wrong aad")).is_err()); + } + + #[test] + fn test_aes256_gcm_encryption_roundtrip() { + let key = SecureKey::generate(AesKeySize::Bits256); + let cipher = AesGcmCipher::new(key); + + let plaintext = b"Hello, Iceberg encryption!"; + let aad = b"additional authenticated data"; + + // Test without AAD + let ciphertext = cipher.encrypt(plaintext, None).unwrap(); + let decrypted = cipher.decrypt(&ciphertext, None).unwrap(); + assert_eq!(decrypted, plaintext); + + // Test with AAD + let ciphertext = cipher.encrypt(plaintext, Some(aad)).unwrap(); + let decrypted = cipher.decrypt(&ciphertext, Some(aad)).unwrap(); + assert_eq!(decrypted, plaintext); + + // Test with wrong AAD fails + assert!(cipher.decrypt(&ciphertext, Some(b"wrong aad")).is_err()); + } + + #[test] + fn test_cross_key_size_incompatibility() { + let plaintext = b"Cross-key test"; + + let key128 = SecureKey::generate(AesKeySize::Bits128); + let key256 = SecureKey::generate(AesKeySize::Bits256); + + let cipher128 = AesGcmCipher::new(key128); + let cipher256 = AesGcmCipher::new(key256); + + // Ciphertext from 128-bit key should not decrypt with 256-bit key + let ciphertext = cipher128.encrypt(plaintext, None).unwrap(); + assert!(cipher256.decrypt(&ciphertext, None).is_err()); + } + + #[test] + fn test_encryption_with_empty_plaintext() { + let key = SecureKey::generate(AesKeySize::Bits128); + let cipher = AesGcmCipher::new(key); + + let plaintext = b""; + let ciphertext = cipher.encrypt(plaintext, None).unwrap(); + + // Even empty plaintext produces nonce + tag + assert_eq!(ciphertext.len(), 12 + 16); // 12-byte nonce + 16-byte tag + + let decrypted = cipher.decrypt(&ciphertext, None).unwrap(); + assert_eq!(decrypted, plaintext); + } + + #[test] + fn test_decryption_with_tampered_ciphertext() { + let key = SecureKey::generate(AesKeySize::Bits128); + let cipher = AesGcmCipher::new(key); + + let plaintext = b"Sensitive data"; + let mut ciphertext = cipher.encrypt(plaintext, None).unwrap(); + + // Tamper with the encrypted portion (after the nonce) + if ciphertext.len() > 12 { + ciphertext[12] ^= 0xFF; + } + + // Decryption should fail due to authentication tag mismatch + assert!(cipher.decrypt(&ciphertext, None).is_err()); + } + + #[test] + fn test_different_keys_produce_different_ciphertexts() { + let key1 = SecureKey::generate(AesKeySize::Bits128); + let key2 = SecureKey::generate(AesKeySize::Bits128); + + let cipher1 = AesGcmCipher::new(key1); + let cipher2 = AesGcmCipher::new(key2); + + let plaintext = b"Same plaintext"; + + let ciphertext1 = cipher1.encrypt(plaintext, None).unwrap(); + let ciphertext2 = cipher2.encrypt(plaintext, None).unwrap(); + + // Different keys should produce different ciphertexts (comparing the encrypted portion) + // Note: The nonces will also be different, but we're mainly interested in the encrypted data + assert_ne!(&ciphertext1[12..], &ciphertext2[12..]); + } + + #[test] + fn test_ciphertext_format_java_compatible() { + // Test that our ciphertext format matches Java's: [12-byte nonce][ciphertext][16-byte tag] + let key = SecureKey::generate(AesKeySize::Bits128); + let cipher = AesGcmCipher::new(key); + + let plaintext = b"Test data"; + let ciphertext = cipher.encrypt(plaintext, None).unwrap(); + + // Format should be: [12-byte nonce][encrypted_data + 16-byte GCM tag] + assert_eq!( + ciphertext.len(), + 12 + plaintext.len() + 16, + "Ciphertext should be nonce + plaintext + tag length" + ); + + // Verify we can decrypt by extracting nonce from the beginning + let nonce = &ciphertext[..12]; + assert_eq!(nonce.len(), 12, "Nonce should be 12 bytes"); + + // The rest is encrypted data + tag + let encrypted_with_tag = &ciphertext[12..]; + assert_eq!( + encrypted_with_tag.len(), + plaintext.len() + 16, + "Encrypted portion should be plaintext length + 16-byte tag" + ); + } +} diff --git a/crates/iceberg/src/encryption/mod.rs b/crates/iceberg/src/encryption/mod.rs new file mode 100644 index 0000000000..097f4f24e3 --- /dev/null +++ b/crates/iceberg/src/encryption/mod.rs @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Encryption module for Apache Iceberg. +//! +//! This module provides core cryptographic primitives for encrypting +//! and decrypting data in Iceberg tables. + +mod crypto; + +pub use crypto::{AesGcmCipher, AesKeySize, SecureKey}; diff --git a/crates/iceberg/src/expr/visitors/page_index_evaluator.rs b/crates/iceberg/src/expr/visitors/page_index_evaluator.rs index 66e2898532..96d1c651cd 100644 --- a/crates/iceberg/src/expr/visitors/page_index_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/page_index_evaluator.rs @@ -791,7 +791,7 @@ mod tests { use parquet::arrow::arrow_reader::{ ArrowReaderOptions, ParquetRecordBatchReaderBuilder, RowSelector, }; - use parquet::file::metadata::ParquetMetaData; + use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData}; use parquet::file::properties::WriterProperties; use rand::{Rng, thread_rng}; use tempfile::NamedTempFile; @@ -895,7 +895,7 @@ mod tests { writer.close().unwrap(); let file = temp_file.reopen().unwrap(); - let options = ArrowReaderOptions::new().with_page_index(true); + let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); let reader = ParquetRecordBatchReaderBuilder::try_new_with_options(file, options).unwrap(); let metadata = reader.metadata().clone(); @@ -936,7 +936,7 @@ mod tests { writer.close().unwrap(); let file = temp_file.reopen().unwrap(); - let options = ArrowReaderOptions::new().with_page_index(true); + let options = ArrowReaderOptions::new().with_page_index_policy(PageIndexPolicy::Required); let reader = ParquetRecordBatchReaderBuilder::try_new_with_options(file, options).unwrap(); let metadata = reader.metadata(); diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index afc8b207aa..d00ba1ba6a 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -19,6 +19,7 @@ use std::ops::Range; use std::sync::{Arc, OnceLock}; use bytes::Bytes; +use futures::{Stream, StreamExt}; use super::storage::config::{PROP_METADATA_LOCATION, PROP_TABLE_IDENT}; use super::storage::{ @@ -141,6 +142,18 @@ impl FileIO { self.get_storage()?.delete_prefix(path.as_ref()).await } + /// Delete multiple files from a stream of paths. + /// + /// # Arguments + /// + /// * paths: A stream of absolute paths starting with the scheme string used to construct [`FileIO`]. + pub async fn delete_stream( + &self, + paths: impl Stream + Send + 'static, + ) -> Result<()> { + self.get_storage()?.delete_stream(paths.boxed()).await + } + /// Check file exists. /// /// # Arguments diff --git a/crates/iceberg/src/io/storage/local_fs.rs b/crates/iceberg/src/io/storage/local_fs.rs index d6dd5b433b..e96e951baa 100644 --- a/crates/iceberg/src/io/storage/local_fs.rs +++ b/crates/iceberg/src/io/storage/local_fs.rs @@ -29,6 +29,8 @@ use std::sync::Arc; use async_trait::async_trait; use bytes::Bytes; +use futures::StreamExt; +use futures::stream::BoxStream; use serde::{Deserialize, Serialize}; use crate::io::{ @@ -200,6 +202,13 @@ impl Storage for LocalFsStorage { Ok(()) } + async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> { + while let Some(path) = paths.next().await { + self.delete(&path).await?; + } + Ok(()) + } + fn new_input(&self, path: &str) -> Result { Ok(InputFile::new(Arc::new(self.clone()), path.to_string())) } @@ -534,4 +543,61 @@ mod tests { assert!(path.exists()); } + + #[tokio::test] + async fn test_local_fs_storage_delete_stream() { + use futures::stream; + + let tmp_dir = TempDir::new().unwrap(); + let storage = LocalFsStorage::new(); + + // Create multiple files + let file1 = tmp_dir.path().join("file1.txt"); + let file2 = tmp_dir.path().join("file2.txt"); + let file3 = tmp_dir.path().join("file3.txt"); + + storage + .write(file1.to_str().unwrap(), Bytes::from("1")) + .await + .unwrap(); + storage + .write(file2.to_str().unwrap(), Bytes::from("2")) + .await + .unwrap(); + storage + .write(file3.to_str().unwrap(), Bytes::from("3")) + .await + .unwrap(); + + // Verify files exist + assert!(storage.exists(file1.to_str().unwrap()).await.unwrap()); + assert!(storage.exists(file2.to_str().unwrap()).await.unwrap()); + assert!(storage.exists(file3.to_str().unwrap()).await.unwrap()); + + // Delete multiple files using stream + let paths = vec![ + file1.to_str().unwrap().to_string(), + file2.to_str().unwrap().to_string(), + ]; + let path_stream = stream::iter(paths).boxed(); + storage.delete_stream(path_stream).await.unwrap(); + + // Verify deleted files no longer exist + assert!(!storage.exists(file1.to_str().unwrap()).await.unwrap()); + assert!(!storage.exists(file2.to_str().unwrap()).await.unwrap()); + + // Verify file3 still exists + assert!(storage.exists(file3.to_str().unwrap()).await.unwrap()); + } + + #[tokio::test] + async fn test_local_fs_storage_delete_stream_empty() { + use futures::stream; + + let storage = LocalFsStorage::new(); + + // Delete with empty stream should succeed + let path_stream = stream::iter(Vec::::new()).boxed(); + storage.delete_stream(path_stream).await.unwrap(); + } } diff --git a/crates/iceberg/src/io/storage/memory.rs b/crates/iceberg/src/io/storage/memory.rs index cb01ee4709..f33dbd07b1 100644 --- a/crates/iceberg/src/io/storage/memory.rs +++ b/crates/iceberg/src/io/storage/memory.rs @@ -28,6 +28,8 @@ use std::sync::{Arc, RwLock}; use async_trait::async_trait; use bytes::Bytes; +use futures::StreamExt; +use futures::stream::BoxStream; use serde::{Deserialize, Serialize}; use crate::io::{ @@ -220,6 +222,13 @@ impl Storage for MemoryStorage { Ok(()) } + async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> { + while let Some(path) = paths.next().await { + self.delete(&path).await?; + } + Ok(()) + } + fn new_input(&self, path: &str) -> Result { Ok(InputFile::new(Arc::new(self.clone()), path.to_string())) } @@ -594,4 +603,56 @@ mod tests { assert_eq!(storage.read("/path/to/file").await.unwrap(), content); assert_eq!(storage.read("path/to/file").await.unwrap(), content); } + + #[tokio::test] + async fn test_memory_storage_delete_stream() { + use futures::stream; + + let storage = MemoryStorage::new(); + + // Create multiple files + storage + .write("memory://file1.txt", Bytes::from("1")) + .await + .unwrap(); + storage + .write("memory://file2.txt", Bytes::from("2")) + .await + .unwrap(); + storage + .write("memory://file3.txt", Bytes::from("3")) + .await + .unwrap(); + + // Verify files exist + assert!(storage.exists("memory://file1.txt").await.unwrap()); + assert!(storage.exists("memory://file2.txt").await.unwrap()); + assert!(storage.exists("memory://file3.txt").await.unwrap()); + + // Delete multiple files using stream + let paths = vec![ + "memory://file1.txt".to_string(), + "memory://file2.txt".to_string(), + ]; + let path_stream = stream::iter(paths).boxed(); + storage.delete_stream(path_stream).await.unwrap(); + + // Verify deleted files no longer exist + assert!(!storage.exists("memory://file1.txt").await.unwrap()); + assert!(!storage.exists("memory://file2.txt").await.unwrap()); + + // Verify file3 still exists + assert!(storage.exists("memory://file3.txt").await.unwrap()); + } + + #[tokio::test] + async fn test_memory_storage_delete_stream_empty() { + use futures::stream; + + let storage = MemoryStorage::new(); + + // Delete with empty stream should succeed + let path_stream = stream::iter(Vec::::new()).boxed(); + storage.delete_stream(path_stream).await.unwrap(); + } } diff --git a/crates/iceberg/src/io/storage/mod.rs b/crates/iceberg/src/io/storage/mod.rs index 1ebb1430a0..97e612f5bc 100644 --- a/crates/iceberg/src/io/storage/mod.rs +++ b/crates/iceberg/src/io/storage/mod.rs @@ -28,6 +28,7 @@ use std::sync::Arc; use async_trait::async_trait; use bytes::Bytes; pub use config::*; +use futures::stream::BoxStream; pub use local_fs::{LocalFsStorage, LocalFsStorageFactory}; pub use memory::{MemoryStorage, MemoryStorageFactory}; #[cfg(feature = "storage-s3")] @@ -99,6 +100,9 @@ pub trait Storage: Debug + Send + Sync { /// Delete all files with the given prefix async fn delete_prefix(&self, path: &str) -> Result<()>; + /// Delete multiple files from a stream of paths. + async fn delete_stream(&self, paths: BoxStream<'static, String>) -> Result<()>; + /// Create a new input file for reading fn new_input(&self, path: &str) -> Result; diff --git a/crates/iceberg/src/io/storage/opendal/mod.rs b/crates/iceberg/src/io/storage/opendal/mod.rs index 2bc56d31c4..ca0ce8a245 100644 --- a/crates/iceberg/src/io/storage/opendal/mod.rs +++ b/crates/iceberg/src/io/storage/opendal/mod.rs @@ -25,6 +25,8 @@ use async_trait::async_trait; #[cfg(feature = "storage-azdls")] use azdls::AzureStorageScheme; use bytes::Bytes; +use futures::StreamExt; +use futures::stream::BoxStream; use opendal::Operator; use opendal::layers::RetryLayer; #[cfg(feature = "storage-azdls")] @@ -438,6 +440,13 @@ impl Storage for OpenDalStorage { Ok(op.delete(relative_path).await?) } + async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> { + while let Some(path) = paths.next().await { + self.delete(&path).await?; + } + Ok(()) + } + async fn delete_prefix(&self, path: &str) -> Result<()> { let (op, relative_path) = self.create_operator(&path)?; let path = if relative_path.ends_with('/') { diff --git a/crates/iceberg/src/lib.rs b/crates/iceberg/src/lib.rs index 46e106aeff..ae0708146b 100644 --- a/crates/iceberg/src/lib.rs +++ b/crates/iceberg/src/lib.rs @@ -71,6 +71,7 @@ pub use error::{Error, ErrorKind, Result}; mod catalog; +pub use catalog::utils::drop_table_data; pub use catalog::*; pub mod table; @@ -92,13 +93,12 @@ mod runtime; pub mod arrow; pub(crate) mod delete_file_index; +pub mod encryption; pub mod test_utils; -mod utils; pub mod writer; mod delete_vector; pub mod metadata_columns; pub mod puffin; - /// Utility functions and modules. pub mod util; diff --git a/crates/iceberg/src/puffin/metadata.rs b/crates/iceberg/src/puffin/metadata.rs index 1d39cf249b..e2dfc10c23 100644 --- a/crates/iceberg/src/puffin/metadata.rs +++ b/crates/iceberg/src/puffin/metadata.rs @@ -985,6 +985,9 @@ mod tests { assert!(result.is_ok()); let metadata = result.unwrap(); assert_eq!(metadata.blobs.len(), 1); - assert_eq!(metadata.blobs[0].compression_codec, CompressionCodec::Gzip); + assert_eq!( + metadata.blobs[0].compression_codec, + CompressionCodec::gzip_default() + ); } } diff --git a/crates/iceberg/src/puffin/mod.rs b/crates/iceberg/src/puffin/mod.rs index 854d4070ff..0e054cac51 100644 --- a/crates/iceberg/src/puffin/mod.rs +++ b/crates/iceberg/src/puffin/mod.rs @@ -26,30 +26,22 @@ pub use blob::{APACHE_DATASKETCHES_THETA_V1, Blob, DELETION_VECTOR_V1}; pub use crate::compression::CompressionCodec; -/// Compression codecs supported by the Puffin spec. -const SUPPORTED_PUFFIN_CODECS: &[CompressionCodec] = &[ - CompressionCodec::None, - CompressionCodec::Lz4, - CompressionCodec::Zstd, -]; - /// Validates that the compression codec is supported for Puffin files. /// Returns an error if the codec is not supported. fn validate_puffin_compression(codec: CompressionCodec) -> Result<()> { - if !SUPPORTED_PUFFIN_CODECS.contains(&codec) { - let supported_names: Vec = SUPPORTED_PUFFIN_CODECS - .iter() - .map(|c| format!("{c:?}")) - .collect(); - return Err(Error::new( + match codec { + CompressionCodec::None | CompressionCodec::Lz4 | CompressionCodec::Zstd(_) => Ok(()), + other => Err(Error::new( ErrorKind::DataInvalid, format!( - "Compression codec {codec:?} is not supported for Puffin files. Only {} are supported.", - supported_names.join(", ") + "Compression codec {} is not supported for Puffin files. Only {}, {}, and {} are supported.", + other.name(), + CompressionCodec::None.name(), + CompressionCodec::Lz4.name(), + CompressionCodec::zstd_default().name() ), - )); + )), } - Ok(()) } mod metadata; @@ -70,12 +62,13 @@ mod tests { #[test] fn test_puffin_codec_validation() { - // All codecs in SUPPORTED_PUFFIN_CODECS should be valid - for codec in SUPPORTED_PUFFIN_CODECS { - assert!(validate_puffin_compression(*codec).is_ok()); - } + // Supported codecs + assert!(validate_puffin_compression(CompressionCodec::None).is_ok()); + assert!(validate_puffin_compression(CompressionCodec::Lz4).is_ok()); + assert!(validate_puffin_compression(CompressionCodec::zstd_default()).is_ok()); + assert!(validate_puffin_compression(CompressionCodec::Zstd(5)).is_ok()); - // Gzip should not be supported for Puffin files - assert!(validate_puffin_compression(CompressionCodec::Gzip).is_err()); + // Unsupported codecs + assert!(validate_puffin_compression(CompressionCodec::gzip_default()).is_err()); } } diff --git a/crates/iceberg/src/puffin/reader.rs b/crates/iceberg/src/puffin/reader.rs index d272f02d41..0aced4186f 100644 --- a/crates/iceberg/src/puffin/reader.rs +++ b/crates/iceberg/src/puffin/reader.rs @@ -144,7 +144,7 @@ mod tests { sequence_number: 1, offset: 4, length: 10, - compression_codec: CompressionCodec::Gzip, + compression_codec: CompressionCodec::gzip_default(), properties: HashMap::new(), }; @@ -153,7 +153,7 @@ mod tests { assert!(result.is_err()); let err = result.unwrap_err(); assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!(err.to_string().contains("Gzip")); + assert!(err.to_string().contains("gzip")); assert!( err.to_string() .contains("is not supported for Puffin files") diff --git a/crates/iceberg/src/puffin/test_utils.rs b/crates/iceberg/src/puffin/test_utils.rs index 39fecc6f80..e0844e2002 100644 --- a/crates/iceberg/src/puffin/test_utils.rs +++ b/crates/iceberg/src/puffin/test_utils.rs @@ -77,7 +77,7 @@ pub(crate) fn zstd_compressed_metric_blob_0_metadata() -> BlobMetadata { sequence_number: METRIC_BLOB_0_SEQUENCE_NUMBER, offset: 4, length: 22, - compression_codec: CompressionCodec::Zstd, + compression_codec: CompressionCodec::zstd_default(), properties: HashMap::new(), } } @@ -134,7 +134,7 @@ pub(crate) fn zstd_compressed_metric_blob_1_metadata() -> BlobMetadata { sequence_number: METRIC_BLOB_1_SEQUENCE_NUMBER, offset: 26, length: 77, - compression_codec: CompressionCodec::Zstd, + compression_codec: CompressionCodec::zstd_default(), properties: HashMap::new(), } } diff --git a/crates/iceberg/src/puffin/writer.rs b/crates/iceberg/src/puffin/writer.rs index 30b97f09dd..4af4970b04 100644 --- a/crates/iceberg/src/puffin/writer.rs +++ b/crates/iceberg/src/puffin/writer.rs @@ -251,7 +251,8 @@ mod tests { async fn test_write_zstd_compressed_metric_data() { let temp_dir = TempDir::new().unwrap(); let blobs = vec![blob_0(), blob_1()]; - let blobs_with_compression = blobs_with_compression(blobs.clone(), CompressionCodec::Zstd); + let blobs_with_compression = + blobs_with_compression(blobs.clone(), CompressionCodec::zstd_default()); let input_file = write_puffin_file(&temp_dir, blobs_with_compression, file_properties()) .await @@ -323,7 +324,8 @@ mod tests { async fn test_zstd_compressed_metric_data_is_bit_identical_to_java_generated_file() { let temp_dir = TempDir::new().unwrap(); let blobs = vec![blob_0(), blob_1()]; - let blobs_with_compression = blobs_with_compression(blobs, CompressionCodec::Zstd); + let blobs_with_compression = + blobs_with_compression(blobs, CompressionCodec::zstd_default()); assert_files_are_bit_identical( write_puffin_file(&temp_dir, blobs_with_compression, file_properties()) @@ -338,14 +340,15 @@ mod tests { async fn test_gzip_compression_rejected() { let temp_dir = TempDir::new().unwrap(); let blobs = vec![blob_0()]; - let blobs_with_compression = blobs_with_compression(blobs, CompressionCodec::Gzip); + let blobs_with_compression = + blobs_with_compression(blobs, CompressionCodec::gzip_default()); let result = write_puffin_file(&temp_dir, blobs_with_compression, file_properties()).await; assert!(result.is_err()); let err = result.unwrap_err(); assert_eq!(err.kind(), ErrorKind::DataInvalid); - assert!(err.to_string().contains("Gzip")); + assert!(err.to_string().contains("gzip")); assert!( err.to_string() .contains("is not supported for Puffin files") diff --git a/crates/iceberg/src/scan/incremental/mod.rs b/crates/iceberg/src/scan/incremental/mod.rs index 59c5d75811..74b9090f4e 100644 --- a/crates/iceberg/src/scan/incremental/mod.rs +++ b/crates/iceberg/src/scan/incremental/mod.rs @@ -40,8 +40,8 @@ use crate::spec::{ DataContentType, ManifestEntryRef, ManifestStatus, Snapshot, SnapshotRef, TableMetadataRef, }; use crate::table::Table; +use crate::util::available_parallelism; use crate::util::snapshot::ancestors_between; -use crate::utils::available_parallelism; use crate::{Error, ErrorKind, Result}; mod context; diff --git a/crates/iceberg/src/scan/mod.rs b/crates/iceberg/src/scan/mod.rs index 8157b2753d..97d7d8afb1 100644 --- a/crates/iceberg/src/scan/mod.rs +++ b/crates/iceberg/src/scan/mod.rs @@ -43,7 +43,7 @@ use crate::metadata_columns::{ use crate::runtime::spawn; use crate::spec::{DataContentType, SnapshotRef}; use crate::table::Table; -use crate::utils::available_parallelism; +use crate::util::available_parallelism; use crate::{Error, ErrorKind, Result}; /// A stream of arrow [`RecordBatch`]es. @@ -773,6 +773,39 @@ pub mod tests { } } + /// Creates a fixture with 5 snapshots chained as: + /// S1 (root) -> S2 -> S3 -> S4 -> S5 (current) + /// Useful for testing snapshot history traversal. + pub fn new_with_deep_history() -> Self { + let tmp_dir = TempDir::new().unwrap(); + let table_location = tmp_dir.path().join("table1"); + let table_metadata1_location = table_location.join("metadata/v1.json"); + + let file_io = FileIO::new_with_fs(); + + let table_metadata = { + let json_str = fs::read_to_string(format!( + "{}/testdata/example_table_metadata_v2_deep_history.json", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + serde_json::from_str::(&json_str).unwrap() + }; + + let table = Table::builder() + .metadata(table_metadata) + .identifier(TableIdent::from_strs(["db", "table1"]).unwrap()) + .file_io(file_io.clone()) + .metadata_location(table_metadata1_location.as_os_str().to_str().unwrap()) + .build() + .unwrap(); + + Self { + table_location: table_location.to_str().unwrap().to_string(), + table, + } + } + pub fn new_unpartitioned() -> Self { let tmp_dir = TempDir::new().unwrap(); let table_location = tmp_dir.path().join("table1"); diff --git a/crates/iceberg/src/spec/manifest/writer.rs b/crates/iceberg/src/spec/manifest/writer.rs index cc5ef737fb..1b3b605fd8 100644 --- a/crates/iceberg/src/spec/manifest/writer.rs +++ b/crates/iceberg/src/spec/manifest/writer.rs @@ -32,10 +32,14 @@ use crate::spec::manifest::_serde::{ManifestEntryV1, ManifestEntryV2}; use crate::spec::manifest::{manifest_schema_v1, manifest_schema_v2}; use crate::spec::{ DataContentType, DataFile, FieldSummary, ManifestEntry, ManifestFile, ManifestMetadata, - ManifestStatus, PrimitiveLiteral, SchemaRef, StructType, UNASSIGNED_SNAPSHOT_ID, + ManifestStatus, PrimitiveLiteral, SchemaRef, StructType, }; use crate::{Error, ErrorKind}; +/// Placeholder for snapshot ID. The field with this value must be replaced +/// with the actual snapshot ID before it is committed. +const UNASSIGNED_SNAPSHOT_ID: i64 = -1; + /// The builder used to create a [`ManifestWriter`]. pub struct ManifestWriterBuilder { output: OutputFile, diff --git a/crates/iceberg/src/spec/mod.rs b/crates/iceberg/src/spec/mod.rs index 707ebbb630..b23ca1eda0 100644 --- a/crates/iceberg/src/spec/mod.rs +++ b/crates/iceberg/src/spec/mod.rs @@ -50,6 +50,7 @@ pub use sort::*; pub use statistic_file::*; pub use table_metadata::*; pub(crate) use table_metadata_builder::FIRST_FIELD_ID; +pub(crate) use table_properties::parse_metadata_file_compression; pub use table_properties::*; pub use transform::*; pub(crate) use values::decimal_utils; diff --git a/crates/iceberg/src/spec/snapshot.rs b/crates/iceberg/src/spec/snapshot.rs index f60579e014..72b5417c47 100644 --- a/crates/iceberg/src/spec/snapshot.rs +++ b/crates/iceberg/src/spec/snapshot.rs @@ -33,8 +33,6 @@ use crate::{Error, ErrorKind}; /// The ref name of the main branch of the table. pub const MAIN_BRANCH: &str = "main"; -/// Placeholder for snapshot ID. The field with this value must be replaced with the actual snapshot ID before it is committed. -pub const UNASSIGNED_SNAPSHOT_ID: i64 = -1; /// Reference to [`Snapshot`]. pub type SnapshotRef = Arc; diff --git a/crates/iceberg/src/spec/table_metadata.rs b/crates/iceberg/src/spec/table_metadata.rs index 3e6374d58d..607fd98350 100644 --- a/crates/iceberg/src/spec/table_metadata.rs +++ b/crates/iceberg/src/spec/table_metadata.rs @@ -35,8 +35,9 @@ pub use super::table_metadata_builder::{TableMetadataBuildResult, TableMetadataB use super::{ DEFAULT_PARTITION_SPEC_ID, PartitionSpecRef, PartitionStatisticsFile, SchemaId, SchemaRef, SnapshotRef, SnapshotRetention, SortOrder, SortOrderRef, StatisticsFile, StructType, - TableProperties, + TableProperties, parse_metadata_file_compression, }; +use crate::catalog::MetadataLocation; use crate::compression::CompressionCodec; use crate::error::{Result, timestamp_ms_to_utc}; use crate::io::FileIO; @@ -46,6 +47,9 @@ use crate::{Error, ErrorKind}; static MAIN_BRANCH: &str = "main"; pub(crate) static ONE_MINUTE_MS: i64 = 60_000; +/// Sentinel value used by the Java implementation and older metadata files +/// to represent a missing/empty current snapshot ID. During deserialization, +/// this value is normalized to `None`. pub(crate) static EMPTY_SNAPSHOT_ID: i64 = -1; pub(crate) static INITIAL_SEQUENCE_NUMBER: i64 = 0; @@ -360,6 +364,18 @@ impl TableMetadata { &self.properties } + /// Returns the metadata compression codec from table properties. + /// + /// Returns `CompressionCodec::None` if compression is disabled or not configured. + /// Returns `CompressionCodec::Gzip` if gzip compression is enabled. + /// + /// # Errors + /// + /// Returns an error if the compression codec property has an invalid value. + pub fn metadata_compression_codec(&self) -> Result { + parse_metadata_file_compression(&self.properties) + } + /// Returns typed table properties parsed from the raw properties map with defaults. pub fn table_properties(&self) -> Result { TableProperties::try_from(&self.properties).map_err(|e| { @@ -444,7 +460,7 @@ impl TableMetadata { && metadata_content[0] == 0x1F && metadata_content[1] == 0x8B { - let decompressed_data = CompressionCodec::Gzip + let decompressed_data = CompressionCodec::gzip_default() .decompress(metadata_content.to_vec()) .map_err(|e| { Error::new( @@ -466,11 +482,39 @@ impl TableMetadata { pub async fn write_to( &self, file_io: &FileIO, - metadata_location: impl AsRef, + metadata_location: &MetadataLocation, ) -> Result<()> { + let json_data = serde_json::to_vec(self)?; + + // Check if compression codec from properties matches the one in metadata_location + let codec = parse_metadata_file_compression(&self.properties)?; + + if codec != metadata_location.compression_codec() { + return Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Compression codec mismatch: metadata_location has {:?}, but table properties specify {:?}", + metadata_location.compression_codec(), + codec + ), + )); + } + + // Apply compression based on codec + let data_to_write = match codec { + CompressionCodec::Gzip(_) => codec.compress(json_data)?, + CompressionCodec::None => json_data, + _ => { + return Err(Error::new( + ErrorKind::DataInvalid, + format!("Unsupported metadata compression codec: {codec:?}"), + )); + } + }; + file_io - .new_output(metadata_location)? - .write(serde_json::to_vec(self)?.into()) + .new_output(metadata_location.to_string())? + .write(data_to_write.into()) .await } @@ -724,8 +768,8 @@ pub(super) mod _serde { use uuid::Uuid; use super::{ - DEFAULT_PARTITION_SPEC_ID, FormatVersion, MAIN_BRANCH, MetadataLog, SnapshotLog, - TableMetadata, + DEFAULT_PARTITION_SPEC_ID, EMPTY_SNAPSHOT_ID, FormatVersion, MAIN_BRANCH, MetadataLog, + SnapshotLog, TableMetadata, }; use crate::spec::schema::_serde::{SchemaV1, SchemaV2}; use crate::spec::snapshot::_serde::{SnapshotV1, SnapshotV2, SnapshotV3}; @@ -909,7 +953,7 @@ pub(super) mod _serde { encryption_keys, snapshots, } = value; - let current_snapshot_id = if let &Some(-1) = &value.current_snapshot_id { + let current_snapshot_id = if value.current_snapshot_id == Some(EMPTY_SNAPSHOT_ID) { None } else { value.current_snapshot_id @@ -1022,7 +1066,7 @@ pub(super) mod _serde { fn try_from(value: TableMetadataV2) -> Result { let snapshots = value.snapshots; let value = value.shared; - let current_snapshot_id = if let &Some(-1) = &value.current_snapshot_id { + let current_snapshot_id = if value.current_snapshot_id == Some(EMPTY_SNAPSHOT_ID) { None } else { value.current_snapshot_id @@ -1129,7 +1173,7 @@ pub(super) mod _serde { impl TryFrom for TableMetadata { type Error = Error; fn try_from(value: TableMetadataV1) -> Result { - let current_snapshot_id = if let &Some(-1) = &value.current_snapshot_id { + let current_snapshot_id = if value.current_snapshot_id == Some(EMPTY_SNAPSHOT_ID) { None } else { value.current_snapshot_id @@ -1567,6 +1611,7 @@ mod tests { use uuid::Uuid; use super::{FormatVersion, MetadataLog, SnapshotLog, TableMetadataBuilder}; + use crate::catalog::MetadataLocation; use crate::compression::CompressionCodec; use crate::io::FileIO; use crate::spec::table_metadata::TableMetadata; @@ -1574,7 +1619,7 @@ mod tests { BlobMetadata, EncryptedKey, INITIAL_ROW_ID, Literal, NestedField, NullOrder, Operation, PartitionSpec, PartitionStatisticsFile, PrimitiveLiteral, PrimitiveType, Schema, Snapshot, SnapshotReference, SnapshotRetention, SortDirection, SortField, SortOrder, StatisticsFile, - Summary, Transform, Type, UnboundPartitionField, + Summary, TableProperties, Transform, Type, UnboundPartitionField, }; use crate::{ErrorKind, TableCreation}; @@ -3258,6 +3303,18 @@ mod tests { check_table_metadata_serde(&metadata, expected); } + #[test] + fn test_empty_snapshot_id_is_normalized_to_none() { + let metadata = + fs::read_to_string("testdata/table_metadata/TableMetadataV1Valid.json").unwrap(); + let deserialized: TableMetadata = serde_json::from_str(&metadata).unwrap(); + assert_eq!( + deserialized.current_snapshot_id(), + None, + "current_snapshot_id of -1 should be deserialized as None" + ); + } + #[test] fn test_table_metadata_v1_compat() { let metadata = @@ -3547,7 +3604,8 @@ mod tests { let original_metadata: TableMetadata = get_test_table_metadata("TableMetadataV2Valid.json"); // Define the metadata location - let metadata_location = format!("{temp_path}/metadata.json"); + let metadata_location = MetadataLocation::new_with_metadata(temp_path, &original_metadata); + let metadata_location_str = metadata_location.to_string(); // Write the metadata original_metadata @@ -3556,10 +3614,10 @@ mod tests { .unwrap(); // Verify the file exists - assert!(fs::metadata(&metadata_location).is_ok()); + assert!(fs::metadata(&metadata_location_str).is_ok()); // Read the metadata back - let read_metadata = TableMetadata::read_from(&file_io, &metadata_location) + let read_metadata = TableMetadata::read_from(&file_io, &metadata_location_str) .await .unwrap(); @@ -3575,7 +3633,7 @@ mod tests { let original_metadata: TableMetadata = get_test_table_metadata("TableMetadataV2Valid.json"); let json = serde_json::to_string(&original_metadata).unwrap(); - let compressed = CompressionCodec::Gzip + let compressed = CompressionCodec::gzip_default() .compress(json.into_bytes()) .expect("failed to compress metadata"); std::fs::write(&metadata_location, &compressed).expect("failed to write metadata"); @@ -3603,6 +3661,63 @@ mod tests { assert!(result.is_err()); } + #[tokio::test] + async fn test_table_metadata_write_with_gzip_compression() { + let temp_dir = TempDir::new().unwrap(); + let temp_path = temp_dir.path().to_str().unwrap(); + let file_io = FileIO::new_with_fs(); + + // Get a test metadata and add gzip compression property + let original_metadata: TableMetadata = get_test_table_metadata("TableMetadataV2Valid.json"); + + // Modify properties to enable gzip compression (using mixed case to test case-insensitive matching) + let mut props = original_metadata.properties.clone(); + props.insert( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "GziP".to_string(), + ); + // Use builder to create new metadata with updated properties + let compressed_metadata = + TableMetadataBuilder::new_from_metadata(original_metadata.clone(), None) + .assign_uuid(original_metadata.table_uuid) + .set_properties(props.clone()) + .unwrap() + .build() + .unwrap() + .metadata; + + // Create MetadataLocation with compression codec from metadata + let metadata_location = + MetadataLocation::new_with_metadata(temp_path, &compressed_metadata); + let metadata_location_str = metadata_location.to_string(); + + // Verify the location has the .gz extension + assert!(metadata_location_str.contains(".gz.metadata.json")); + + // Write the metadata with compression + compressed_metadata + .write_to(&file_io, &metadata_location) + .await + .unwrap(); + + // Verify the compressed file exists + assert!(std::path::Path::new(&metadata_location_str).exists()); + + // Read the raw file and check it's gzip compressed + let raw_content = std::fs::read(&metadata_location_str).unwrap(); + assert!(raw_content.len() > 2); + assert_eq!(raw_content[0], 0x1F); // gzip magic number + assert_eq!(raw_content[1], 0x8B); // gzip magic number + + // Read the metadata back using the compressed location + let read_metadata = TableMetadata::read_from(&file_io, &metadata_location_str) + .await + .unwrap(); + + // Verify the complete round-trip: read metadata should match what we wrote + assert_eq!(read_metadata, compressed_metadata); + } + #[test] fn test_partition_name_exists() { let schema = Schema::builder() diff --git a/crates/iceberg/src/spec/table_metadata_builder.rs b/crates/iceberg/src/spec/table_metadata_builder.rs index 62311a15a2..65dbae1bfc 100644 --- a/crates/iceberg/src/spec/table_metadata_builder.rs +++ b/crates/iceberg/src/spec/table_metadata_builder.rs @@ -570,7 +570,7 @@ impl TableMetadataBuilder { /// Remove a reference /// - /// If `ref_name='main'` the current snapshot id is set to -1. + /// If `ref_name='main'` the current snapshot id is set to `None`. pub fn remove_ref(mut self, ref_name: &str) -> Self { if ref_name == MAIN_BRANCH { self.metadata.current_snapshot_id = None; diff --git a/crates/iceberg/src/spec/table_properties.rs b/crates/iceberg/src/spec/table_properties.rs index 413604f51c..a3d4e7fdaa 100644 --- a/crates/iceberg/src/spec/table_properties.rs +++ b/crates/iceberg/src/spec/table_properties.rs @@ -16,24 +16,89 @@ // under the License. use std::collections::HashMap; +use std::fmt::Display; +use std::str::FromStr; + +use crate::compression::CompressionCodec; +use crate::error::{Error, ErrorKind, Result}; // Helper function to parse a property from a HashMap // If the property is not found, use the default value -fn parse_property( +fn parse_property( properties: &HashMap, key: &str, default: T, -) -> Result +) -> Result where - ::Err: std::fmt::Display, + ::Err: Display, { properties.get(key).map_or(Ok(default), |value| { - value - .parse::() - .map_err(|e| anyhow::anyhow!("Invalid value for {key}: {e}")) + value.parse::().map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid value for {key}: {e}"), + ) + }) }) } +/// Parse compression codec for metadata files from table properties. +/// Retrieves the compression codec property, applies defaults, and parses the value. +/// Only "none" (or empty string) and "gzip" are supported for metadata compression. +/// +/// # Arguments +/// +/// * `properties` - HashMap containing table properties +/// +/// # Errors +/// +/// Returns an error if the codec is not "none", "", or "gzip" (case-insensitive). +/// Lz4 and Zstd are not supported for metadata file compression. +pub(crate) fn parse_metadata_file_compression( + properties: &HashMap, +) -> Result { + let value = properties + .get(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC) + .map(|s| s.as_str()) + .unwrap_or(TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT); + + // Handle empty string as None + if value.is_empty() { + return Ok(CompressionCodec::None); + } + + // Lowercase the value for case-insensitive parsing + let lowercase_value = value.to_lowercase(); + + // Use serde to parse the codec (which has rename_all = "lowercase") + let codec: CompressionCodec = serde_json::from_value(serde_json::Value::String( + lowercase_value, + )) + .map_err(|_| { + Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported.", + CompressionCodec::None.name(), + CompressionCodec::gzip_default().name() + ), + ) + })?; + + // Validate that only None and Gzip are used for metadata + match codec { + CompressionCodec::None | CompressionCodec::Gzip(_) => Ok(codec), + _ => Err(Error::new( + ErrorKind::DataInvalid, + format!( + "Invalid metadata compression codec: {value}. Only '{}' and '{}' are supported for metadata files.", + CompressionCodec::None.name(), + CompressionCodec::gzip_default().name() + ), + )), + } +} + /// TableProperties that contains the properties of a table. #[derive(Debug)] pub struct TableProperties { @@ -49,8 +114,13 @@ pub struct TableProperties { pub write_format_default: String, /// The target file size for files. pub write_target_file_size_bytes: usize, + /// Compression codec for metadata files (JSON) + pub metadata_compression_codec: CompressionCodec, /// Whether to use `FanoutWriter` for partitioned tables. pub write_datafusion_fanout_enabled: bool, + /// Whether garbage collection is enabled on drop. + /// When `false`, data files will not be deleted when a table is dropped. + pub gc_enabled: bool, } impl TableProperties { @@ -139,18 +209,30 @@ impl TableProperties { pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES: &str = "write.target-file-size-bytes"; /// Default target file size pub const PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT: usize = 512 * 1024 * 1024; // 512 MB + + /// Compression codec for metadata files (JSON) + pub const PROPERTY_METADATA_COMPRESSION_CODEC: &str = "write.metadata.compression-codec"; + /// Default metadata compression codec - uncompressed + pub const PROPERTY_METADATA_COMPRESSION_CODEC_DEFAULT: &str = "none"; /// Whether to use `FanoutWriter` for partitioned tables (handles unsorted data). /// If false, uses `ClusteredWriter` (requires sorted data, more memory efficient). pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED: &str = "write.datafusion.fanout.enabled"; /// Default value for fanout writer enabled pub const PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT: bool = true; + + /// Property key for enabling garbage collection on drop. + /// When set to `false`, data files will not be deleted when a table is dropped. + /// Defaults to `true`. + pub const PROPERTY_GC_ENABLED: &str = "gc.enabled"; + /// Default value for gc.enabled + pub const PROPERTY_GC_ENABLED_DEFAULT: bool = true; } impl TryFrom<&HashMap> for TableProperties { // parse by entry key or use default value - type Error = anyhow::Error; + type Error = Error; - fn try_from(props: &HashMap) -> Result { + fn try_from(props: &HashMap) -> Result { Ok(TableProperties { commit_num_retries: parse_property( props, @@ -182,11 +264,17 @@ impl TryFrom<&HashMap> for TableProperties { TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES, TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT, )?, + metadata_compression_codec: parse_metadata_file_compression(props)?, write_datafusion_fanout_enabled: parse_property( props, TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED, TableProperties::PROPERTY_DATAFUSION_WRITE_FANOUT_ENABLED_DEFAULT, )?, + gc_enabled: parse_property( + props, + TableProperties::PROPERTY_GC_ENABLED, + TableProperties::PROPERTY_GC_ENABLED_DEFAULT, + )?, }) } } @@ -194,6 +282,7 @@ impl TryFrom<&HashMap> for TableProperties { #[cfg(test)] mod tests { use super::*; + use crate::compression::CompressionCodec; #[test] fn test_table_properties_default() { @@ -219,6 +308,77 @@ mod tests { table_properties.write_target_file_size_bytes, TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT ); + // Test compression defaults (none means CompressionCodec::None) + assert_eq!( + table_properties.metadata_compression_codec, + CompressionCodec::None + ); + assert_eq!( + table_properties.gc_enabled, + TableProperties::PROPERTY_GC_ENABLED_DEFAULT + ); + } + + #[test] + fn test_table_properties_compression() { + let props = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "gzip".to_string(), + )]); + let table_properties = TableProperties::try_from(&props).unwrap(); + assert_eq!( + table_properties.metadata_compression_codec, + CompressionCodec::gzip_default() + ); + } + + #[test] + fn test_table_properties_compression_none() { + let props = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "none".to_string(), + )]); + let table_properties = TableProperties::try_from(&props).unwrap(); + assert_eq!( + table_properties.metadata_compression_codec, + CompressionCodec::None + ); + } + + #[test] + fn test_table_properties_compression_case_insensitive() { + // Test uppercase + let props_upper = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "GZIP".to_string(), + )]); + let table_properties = TableProperties::try_from(&props_upper).unwrap(); + assert_eq!( + table_properties.metadata_compression_codec, + CompressionCodec::gzip_default() + ); + + // Test mixed case + let props_mixed = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "GzIp".to_string(), + )]); + let table_properties = TableProperties::try_from(&props_mixed).unwrap(); + assert_eq!( + table_properties.metadata_compression_codec, + CompressionCodec::gzip_default() + ); + + // Test "NONE" should also be case-insensitive + let props_none_upper = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "NONE".to_string(), + )]); + let table_properties = TableProperties::try_from(&props_none_upper).unwrap(); + assert_eq!( + table_properties.metadata_compression_codec, + CompressionCodec::None + ); } #[test] @@ -240,12 +400,17 @@ mod tests { TableProperties::PROPERTY_WRITE_TARGET_FILE_SIZE_BYTES.to_string(), "512".to_string(), ), + ( + TableProperties::PROPERTY_GC_ENABLED.to_string(), + "false".to_string(), + ), ]); let table_properties = TableProperties::try_from(&props).unwrap(); assert_eq!(table_properties.commit_num_retries, 10); assert_eq!(table_properties.commit_max_retry_wait_ms, 20); assert_eq!(table_properties.write_format_default, "avro".to_string()); assert_eq!(table_properties.write_target_file_size_bytes, 512); + assert!(!table_properties.gc_enabled); } #[test] @@ -292,5 +457,130 @@ mod tests { assert!(table_properties.to_string().contains( "Invalid value for write.target-file-size-bytes: invalid digit found in string" )); + + let invalid_gc_enabled = HashMap::from([( + TableProperties::PROPERTY_GC_ENABLED.to_string(), + "notabool".to_string(), + )]); + let table_properties = TableProperties::try_from(&invalid_gc_enabled).unwrap_err(); + assert!( + table_properties + .to_string() + .contains("Invalid value for gc.enabled") + ); + } + + #[test] + fn test_table_properties_compression_invalid_rejected() { + let invalid_codecs = ["lz4", "zstd", "snappy"]; + + for codec in invalid_codecs { + let props = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + codec.to_string(), + )]); + let err = TableProperties::try_from(&props).unwrap_err(); + let err_msg = err.to_string(); + assert!( + err_msg.contains(&format!("Invalid metadata compression codec: {codec}")), + "Expected error message to contain codec '{codec}', got: {err_msg}" + ); + assert!( + err_msg.contains("Only 'none' and 'gzip' are supported"), + "Expected error message to contain supported codecs, got: {err_msg}" + ); + } + } + + #[test] + fn test_parse_metadata_file_compression_valid() { + // Test with "none" + let props = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "none".to_string(), + )]); + assert_eq!( + parse_metadata_file_compression(&props).unwrap(), + CompressionCodec::None + ); + + // Test with empty string + let props = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "".to_string(), + )]); + assert_eq!( + parse_metadata_file_compression(&props).unwrap(), + CompressionCodec::None + ); + + // Test with "gzip" + let props = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "gzip".to_string(), + )]); + assert_eq!( + parse_metadata_file_compression(&props).unwrap(), + CompressionCodec::gzip_default() + ); + + // Test case insensitivity - "NONE" + let props = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "NONE".to_string(), + )]); + assert_eq!( + parse_metadata_file_compression(&props).unwrap(), + CompressionCodec::None + ); + + // Test case insensitivity - "GZIP" + let props = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "GZIP".to_string(), + )]); + assert_eq!( + parse_metadata_file_compression(&props).unwrap(), + CompressionCodec::gzip_default() + ); + + // Test case insensitivity - "GzIp" + let props = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + "GzIp".to_string(), + )]); + assert_eq!( + parse_metadata_file_compression(&props).unwrap(), + CompressionCodec::gzip_default() + ); + + // Test default when property is missing + let props = HashMap::new(); + assert_eq!( + parse_metadata_file_compression(&props).unwrap(), + CompressionCodec::None + ); + } + + #[test] + fn test_parse_metadata_file_compression_invalid() { + let invalid_codecs = ["lz4", "zstd", "snappy"]; + + for codec in invalid_codecs { + let props = HashMap::from([( + TableProperties::PROPERTY_METADATA_COMPRESSION_CODEC.to_string(), + codec.to_string(), + )]); + let err = parse_metadata_file_compression(&props).unwrap_err(); + let err_msg = err.to_string(); + assert!( + err_msg.contains("Invalid metadata compression codec"), + "Expected error message to contain 'Invalid metadata compression codec', got: {err_msg}" + ); + assert!( + err_msg.contains("Only 'none' and 'gzip' are supported"), + "Expected error message to contain supported codecs, got: {err_msg}" + ); + } } } diff --git a/crates/iceberg/src/transform/temporal.rs b/crates/iceberg/src/transform/temporal.rs index d0a0da249b..1cd4d6a436 100644 --- a/crates/iceberg/src/transform/temporal.rs +++ b/crates/iceberg/src/transform/temporal.rs @@ -81,7 +81,7 @@ impl TransformFunction for Year { fn transform_literal(&self, input: &crate::spec::Datum) -> Result> { let val = match (input.data_type(), input.literal()) { (PrimitiveType::Date, PrimitiveLiteral::Int(v)) => { - Date32Type::to_naive_date(*v).year() - UNIX_EPOCH_YEAR + Date32Type::to_naive_date_opt(*v).unwrap().year() - UNIX_EPOCH_YEAR } (PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => { Self::timestamp_to_year_micros(*v)? @@ -178,8 +178,8 @@ impl TransformFunction for Month { fn transform_literal(&self, input: &crate::spec::Datum) -> Result> { let val = match (input.data_type(), input.literal()) { (PrimitiveType::Date, PrimitiveLiteral::Int(v)) => { - (Date32Type::to_naive_date(*v).year() - UNIX_EPOCH_YEAR) * 12 - + Date32Type::to_naive_date(*v).month0() as i32 + (Date32Type::to_naive_date_opt(*v).unwrap().year() - UNIX_EPOCH_YEAR) * 12 + + Date32Type::to_naive_date_opt(*v).unwrap().month0() as i32 } (PrimitiveType::Timestamp, PrimitiveLiteral::Long(v)) => { Self::timestamp_to_month_micros(*v)? diff --git a/crates/iceberg/src/util/mod.rs b/crates/iceberg/src/util/mod.rs index b614c981ec..28eda66d49 100644 --- a/crates/iceberg/src/util/mod.rs +++ b/crates/iceberg/src/util/mod.rs @@ -15,5 +15,31 @@ // specific language governing permissions and limitations // under the License. +use std::num::NonZeroUsize; + /// Utilities for working with snapshots. pub mod snapshot; + +// Use a default value of 1 as the safest option. +// See https://doc.rust-lang.org/std/thread/fn.available_parallelism.html#limitations +// for more details. +const DEFAULT_PARALLELISM: usize = 1; + +/// Uses [`std::thread::available_parallelism`] in order to +/// retrieve an estimate of the default amount of parallelism +/// that should be used. Note that [`std::thread::available_parallelism`] +/// returns a `Result` as it can fail, so here we use +/// a default value instead. +/// Note: we don't use a OnceCell or LazyCell here as there +/// are circumstances where the level of available +/// parallelism can change during the lifetime of an executing +/// process, but this should not be called in a hot loop. +pub(crate) fn available_parallelism() -> NonZeroUsize { + std::thread::available_parallelism().unwrap_or_else(|_err| { + // Failed to get the level of parallelism. + // TODO: log/trace when this fallback occurs. + + // Using a default value. + NonZeroUsize::new(DEFAULT_PARALLELISM).unwrap() + }) +} diff --git a/crates/iceberg/src/util/snapshot.rs b/crates/iceberg/src/util/snapshot.rs index d73a255f5e..98997ae815 100644 --- a/crates/iceberg/src/util/snapshot.rs +++ b/crates/iceberg/src/util/snapshot.rs @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -// Taken from https://github.com/apache/iceberg-rust/pull/1470 - use crate::spec::{SnapshotRef, TableMetadataRef}; struct Ancestors { @@ -29,60 +27,159 @@ impl Iterator for Ancestors { fn next(&mut self) -> Option { let snapshot = self.next.take()?; - let result = snapshot.clone(); self.next = snapshot .parent_snapshot_id() .and_then(|id| (self.get_snapshot)(id)); - Some(result) + Some(snapshot) } } -/// Iterate starting from `snapshot` (inclusive) to the root snapshot. +/// Iterate starting from `snapshot_id` (inclusive) to the root snapshot. pub fn ancestors_of( table_metadata: &TableMetadataRef, - snapshot: i64, -) -> Box + Send> { - if let Some(snapshot) = table_metadata.snapshot_by_id(snapshot) { - let table_metadata = table_metadata.clone(); - Box::new(Ancestors { - next: Some(snapshot.clone()), - get_snapshot: Box::new(move |id| table_metadata.snapshot_by_id(id).cloned()), - }) - } else { - Box::new(std::iter::empty()) + snapshot_id: i64, +) -> impl Iterator + Send { + let initial = table_metadata.snapshot_by_id(snapshot_id).cloned(); + let table_metadata = table_metadata.clone(); + Ancestors { + next: initial, + get_snapshot: Box::new(move |id| table_metadata.snapshot_by_id(id).cloned()), } } -/// Iterate starting from `snapshot` (inclusive) to `oldest_snapshot_id` (exclusive). +/// Iterate starting from `latest_snapshot_id` (inclusive) to `oldest_snapshot_id` (exclusive). pub fn ancestors_between( table_metadata: &TableMetadataRef, latest_snapshot_id: i64, oldest_snapshot_id: Option, -) -> Box + Send> { - let Some(oldest_snapshot_id) = oldest_snapshot_id else { - return Box::new(ancestors_of(table_metadata, latest_snapshot_id)); - }; +) -> impl Iterator + Send { + ancestors_of(table_metadata, latest_snapshot_id).take_while(move |snapshot| { + oldest_snapshot_id + .map(|id| snapshot.snapshot_id() != id) + .unwrap_or(true) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::scan::tests::TableTestFixture; + + // Five snapshots chained as: S1 (root) -> S2 -> S3 -> S4 -> S5 (current) + const S1: i64 = 3051729675574597004; + const S2: i64 = 3055729675574597004; + const S3: i64 = 3056729675574597004; + const S4: i64 = 3057729675574597004; + const S5: i64 = 3059729675574597004; - if latest_snapshot_id == oldest_snapshot_id { - return Box::new(std::iter::empty()); + fn metadata() -> TableMetadataRef { + let fixture = TableTestFixture::new_with_deep_history(); + std::sync::Arc::new(fixture.table.metadata().clone()) } - Box::new( - ancestors_of(table_metadata, latest_snapshot_id) - .take_while(move |snapshot| snapshot.snapshot_id() != oldest_snapshot_id), - ) -} + // --- ancestors_of --- + + #[test] + fn test_ancestors_of_nonexistent_snapshot_returns_empty() { + let meta = metadata(); + let ids: Vec = ancestors_of(&meta, 999).map(|s| s.snapshot_id()).collect(); + assert!(ids.is_empty()); + } + + #[test] + fn test_ancestors_of_root_returns_only_root() { + let meta = metadata(); + let ids: Vec = ancestors_of(&meta, S1).map(|s| s.snapshot_id()).collect(); + assert_eq!(ids, vec![S1]); + } + + #[test] + fn test_ancestors_of_leaf_returns_full_chain() { + let meta = metadata(); + let ids: Vec = ancestors_of(&meta, S5).map(|s| s.snapshot_id()).collect(); + assert_eq!(ids, vec![S5, S4, S3, S2, S1]); + } + + #[test] + fn test_ancestors_of_mid_chain_returns_partial_chain() { + let meta = metadata(); + let ids: Vec = ancestors_of(&meta, S3).map(|s| s.snapshot_id()).collect(); + assert_eq!(ids, vec![S3, S2, S1]); + } + + #[test] + fn test_ancestors_of_second_snapshot() { + let meta = metadata(); + let ids: Vec = ancestors_of(&meta, S2).map(|s| s.snapshot_id()).collect(); + assert_eq!(ids, vec![S2, S1]); + } -/// Returns the root (oldest/first) snapshot ID in the table's snapshot lineage. -/// -/// # Arguments -/// * `table_metadata` - The table metadata -/// -/// # Returns -/// The snapshot ID of the root snapshot, or None if there are no snapshots -pub fn root_snapshot_id(table_metadata: &TableMetadataRef) -> Option { - let current = table_metadata.current_snapshot()?; - ancestors_of(table_metadata, current.snapshot_id()) - .last() - .map(|s| s.snapshot_id()) + // --- ancestors_between --- + + #[test] + fn test_ancestors_between_same_id_returns_empty() { + let meta = metadata(); + let ids: Vec = ancestors_between(&meta, S3, Some(S3)) + .map(|s| s.snapshot_id()) + .collect(); + assert!(ids.is_empty()); + } + + #[test] + fn test_ancestors_between_no_oldest_returns_all_ancestors() { + let meta = metadata(); + let ids: Vec = ancestors_between(&meta, S5, None) + .map(|s| s.snapshot_id()) + .collect(); + assert_eq!(ids, vec![S5, S4, S3, S2, S1]); + } + + #[test] + fn test_ancestors_between_excludes_oldest_snapshot() { + let meta = metadata(); + // S5 down to (but not including) S2 + let ids: Vec = ancestors_between(&meta, S5, Some(S2)) + .map(|s| s.snapshot_id()) + .collect(); + assert_eq!(ids, vec![S5, S4, S3]); + } + + #[test] + fn test_ancestors_between_adjacent_snapshots() { + let meta = metadata(); + // S3 down to (but not including) S2 — only S3 itself + let ids: Vec = ancestors_between(&meta, S3, Some(S2)) + .map(|s| s.snapshot_id()) + .collect(); + assert_eq!(ids, vec![S3]); + } + + #[test] + fn test_ancestors_between_leaf_and_root() { + let meta = metadata(); + // S5 down to (but not including) S1 + let ids: Vec = ancestors_between(&meta, S5, Some(S1)) + .map(|s| s.snapshot_id()) + .collect(); + assert_eq!(ids, vec![S5, S4, S3, S2]); + } + + #[test] + fn test_ancestors_between_nonexistent_oldest_returns_full_chain() { + let meta = metadata(); + // oldest_snapshot_id doesn't exist in the chain, so take_while never stops + let ids: Vec = ancestors_between(&meta, S5, Some(999)) + .map(|s| s.snapshot_id()) + .collect(); + assert_eq!(ids, vec![S5, S4, S3, S2, S1]); + } + + #[test] + fn test_ancestors_between_nonexistent_latest_returns_empty() { + let meta = metadata(); + let ids: Vec = ancestors_between(&meta, 999, Some(S1)) + .map(|s| s.snapshot_id()) + .collect(); + assert!(ids.is_empty()); + } } diff --git a/crates/iceberg/src/utils.rs b/crates/iceberg/src/utils.rs deleted file mode 100644 index 00d3e69bd3..0000000000 --- a/crates/iceberg/src/utils.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::num::NonZeroUsize; - -// Use a default value of 1 as the safest option. -// See https://doc.rust-lang.org/std/thread/fn.available_parallelism.html#limitations -// for more details. -const DEFAULT_PARALLELISM: usize = 1; - -/// Uses [`std::thread::available_parallelism`] in order to -/// retrieve an estimate of the default amount of parallelism -/// that should be used. Note that [`std::thread::available_parallelism`] -/// returns a `Result` as it can fail, so here we use -/// a default value instead. -/// Note: we don't use a OnceCell or LazyCell here as there -/// are circumstances where the level of available -/// parallelism can change during the lifetime of an executing -/// process, but this should not be called in a hot loop. -pub(crate) fn available_parallelism() -> NonZeroUsize { - std::thread::available_parallelism().unwrap_or_else(|_err| { - // Failed to get the level of parallelism. - // TODO: log/trace when this fallback occurs. - - // Using a default value. - NonZeroUsize::new(DEFAULT_PARALLELISM).unwrap() - }) -} diff --git a/crates/iceberg/src/writer/file_writer/parquet_writer.rs b/crates/iceberg/src/writer/file_writer/parquet_writer.rs index 0984d8fc64..840d1a5f16 100644 --- a/crates/iceberg/src/writer/file_writer/parquet_writer.rs +++ b/crates/iceberg/src/writer/file_writer/parquet_writer.rs @@ -820,7 +820,7 @@ mod tests { // write data let mut pw = ParquetWriterBuilder::new( WriterProperties::builder() - .set_max_row_group_size(128) + .set_max_row_group_row_count(Some(128)) .build(), Arc::new(to_write.schema().as_ref().try_into().unwrap()), ) diff --git a/crates/iceberg/testdata/example_table_metadata_v2_deep_history.json b/crates/iceberg/testdata/example_table_metadata_v2_deep_history.json new file mode 100644 index 0000000000..a354958697 --- /dev/null +++ b/crates/iceberg/testdata/example_table_metadata_v2_deep_history.json @@ -0,0 +1,104 @@ +{ + "format-version": 2, + "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", + "location": "s3://bucket/test/location", + "last-sequence-number": 34, + "last-updated-ms": 1602638573590, + "last-column-id": 3, + "current-schema-id": 1, + "schemas": [ + { + "type": "struct", + "schema-id": 0, + "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"} + ] + }, + { + "type": "struct", + "schema-id": 1, + "identifier-field-ids": [1, 2], + "fields": [ + {"id": 1, "name": "x", "required": true, "type": "long"}, + {"id": 2, "name": "y", "required": true, "type": "long", "doc": "comment"}, + {"id": 3, "name": "z", "required": true, "type": "long"} + ] + } + ], + "default-spec-id": 0, + "partition-specs": [ + { + "spec-id": 0, + "fields": [ + {"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000} + ] + } + ], + "last-partition-id": 1000, + "default-sort-order-id": 3, + "sort-orders": [ + { + "order-id": 3, + "fields": [ + {"transform": "identity", "source-id": 2, "direction": "asc", "null-order": "nulls-first"}, + {"transform": "bucket[4]", "source-id": 3, "direction": "desc", "null-order": "nulls-last"} + ] + } + ], + "properties": {}, + "current-snapshot-id": 3059729675574597004, + "snapshots": [ + { + "snapshot-id": 3051729675574597004, + "timestamp-ms": 1515100955770, + "sequence-number": 0, + "summary": {"operation": "append"}, + "manifest-list": "s3://bucket/metadata/snap-3051729675574597004.avro" + }, + { + "snapshot-id": 3055729675574597004, + "parent-snapshot-id": 3051729675574597004, + "timestamp-ms": 1555100955770, + "sequence-number": 1, + "summary": {"operation": "append"}, + "manifest-list": "s3://bucket/metadata/snap-3055729675574597004.avro", + "schema-id": 1 + }, + { + "snapshot-id": 3056729675574597004, + "parent-snapshot-id": 3055729675574597004, + "timestamp-ms": 1575100955770, + "sequence-number": 2, + "summary": {"operation": "append"}, + "manifest-list": "s3://bucket/metadata/snap-3056729675574597004.avro", + "schema-id": 1 + }, + { + "snapshot-id": 3057729675574597004, + "parent-snapshot-id": 3056729675574597004, + "timestamp-ms": 1595100955770, + "sequence-number": 3, + "summary": {"operation": "overwrite"}, + "manifest-list": "s3://bucket/metadata/snap-3057729675574597004.avro", + "schema-id": 1 + }, + { + "snapshot-id": 3059729675574597004, + "parent-snapshot-id": 3057729675574597004, + "timestamp-ms": 1602638573590, + "sequence-number": 4, + "summary": {"operation": "append"}, + "manifest-list": "s3://bucket/metadata/snap-3059729675574597004.avro", + "schema-id": 1 + } + ], + "snapshot-log": [ + {"snapshot-id": 3051729675574597004, "timestamp-ms": 1515100955770}, + {"snapshot-id": 3055729675574597004, "timestamp-ms": 1555100955770}, + {"snapshot-id": 3056729675574597004, "timestamp-ms": 1575100955770}, + {"snapshot-id": 3057729675574597004, "timestamp-ms": 1595100955770}, + {"snapshot-id": 3059729675574597004, "timestamp-ms": 1602638573590} + ], + "metadata-log": [], + "refs": {"main": {"snapshot-id": 3059729675574597004, "type": "branch"}} +} diff --git a/crates/integrations/datafusion/src/physical_plan/commit.rs b/crates/integrations/datafusion/src/physical_plan/commit.rs index f876908ae6..3b3ff3d6b3 100644 --- a/crates/integrations/datafusion/src/physical_plan/commit.rs +++ b/crates/integrations/datafusion/src/physical_plan/commit.rs @@ -47,7 +47,7 @@ pub(crate) struct IcebergCommitExec { input: Arc, schema: ArrowSchemaRef, count_schema: ArrowSchemaRef, - plan_properties: PlanProperties, + plan_properties: Arc, } impl IcebergCommitExec { @@ -72,13 +72,13 @@ impl IcebergCommitExec { } // Compute the plan properties for this execution plan - fn compute_properties(schema: ArrowSchemaRef) -> PlanProperties { - PlanProperties::new( + fn compute_properties(schema: ArrowSchemaRef) -> Arc { + Arc::new(PlanProperties::new( EquivalenceProperties::new(schema), Partitioning::UnknownPartitioning(1), EmissionType::Final, Boundedness::Bounded, - ) + )) } // Create a record batch with just the count of rows written @@ -133,7 +133,7 @@ impl ExecutionPlan for IcebergCommitExec { self } - fn properties(&self) -> &PlanProperties { + fn properties(&self) -> &Arc { &self.plan_properties } @@ -301,7 +301,7 @@ mod tests { struct MockWriteExec { schema: Arc, data_files_json: Vec, - plan_properties: PlanProperties, + plan_properties: Arc, } impl MockWriteExec { @@ -312,12 +312,12 @@ mod tests { false, )])); - let plan_properties = PlanProperties::new( + let plan_properties = Arc::new(PlanProperties::new( EquivalenceProperties::new(schema.clone()), Partitioning::UnknownPartitioning(1), EmissionType::Final, Boundedness::Bounded, - ); + )); Self { schema, @@ -340,7 +340,7 @@ mod tests { self.schema.clone() } - fn properties(&self) -> &PlanProperties { + fn properties(&self) -> &Arc { &self.plan_properties } diff --git a/crates/integrations/datafusion/src/physical_plan/metadata_scan.rs b/crates/integrations/datafusion/src/physical_plan/metadata_scan.rs index 9a9d0aa0d9..a1a65dec1f 100644 --- a/crates/integrations/datafusion/src/physical_plan/metadata_scan.rs +++ b/crates/integrations/datafusion/src/physical_plan/metadata_scan.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use std::sync::Arc; + use datafusion::catalog::TableProvider; use datafusion::physical_expr::EquivalenceProperties; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; @@ -27,17 +29,17 @@ use crate::metadata_table::IcebergMetadataTableProvider; #[derive(Debug)] pub struct IcebergMetadataScan { provider: IcebergMetadataTableProvider, - properties: PlanProperties, + properties: Arc, } impl IcebergMetadataScan { pub fn new(provider: IcebergMetadataTableProvider) -> Self { - let properties = PlanProperties::new( + let properties = Arc::new(PlanProperties::new( EquivalenceProperties::new(provider.schema()), Partitioning::UnknownPartitioning(1), EmissionType::Incremental, Boundedness::Bounded, - ); + )); Self { provider, properties, @@ -64,7 +66,7 @@ impl ExecutionPlan for IcebergMetadataScan { self } - fn properties(&self) -> &PlanProperties { + fn properties(&self) -> &Arc { &self.properties } diff --git a/crates/integrations/datafusion/src/physical_plan/mod.rs b/crates/integrations/datafusion/src/physical_plan/mod.rs index 5a9845cde0..aeac30de32 100644 --- a/crates/integrations/datafusion/src/physical_plan/mod.rs +++ b/crates/integrations/datafusion/src/physical_plan/mod.rs @@ -26,5 +26,6 @@ pub(crate) mod write; pub(crate) const DATA_FILES_COL_NAME: &str = "data_files"; +pub use expr_to_predicate::convert_filters_to_predicate; pub use project::project_with_partition; pub use scan::IcebergTableScan; diff --git a/crates/integrations/datafusion/src/physical_plan/scan.rs b/crates/integrations/datafusion/src/physical_plan/scan.rs index 7bb4e7ed0e..234ab26470 100644 --- a/crates/integrations/datafusion/src/physical_plan/scan.rs +++ b/crates/integrations/datafusion/src/physical_plan/scan.rs @@ -46,7 +46,7 @@ pub struct IcebergTableScan { snapshot_id: Option, /// Stores certain, often expensive to compute, /// plan properties used in query optimization. - plan_properties: PlanProperties, + plan_properties: Arc, /// Projection column names, None means all columns projection: Option>, /// Filters to apply to the table scan @@ -104,16 +104,16 @@ impl IcebergTableScan { } /// Computes [`PlanProperties`] used in query optimization. - fn compute_properties(schema: ArrowSchemaRef) -> PlanProperties { + fn compute_properties(schema: ArrowSchemaRef) -> Arc { // TODO: // This is more or less a placeholder, to be replaced // once we support output-partitioning - PlanProperties::new( + Arc::new(PlanProperties::new( EquivalenceProperties::new(schema), Partitioning::UnknownPartitioning(1), EmissionType::Incremental, Boundedness::Bounded, - ) + )) } } @@ -137,7 +137,7 @@ impl ExecutionPlan for IcebergTableScan { Ok(self) } - fn properties(&self) -> &PlanProperties { + fn properties(&self) -> &Arc { &self.plan_properties } diff --git a/crates/integrations/datafusion/src/physical_plan/write.rs b/crates/integrations/datafusion/src/physical_plan/write.rs index 0dea150d31..3b227e20fa 100644 --- a/crates/integrations/datafusion/src/physical_plan/write.rs +++ b/crates/integrations/datafusion/src/physical_plan/write.rs @@ -64,7 +64,7 @@ pub(crate) struct IcebergWriteExec { table: Table, input: Arc, result_schema: ArrowSchemaRef, - plan_properties: PlanProperties, + plan_properties: Arc, } impl IcebergWriteExec { @@ -82,13 +82,13 @@ impl IcebergWriteExec { fn compute_properties( input: &Arc, schema: ArrowSchemaRef, - ) -> PlanProperties { - PlanProperties::new( + ) -> Arc { + Arc::new(PlanProperties::new( EquivalenceProperties::new(schema), Partitioning::UnknownPartitioning(input.output_partitioning().partition_count()), EmissionType::Final, Boundedness::Bounded, - ) + )) } // Create a record batch with serialized data files @@ -153,7 +153,7 @@ impl ExecutionPlan for IcebergWriteExec { vec![true; self.children().len()] } - fn properties(&self) -> &PlanProperties { + fn properties(&self) -> &Arc { &self.plan_properties } @@ -336,17 +336,17 @@ mod tests { struct MockExecutionPlan { schema: ArrowSchemaRef, batches: Vec, - properties: PlanProperties, + properties: Arc, } impl MockExecutionPlan { fn new(schema: ArrowSchemaRef, batches: Vec) -> Self { - let properties = PlanProperties::new( + let properties = Arc::new(PlanProperties::new( EquivalenceProperties::new(schema.clone()), Partitioning::UnknownPartitioning(1), EmissionType::Final, Boundedness::Bounded, - ); + )); Self { schema, @@ -383,7 +383,7 @@ mod tests { self } - fn properties(&self) -> &PlanProperties { + fn properties(&self) -> &Arc { &self.properties } diff --git a/crates/integrations/datafusion/tests/integration_datafusion_test.rs b/crates/integrations/datafusion/tests/integration_datafusion_test.rs index b99c0875c8..cebac75dd9 100644 --- a/crates/integrations/datafusion/tests/integration_datafusion_test.rs +++ b/crates/integrations/datafusion/tests/integration_datafusion_test.rs @@ -535,9 +535,9 @@ fn get_nested_struct_type() -> StructType { 10, "address", Type::Struct(StructType::new(vec![ - NestedField::required(11, "street", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(12, "city", Type::Primitive(PrimitiveType::String)).into(), - NestedField::required(13, "zip", Type::Primitive(PrimitiveType::Int)).into(), + NestedField::optional(11, "street", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(12, "city", Type::Primitive(PrimitiveType::String)).into(), + NestedField::optional(13, "zip", Type::Primitive(PrimitiveType::Int)).into(), ])), ) .into(), @@ -662,7 +662,7 @@ async fn test_insert_into_nested() -> Result<()> { expect![[r#" Field { "id": Int32, metadata: {"PARQUET:field_id": "1"} }, Field { "name": Utf8, metadata: {"PARQUET:field_id": "2"} }, - Field { "profile": nullable Struct("address": Struct("street": non-null Utf8, metadata: {"PARQUET:field_id": "6"}, "city": non-null Utf8, metadata: {"PARQUET:field_id": "7"}, "zip": non-null Int32, metadata: {"PARQUET:field_id": "8"}), metadata: {"PARQUET:field_id": "4"}, "contact": Struct("email": Utf8, metadata: {"PARQUET:field_id": "9"}, "phone": Utf8, metadata: {"PARQUET:field_id": "10"}), metadata: {"PARQUET:field_id": "5"}), metadata: {"PARQUET:field_id": "3"} }"#]], + Field { "profile": nullable Struct("address": Struct("street": Utf8, metadata: {"PARQUET:field_id": "6"}, "city": Utf8, metadata: {"PARQUET:field_id": "7"}, "zip": Int32, metadata: {"PARQUET:field_id": "8"}), metadata: {"PARQUET:field_id": "4"}, "contact": Struct("email": Utf8, metadata: {"PARQUET:field_id": "9"}, "phone": Utf8, metadata: {"PARQUET:field_id": "10"}), metadata: {"PARQUET:field_id": "5"}), metadata: {"PARQUET:field_id": "3"} }"#]], expect![[r#" id: PrimitiveArray [ @@ -681,7 +681,7 @@ async fn test_insert_into_nested() -> Result<()> { valid, ] [ - -- child 0: "address" (Struct([Field { name: "street", data_type: Utf8, metadata: {"PARQUET:field_id": "6"} }, Field { name: "city", data_type: Utf8, metadata: {"PARQUET:field_id": "7"} }, Field { name: "zip", data_type: Int32, metadata: {"PARQUET:field_id": "8"} }])) + -- child 0: "address" (Struct([Field { name: "street", data_type: Utf8, nullable: true, metadata: {"PARQUET:field_id": "6"} }, Field { name: "city", data_type: Utf8, nullable: true, metadata: {"PARQUET:field_id": "7"} }, Field { name: "zip", data_type: Int32, nullable: true, metadata: {"PARQUET:field_id": "8"} }])) StructArray -- validity: [ diff --git a/crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt index 54d74f5e70..aa68ab2762 100644 --- a/crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt @@ -28,8 +28,7 @@ logical_plan physical_plan 01)FilterExec: data@1 = 0102 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----CooperativeExec -04)------IcebergTableScan projection:[id,data] predicate:[data = 0102] +03)----IcebergTableScan projection:[id,data] predicate:[data = 0102] # Verify empty result from empty table query I? diff --git a/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt index 466a45f8c6..496f719261 100644 --- a/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt @@ -39,8 +39,7 @@ logical_plan physical_plan 01)FilterExec: is_active@1 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----CooperativeExec -04)------IcebergTableScan projection:[id,is_active,description] predicate:[is_active = true] +03)----IcebergTableScan projection:[id,is_active,description] predicate:[is_active = true] # Query with is_active = true query ITT rowsort @@ -60,8 +59,7 @@ logical_plan physical_plan 01)FilterExec: NOT is_active@1 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----CooperativeExec -04)------IcebergTableScan projection:[id,is_active,description] predicate:[is_active = false] +03)----IcebergTableScan projection:[id,is_active,description] predicate:[is_active = false] # Query with is_active = false query ITT rowsort @@ -80,8 +78,7 @@ logical_plan physical_plan 01)FilterExec: NOT is_active@1 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----CooperativeExec -04)------IcebergTableScan projection:[id,is_active,description] predicate:[is_active = false] +03)----IcebergTableScan projection:[id,is_active,description] predicate:[is_active = false] # Query with is_active != true (includes false and NULL) query ITT rowsort diff --git a/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt index a160ca77f4..3d8b151aa9 100644 --- a/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt @@ -37,8 +37,7 @@ logical_plan physical_plan 01)FilterExec: name@1 LIKE Al% 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----CooperativeExec -04)------IcebergTableScan projection:[id,name] predicate:[name STARTS WITH "Al"] +03)----IcebergTableScan projection:[id,name] predicate:[name STARTS WITH "Al"] # Test LIKE filtering with case-sensitive match query IT rowsort @@ -57,8 +56,7 @@ logical_plan physical_plan 01)FilterExec: name@1 NOT LIKE Al% 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----CooperativeExec -04)------IcebergTableScan projection:[id,name] predicate:[name NOT STARTS WITH "Al"] +03)----IcebergTableScan projection:[id,name] predicate:[name NOT STARTS WITH "Al"] # Test NOT LIKE filtering query IT rowsort diff --git a/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt b/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt index 3427625291..ffa74173dc 100644 --- a/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt +++ b/crates/sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt @@ -50,8 +50,7 @@ logical_plan physical_plan 01)FilterExec: ts@1 = 1672921800000000000 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----CooperativeExec -04)------IcebergTableScan projection:[id,ts] predicate:[ts = 2023-01-05 12:30:00] +03)----IcebergTableScan projection:[id,ts] predicate:[ts = 2023-01-05 12:30:00] # Verify timestamp equality filtering works query I? @@ -69,8 +68,7 @@ logical_plan physical_plan 01)FilterExec: ts@1 > 1673308800000000000 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----CooperativeExec -04)------IcebergTableScan projection:[id,ts] predicate:[ts > 2023-01-10 00:00:00] +03)----IcebergTableScan projection:[id,ts] predicate:[ts > 2023-01-10 00:00:00] # Verify timestamp greater than filtering query I? rowsort @@ -99,8 +97,7 @@ logical_plan physical_plan 01)FilterExec: ts@1 >= 1672876800000000000 AND ts@1 <= 1673827199000000000 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----CooperativeExec -04)------IcebergTableScan projection:[id,ts] predicate:[(ts >= 2023-01-05 00:00:00) AND (ts <= 2023-01-15 23:59:59)] +03)----IcebergTableScan projection:[id,ts] predicate:[(ts >= 2023-01-05 00:00:00) AND (ts <= 2023-01-15 23:59:59)] # Test timestamp range predicate filtering query I? rowsort @@ -165,8 +162,7 @@ logical_plan physical_plan 01)FilterExec: ts@1 > 1672531200000000 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----CooperativeExec -04)------IcebergTableScan projection:[id,ts] predicate:[ts > 2023-01-01 00:00:00] +03)----IcebergTableScan projection:[id,ts] predicate:[ts > 2023-01-01 00:00:00] query I? SELECT * FROM default.default.test_timestamp_micros WHERE ts > CAST('2023-01-01 00:00:00' AS TIMESTAMP) diff --git a/crates/storage/opendal/Cargo.toml b/crates/storage/opendal/Cargo.toml index e0a3cf8ed6..84f7e1147a 100644 --- a/crates/storage/opendal/Cargo.toml +++ b/crates/storage/opendal/Cargo.toml @@ -49,6 +49,7 @@ reqwest = { workspace = true } serde = { workspace = true } typetag = { workspace = true } url = { workspace = true } +futures = { workspace = true } [dev-dependencies] async-trait = { workspace = true } diff --git a/crates/storage/opendal/src/azdls.rs b/crates/storage/opendal/src/azdls.rs index bace220293..f826b0a103 100644 --- a/crates/storage/opendal/src/azdls.rs +++ b/crates/storage/opendal/src/azdls.rs @@ -193,7 +193,7 @@ impl FromStr for AzureStorageScheme { } /// Validates whether the given path matches what's configured for the backend. -fn match_path_with_config( +pub(crate) fn match_path_with_config( path: &AzureStoragePath, config: &AzdlsConfig, configured_scheme: &AzureStorageScheme, @@ -258,7 +258,7 @@ fn azdls_config_build(config: &AzdlsConfig, path: &AzureStoragePath) -> Result(&self, path: &'a str) -> Result<&'a str> { + match self { + #[cfg(feature = "opendal-memory")] + OpenDalStorage::Memory(_) => Ok(path.strip_prefix("memory:/").unwrap_or(&path[1..])), + #[cfg(feature = "opendal-fs")] + OpenDalStorage::LocalFs => Ok(path.strip_prefix("file:/").unwrap_or(&path[1..])), + #[cfg(feature = "opendal-s3")] + OpenDalStorage::S3 { + configured_scheme, .. + } => { + let url = url::Url::parse(path)?; + let bucket = url.host_str().ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid s3 url: {path}, missing bucket"), + ) + })?; + let prefix = format!("{}://{}/", configured_scheme, bucket); + if path.starts_with(&prefix) { + Ok(&path[prefix.len()..]) + } else { + Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid s3 url: {path}, should start with {prefix}"), + )) + } + } + #[cfg(feature = "opendal-gcs")] + OpenDalStorage::Gcs { .. } => { + let url = url::Url::parse(path)?; + let bucket = url.host_str().ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid gcs url: {path}, missing bucket"), + ) + })?; + let prefix = format!("gs://{}/", bucket); + if path.starts_with(&prefix) { + Ok(&path[prefix.len()..]) + } else { + Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid gcs url: {path}, should start with {prefix}"), + )) + } + } + #[cfg(feature = "opendal-oss")] + OpenDalStorage::Oss { .. } => { + let url = url::Url::parse(path)?; + let bucket = url.host_str().ok_or_else(|| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid oss url: {path}, missing bucket"), + ) + })?; + let prefix = format!("oss://{}/", bucket); + if path.starts_with(&prefix) { + Ok(&path[prefix.len()..]) + } else { + Err(Error::new( + ErrorKind::DataInvalid, + format!("Invalid oss url: {path}, should start with {prefix}"), + )) + } + } + #[cfg(feature = "opendal-azdls")] + OpenDalStorage::Azdls { + configured_scheme, + config, + } => { + let azure_path = path.parse::()?; + match_path_with_config(&azure_path, config, configured_scheme)?; + let relative_path_len = azure_path.path.len(); + Ok(&path[path.len() - relative_path_len..]) + } + #[cfg(all( + not(feature = "opendal-s3"), + not(feature = "opendal-fs"), + not(feature = "opendal-gcs"), + not(feature = "opendal-oss"), + not(feature = "opendal-azdls"), + ))] + _ => Err(Error::new( + ErrorKind::FeatureUnsupported, + "No storage service has been enabled", + )), + } + } } #[typetag::serde(name = "OpenDalStorage")] @@ -400,6 +502,40 @@ impl Storage for OpenDalStorage { Ok(op.remove_all(&path).await.map_err(from_opendal_error)?) } + async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> { + let mut deleters: HashMap = HashMap::new(); + + while let Some(path) = paths.next().await { + let bucket = url::Url::parse(&path) + .ok() + .and_then(|u| u.host_str().map(|s| s.to_string())) + .unwrap_or_default(); + + let (relative_path, deleter) = match deleters.entry(bucket) { + Entry::Occupied(entry) => { + (self.relativize_path(&path)?.to_string(), entry.into_mut()) + } + Entry::Vacant(entry) => { + let (op, rel) = self.create_operator(&path)?; + let rel = rel.to_string(); + let deleter = op.deleter().await.map_err(from_opendal_error)?; + (rel, entry.insert(deleter)) + } + }; + + deleter + .delete(relative_path) + .await + .map_err(from_opendal_error)?; + } + + for (_, mut deleter) in deleters { + deleter.close().await.map_err(from_opendal_error)?; + } + + Ok(()) + } + #[allow(unreachable_code, unused_variables)] fn new_input(&self, path: &str) -> Result { Ok(InputFile::new(Arc::new(self.clone()), path.to_string())) @@ -457,4 +593,182 @@ mod tests { let op = default_memory_operator(); assert_eq!(op.info().scheme().to_string(), "memory"); } + + #[cfg(feature = "opendal-memory")] + #[test] + fn test_relativize_path_memory() { + let storage = OpenDalStorage::Memory(default_memory_operator()); + + assert_eq!( + storage.relativize_path("memory:/path/to/file").unwrap(), + "path/to/file" + ); + // Without the scheme prefix, falls back to stripping the leading slash + assert_eq!( + storage.relativize_path("/path/to/file").unwrap(), + "path/to/file" + ); + } + + #[cfg(feature = "opendal-fs")] + #[test] + fn test_relativize_path_fs() { + let storage = OpenDalStorage::LocalFs; + + assert_eq!( + storage + .relativize_path("file:/tmp/data/file.parquet") + .unwrap(), + "tmp/data/file.parquet" + ); + assert_eq!( + storage.relativize_path("/tmp/data/file.parquet").unwrap(), + "tmp/data/file.parquet" + ); + } + + #[cfg(feature = "opendal-s3")] + #[test] + fn test_relativize_path_s3() { + let storage = OpenDalStorage::S3 { + configured_scheme: "s3".to_string(), + config: Arc::new(S3Config::default()), + customized_credential_load: None, + }; + + assert_eq!( + storage + .relativize_path("s3://my-bucket/path/to/file.parquet") + .unwrap(), + "path/to/file.parquet" + ); + + // s3a scheme + let storage_s3a = OpenDalStorage::S3 { + configured_scheme: "s3a".to_string(), + config: Arc::new(S3Config::default()), + customized_credential_load: None, + }; + assert_eq!( + storage_s3a + .relativize_path("s3a://my-bucket/path/to/file.parquet") + .unwrap(), + "path/to/file.parquet" + ); + } + + #[cfg(feature = "opendal-s3")] + #[test] + fn test_relativize_path_s3_scheme_mismatch() { + let storage = OpenDalStorage::S3 { + configured_scheme: "s3".to_string(), + config: Arc::new(S3Config::default()), + customized_credential_load: None, + }; + + // Scheme mismatch should error + assert!( + storage + .relativize_path("s3a://my-bucket/path/to/file.parquet") + .is_err() + ); + } + + #[cfg(feature = "opendal-gcs")] + #[test] + fn test_relativize_path_gcs() { + let storage = OpenDalStorage::Gcs { + config: Arc::new(GcsConfig::default()), + }; + + assert_eq!( + storage + .relativize_path("gs://my-bucket/path/to/file.parquet") + .unwrap(), + "path/to/file.parquet" + ); + } + + #[cfg(feature = "opendal-gcs")] + #[test] + fn test_relativize_path_gcs_invalid_scheme() { + let storage = OpenDalStorage::Gcs { + config: Arc::new(GcsConfig::default()), + }; + + assert!( + storage + .relativize_path("s3://my-bucket/path/to/file.parquet") + .is_err() + ); + } + + #[cfg(feature = "opendal-oss")] + #[test] + fn test_relativize_path_oss() { + let storage = OpenDalStorage::Oss { + config: Arc::new(OssConfig::default()), + }; + + assert_eq!( + storage + .relativize_path("oss://my-bucket/path/to/file.parquet") + .unwrap(), + "path/to/file.parquet" + ); + } + + #[cfg(feature = "opendal-oss")] + #[test] + fn test_relativize_path_oss_invalid_scheme() { + let storage = OpenDalStorage::Oss { + config: Arc::new(OssConfig::default()), + }; + + assert!( + storage + .relativize_path("s3://my-bucket/path/to/file.parquet") + .is_err() + ); + } + + #[cfg(feature = "opendal-azdls")] + #[test] + fn test_relativize_path_azdls() { + let storage = OpenDalStorage::Azdls { + configured_scheme: AzureStorageScheme::Abfss, + config: Arc::new(AzdlsConfig { + account_name: Some("myaccount".to_string()), + endpoint: Some("https://myaccount.dfs.core.windows.net".to_string()), + ..Default::default() + }), + }; + + assert_eq!( + storage + .relativize_path("abfss://myfs@myaccount.dfs.core.windows.net/path/to/file.parquet") + .unwrap(), + "/path/to/file.parquet" + ); + } + + #[cfg(feature = "opendal-azdls")] + #[test] + fn test_relativize_path_azdls_scheme_mismatch() { + let storage = OpenDalStorage::Azdls { + configured_scheme: AzureStorageScheme::Abfss, + config: Arc::new(AzdlsConfig { + account_name: Some("myaccount".to_string()), + endpoint: Some("https://myaccount.dfs.core.windows.net".to_string()), + ..Default::default() + }), + }; + + // wasbs scheme doesn't match configured abfss + assert!( + storage + .relativize_path("wasbs://myfs@myaccount.dfs.core.windows.net/path/to/file.parquet") + .is_err() + ); + } } diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs new file mode 100644 index 0000000000..7c06cf96a5 --- /dev/null +++ b/crates/storage/opendal/src/resolving.rs @@ -0,0 +1,319 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Resolving storage that auto-detects the scheme from a path and delegates +//! to the appropriate [`OpenDalStorage`] variant. + +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::StreamExt; +use futures::stream::BoxStream; +use iceberg::io::{ + FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, + StorageFactory, +}; +use iceberg::{Error, ErrorKind, Result}; +use opendal::Scheme; +use serde::{Deserialize, Serialize}; +use url::Url; + +use crate::OpenDalStorage; +#[cfg(feature = "opendal-s3")] +use crate::s3::CustomAwsCredentialLoader; + +/// Schemes supported by OpenDalResolvingStorage +pub const SCHEME_MEMORY: &str = "memory"; +pub const SCHEME_FILE: &str = "file"; +pub const SCHEME_S3: &str = "s3"; +pub const SCHEME_S3A: &str = "s3a"; +pub const SCHEME_S3N: &str = "s3n"; +pub const SCHEME_GS: &str = "gs"; +pub const SCHEME_GCS: &str = "gcs"; +pub const SCHEME_OSS: &str = "oss"; +pub const SCHEME_ABFSS: &str = "abfss"; +pub const SCHEME_ABFS: &str = "abfs"; +pub const SCHEME_WASBS: &str = "wasbs"; +pub const SCHEME_WASB: &str = "wasb"; + +/// Parse a URL scheme string into an [`opendal::Scheme`]. +fn parse_scheme(scheme: &str) -> Result { + match scheme { + SCHEME_MEMORY => Ok(Scheme::Memory), + SCHEME_FILE | "" => Ok(Scheme::Fs), + SCHEME_S3 | SCHEME_S3A | SCHEME_S3N => Ok(Scheme::S3), + SCHEME_GS | SCHEME_GCS => Ok(Scheme::Gcs), + SCHEME_OSS => Ok(Scheme::Oss), + SCHEME_ABFSS | SCHEME_ABFS | SCHEME_WASBS | SCHEME_WASB => Ok(Scheme::Azdls), + s => s.parse::().map_err(|e| { + Error::new( + ErrorKind::FeatureUnsupported, + format!("Unsupported storage scheme: {s}: {e}"), + ) + }), + } +} + +/// Extract the scheme string from a path URL. +fn extract_scheme(path: &str) -> Result { + let url = Url::parse(path).map_err(|e| { + Error::new( + ErrorKind::DataInvalid, + format!("Invalid path: {path}, failed to parse URL: {e}"), + ) + })?; + Ok(url.scheme().to_string()) +} + +/// Build an [`OpenDalStorage`] variant for the given scheme and config properties. +fn build_storage_for_scheme( + scheme: &str, + props: &HashMap, + #[cfg(feature = "opendal-s3")] customized_credential_load: &Option, +) -> Result { + match parse_scheme(scheme)? { + #[cfg(feature = "opendal-s3")] + Scheme::S3 => { + let config = crate::s3::s3_config_parse(props.clone())?; + Ok(OpenDalStorage::S3 { + configured_scheme: scheme.to_string(), + config: Arc::new(config), + customized_credential_load: customized_credential_load.clone(), + }) + } + #[cfg(feature = "opendal-gcs")] + Scheme::Gcs => { + let config = crate::gcs::gcs_config_parse(props.clone())?; + Ok(OpenDalStorage::Gcs { + config: Arc::new(config), + }) + } + #[cfg(feature = "opendal-oss")] + Scheme::Oss => { + let config = crate::oss::oss_config_parse(props.clone())?; + Ok(OpenDalStorage::Oss { + config: Arc::new(config), + }) + } + #[cfg(feature = "opendal-azdls")] + Scheme::Azdls => { + let configured_scheme: crate::azdls::AzureStorageScheme = scheme.parse()?; + let config = crate::azdls::azdls_config_parse(props.clone())?; + Ok(OpenDalStorage::Azdls { + configured_scheme, + config: Arc::new(config), + }) + } + #[cfg(feature = "opendal-fs")] + Scheme::Fs => Ok(OpenDalStorage::LocalFs), + #[cfg(feature = "opendal-memory")] + Scheme::Memory => Ok(OpenDalStorage::Memory(crate::memory::memory_config_build()?)), + unsupported => Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Unsupported storage scheme: {unsupported}"), + )), + } +} + +/// A resolving storage factory that creates [`OpenDalResolvingStorage`] instances. +/// +/// This factory accepts paths from any supported storage system and dynamically +/// delegates operations to the appropriate [`OpenDalStorage`] variant based on +/// the path scheme. +/// +/// # Example +/// +/// ```rust,ignore +/// use std::sync::Arc; +/// use iceberg::io::FileIOBuilder; +/// use iceberg_storage_opendal::OpenDalResolvingStorageFactory; +/// +/// let factory = OpenDalResolvingStorageFactory::new(); +/// let file_io = FileIOBuilder::new(Arc::new(factory)) +/// .with_prop("s3.region", "us-east-1") +/// .build(); +/// ``` +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct OpenDalResolvingStorageFactory { + /// Custom AWS credential loader for S3 storage. + #[cfg(feature = "opendal-s3")] + #[serde(skip)] + customized_credential_load: Option, +} + +impl Default for OpenDalResolvingStorageFactory { + fn default() -> Self { + Self::new() + } +} + +impl OpenDalResolvingStorageFactory { + /// Create a new resolving storage factory. + pub fn new() -> Self { + Self { + #[cfg(feature = "opendal-s3")] + customized_credential_load: None, + } + } + + /// Set a custom AWS credential loader for S3 storage. + #[cfg(feature = "opendal-s3")] + pub fn with_s3_credential_loader(mut self, loader: CustomAwsCredentialLoader) -> Self { + self.customized_credential_load = Some(loader); + self + } +} + +#[typetag::serde] +impl StorageFactory for OpenDalResolvingStorageFactory { + fn build(&self, config: &StorageConfig) -> Result> { + Ok(Arc::new(OpenDalResolvingStorage { + props: config.props().clone(), + storages: RwLock::new(HashMap::new()), + #[cfg(feature = "opendal-s3")] + customized_credential_load: self.customized_credential_load.clone(), + })) + } +} + +/// A resolving storage that auto-detects the scheme from a path and delegates +/// to the appropriate [`OpenDalStorage`] variant. +/// +/// Sub-storages are lazily created on first use for each scheme and cached +/// for subsequent operations. +#[derive(Debug, Serialize, Deserialize)] +pub struct OpenDalResolvingStorage { + /// Configuration properties shared across all backends. + props: HashMap, + /// Cache of scheme → storage mappings. + #[serde(skip, default)] + storages: RwLock>>, + /// Custom AWS credential loader for S3 storage. + #[cfg(feature = "opendal-s3")] + #[serde(skip)] + customized_credential_load: Option, +} + +impl OpenDalResolvingStorage { + /// Resolve the storage for the given path by extracting the scheme and + /// returning the cached or newly-created [`OpenDalStorage`]. + fn resolve(&self, path: &str) -> Result> { + let scheme = extract_scheme(path)?; + + // Fast path: check read lock first. + { + let cache = self + .storages + .read() + .map_err(|_| Error::new(ErrorKind::Unexpected, "Storage cache lock poisoned"))?; + if let Some(storage) = cache.get(&scheme) { + return Ok(storage.clone()); + } + } + + // Slow path: build and insert under write lock. + let mut cache = self + .storages + .write() + .map_err(|_| Error::new(ErrorKind::Unexpected, "Storage cache lock poisoned"))?; + + // Double-check after acquiring write lock. + if let Some(storage) = cache.get(&scheme) { + return Ok(storage.clone()); + } + + let storage = build_storage_for_scheme( + &scheme, + &self.props, + #[cfg(feature = "opendal-s3")] + &self.customized_credential_load, + )?; + let storage = Arc::new(storage); + cache.insert(scheme, storage.clone()); + Ok(storage) + } +} + +#[async_trait] +#[typetag::serde] +impl Storage for OpenDalResolvingStorage { + async fn exists(&self, path: &str) -> Result { + self.resolve(path)?.exists(path).await + } + + async fn metadata(&self, path: &str) -> Result { + self.resolve(path)?.metadata(path).await + } + + async fn read(&self, path: &str) -> Result { + self.resolve(path)?.read(path).await + } + + async fn reader(&self, path: &str) -> Result> { + self.resolve(path)?.reader(path).await + } + + async fn write(&self, path: &str, bs: Bytes) -> Result<()> { + self.resolve(path)?.write(path, bs).await + } + + async fn writer(&self, path: &str) -> Result> { + self.resolve(path)?.writer(path).await + } + + async fn delete(&self, path: &str) -> Result<()> { + self.resolve(path)?.delete(path).await + } + + async fn delete_prefix(&self, path: &str) -> Result<()> { + self.resolve(path)?.delete_prefix(path).await + } + + async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> { + // Group paths by scheme so each resolved storage receives a batch, + // avoiding repeated operator creation per path. + let mut grouped: HashMap> = HashMap::new(); + while let Some(path) = paths.next().await { + let scheme = extract_scheme(&path)?; + grouped.entry(scheme).or_default().push(path); + } + + for (_, paths) in grouped { + let storage = self.resolve(&paths[0])?; + storage + .delete_stream(futures::stream::iter(paths).boxed()) + .await?; + } + Ok(()) + } + + fn new_input(&self, path: &str) -> Result { + Ok(InputFile::new( + Arc::new(self.resolve(path)?.as_ref().clone()), + path.to_string(), + )) + } + + fn new_output(&self, path: &str) -> Result { + Ok(OutputFile::new( + Arc::new(self.resolve(path)?.as_ref().clone()), + path.to_string(), + )) + } +} diff --git a/crates/storage/opendal/tests/file_io_s3_test.rs b/crates/storage/opendal/tests/file_io_s3_test.rs index 5801af0606..207a4454d7 100644 --- a/crates/storage/opendal/tests/file_io_s3_test.rs +++ b/crates/storage/opendal/tests/file_io_s3_test.rs @@ -24,6 +24,7 @@ mod tests { use std::sync::Arc; use async_trait::async_trait; + use futures::StreamExt; use iceberg::io::{ FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY, }; @@ -203,4 +204,46 @@ mod tests { } } } + + #[tokio::test] + async fn test_file_io_s3_delete_stream() { + let file_io = get_file_io().await; + + // Write multiple files + let paths: Vec = (0..5) + .map(|i| { + format!( + "s3://bucket1/{}/file-{i}", + normalize_test_name_with_parts!("test_file_io_s3_delete_stream") + ) + }) + .collect(); + for path in &paths { + let _ = file_io.delete(path).await; + file_io + .new_output(path) + .unwrap() + .write("delete-me".into()) + .await + .unwrap(); + assert!(file_io.exists(path).await.unwrap()); + } + + // Delete via delete_stream + let stream = futures::stream::iter(paths.clone()).boxed(); + file_io.delete_stream(stream).await.unwrap(); + + // Verify all files are gone + for path in &paths { + assert!(!file_io.exists(path).await.unwrap()); + } + } + + #[tokio::test] + async fn test_file_io_s3_delete_stream_empty() { + let file_io = get_file_io().await; + let stream = futures::stream::empty().boxed(); + // Should succeed with no-op + file_io.delete_stream(stream).await.unwrap(); + } } diff --git a/crates/storage/opendal/tests/resolving_storage_test.rs b/crates/storage/opendal/tests/resolving_storage_test.rs new file mode 100644 index 0000000000..4572ad2c2d --- /dev/null +++ b/crates/storage/opendal/tests/resolving_storage_test.rs @@ -0,0 +1,297 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Integration tests for OpenDalResolvingStorage. +//! +//! These tests assume Docker containers are started externally via `make docker-up`. +//! Each test uses unique file paths based on module path to avoid conflicts. + +#[cfg(all( + feature = "opendal-s3", + feature = "opendal-fs", + feature = "opendal-memory" +))] +mod tests { + use std::sync::Arc; + + use iceberg::io::{ + FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY, + }; + use iceberg_storage_opendal::OpenDalResolvingStorageFactory; + use iceberg_test_utils::{get_minio_endpoint, normalize_test_name_with_parts, set_up}; + + fn get_resolving_file_io() -> iceberg::io::FileIO { + set_up(); + + let minio_endpoint = get_minio_endpoint(); + + FileIOBuilder::new(Arc::new(OpenDalResolvingStorageFactory::new())) + .with_props(vec![ + (S3_ENDPOINT, minio_endpoint), + (S3_ACCESS_KEY_ID, "admin".to_string()), + (S3_SECRET_ACCESS_KEY, "password".to_string()), + (S3_REGION, "us-east-1".to_string()), + ]) + .build() + } + + fn temp_fs_path(name: &str) -> String { + let dir = std::env::temp_dir().join("iceberg_resolving_tests"); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join(name); + // Clean up from previous runs + let _ = std::fs::remove_file(&path); + format!("file:/{}", path.display()) + } + + #[tokio::test] + async fn test_mixed_scheme_write_and_read() { + let file_io = get_resolving_file_io(); + + let s3_path = format!( + "s3://bucket1/{}", + normalize_test_name_with_parts!("test_mixed_scheme_write_and_read") + ); + let fs_path = temp_fs_path("mixed_write_and_read.txt"); + let mem_path = "memory://test_mixed_scheme_write_and_read"; + + // Write to all three schemes + file_io + .new_output(&s3_path) + .unwrap() + .write("from_s3".into()) + .await + .unwrap(); + file_io + .new_output(&fs_path) + .unwrap() + .write("from_fs".into()) + .await + .unwrap(); + file_io + .new_output(mem_path) + .unwrap() + .write("from_memory".into()) + .await + .unwrap(); + + // Read back from all three + assert_eq!( + file_io.new_input(&s3_path).unwrap().read().await.unwrap(), + bytes::Bytes::from("from_s3") + ); + assert_eq!( + file_io.new_input(&fs_path).unwrap().read().await.unwrap(), + bytes::Bytes::from("from_fs") + ); + assert_eq!( + file_io.new_input(mem_path).unwrap().read().await.unwrap(), + bytes::Bytes::from("from_memory") + ); + } + + #[tokio::test] + async fn test_mixed_scheme_exists_independently() { + let file_io = get_resolving_file_io(); + + let s3_path = format!( + "s3://bucket1/{}", + normalize_test_name_with_parts!("test_mixed_scheme_exists_independently") + ); + let fs_path = temp_fs_path("mixed_exists_independently.txt"); + let mem_path = "memory://test_mixed_scheme_exists_independently"; + + // Clean up S3 from previous runs + let _ = file_io.delete(&s3_path).await; + + // None exist initially + assert!(!file_io.exists(&s3_path).await.unwrap()); + assert!(!file_io.exists(&fs_path).await.unwrap()); + assert!(!file_io.exists(mem_path).await.unwrap()); + + // Write only to fs + file_io + .new_output(&fs_path) + .unwrap() + .write("fs_only".into()) + .await + .unwrap(); + + // Only fs exists + assert!(!file_io.exists(&s3_path).await.unwrap()); + assert!(file_io.exists(&fs_path).await.unwrap()); + assert!(!file_io.exists(mem_path).await.unwrap()); + } + + #[tokio::test] + async fn test_mixed_scheme_delete_one_keeps_others() { + let file_io = get_resolving_file_io(); + + let s3_path = format!( + "s3://bucket1/{}", + normalize_test_name_with_parts!("test_mixed_scheme_delete_one_keeps_others") + ); + let fs_path = temp_fs_path("mixed_delete_one_keeps_others.txt"); + let mem_path = "memory://test_mixed_scheme_delete_one_keeps_others"; + + // Write to all three + file_io + .new_output(&s3_path) + .unwrap() + .write("s3".into()) + .await + .unwrap(); + file_io + .new_output(&fs_path) + .unwrap() + .write("fs".into()) + .await + .unwrap(); + file_io + .new_output(mem_path) + .unwrap() + .write("mem".into()) + .await + .unwrap(); + + // Delete only the fs file + file_io.delete(&fs_path).await.unwrap(); + + // fs gone, S3 and memory still there + assert!(file_io.exists(&s3_path).await.unwrap()); + assert!(!file_io.exists(&fs_path).await.unwrap()); + assert!(file_io.exists(mem_path).await.unwrap()); + + assert_eq!( + file_io.new_input(&s3_path).unwrap().read().await.unwrap(), + bytes::Bytes::from("s3") + ); + assert_eq!( + file_io.new_input(mem_path).unwrap().read().await.unwrap(), + bytes::Bytes::from("mem") + ); + } + + #[tokio::test] + async fn test_mixed_scheme_interleaved_operations() { + let file_io = get_resolving_file_io(); + + let s3_path = format!( + "s3://bucket1/{}", + normalize_test_name_with_parts!("test_mixed_scheme_interleaved") + ); + let fs_path = temp_fs_path("mixed_interleaved.txt"); + let mem_path = "memory://test_mixed_scheme_interleaved"; + + // Interleave: write fs, write memory, write s3 + file_io + .new_output(&fs_path) + .unwrap() + .write("fs_data".into()) + .await + .unwrap(); + file_io + .new_output(mem_path) + .unwrap() + .write("mem_data".into()) + .await + .unwrap(); + file_io + .new_output(&s3_path) + .unwrap() + .write("s3_data".into()) + .await + .unwrap(); + + // Read in reverse order: s3, memory, fs + assert_eq!( + file_io.new_input(&s3_path).unwrap().read().await.unwrap(), + bytes::Bytes::from("s3_data") + ); + assert_eq!( + file_io.new_input(mem_path).unwrap().read().await.unwrap(), + bytes::Bytes::from("mem_data") + ); + assert_eq!( + file_io.new_input(&fs_path).unwrap().read().await.unwrap(), + bytes::Bytes::from("fs_data") + ); + } + + #[tokio::test] + async fn test_invalid_scheme() { + let file_io = get_resolving_file_io(); + let result = file_io.exists("unknown://bucket/key").await; + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("Unsupported storage scheme"), + ); + } + + #[tokio::test] + async fn test_missing_scheme() { + let file_io = get_resolving_file_io(); + let result = file_io.exists("no-scheme-path").await; + assert!(result.is_err()); + } + + #[cfg(feature = "opendal-s3")] + #[tokio::test] + async fn test_with_custom_credential_loader() { + use async_trait::async_trait; + use iceberg_storage_opendal::CustomAwsCredentialLoader; + use reqsign::{AwsCredential, AwsCredentialLoad}; + use reqwest::Client; + + struct MinioCredentialLoader; + + #[async_trait] + impl AwsCredentialLoad for MinioCredentialLoader { + async fn load_credential( + &self, + _client: Client, + ) -> anyhow::Result> { + Ok(Some(AwsCredential { + access_key_id: "admin".to_string(), + secret_access_key: "password".to_string(), + session_token: None, + expires_in: None, + })) + } + } + + set_up(); + let minio_endpoint = get_minio_endpoint(); + + let factory = OpenDalResolvingStorageFactory::new().with_s3_credential_loader( + CustomAwsCredentialLoader::new(Arc::new(MinioCredentialLoader)), + ); + + let file_io = FileIOBuilder::new(Arc::new(factory)) + .with_props(vec![ + (S3_ENDPOINT, minio_endpoint), + (S3_REGION, "us-east-1".to_string()), + ]) + .build(); + + // Should be able to access S3 using the custom credential loader + assert!(file_io.exists("s3://bucket1/").await.unwrap()); + } +}