diff --git a/.github/actions/max_disk_space/action.yaml b/.github/actions/max_disk_space/action.yaml new file mode 100644 index 000000000..759e53afb --- /dev/null +++ b/.github/actions/max_disk_space/action.yaml @@ -0,0 +1,13 @@ +name: 'Maximize disk space' +description: 'Maximize available disk space by removing unwanted software' + +runs: + using: 'composite' + steps: + - name: Maximize available disk space + uses: AdityaGarg8/remove-unwanted-software@v5 + with: + remove-android: 'true' + remove-dotnet: 'true' + remove-haskell: 'true' + remove-codeql: 'true' diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5874d7e47..429a6ec79 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -8,7 +8,7 @@ on: workflow_dispatch: inputs: force-publish: - description: 'Force publish even if no version change detected' + description: 'Force publish even if no version change was detected' required: false type: choice options: @@ -41,6 +41,8 @@ jobs: lookup-only: true - uses: actions/checkout@v4 if: steps.look-up.outputs.cache-hit != 'true' + - uses: ./.github/actions/max_disk_space + if: steps.look-up.outputs.cache-hit != 'true' - uses: actions/cache@v4 if: steps.look-up.outputs.cache-hit != 'true' with: @@ -79,11 +81,15 @@ jobs: numpy-version: 2 - python-version: '3.12' numpy-version: 1 + - python-version: '3.12' + numpy-version: 2 + spec-from-main: true # - python-version: '3.13' # numpy-version: 2 steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/max_disk_space - uses: actions/setup-python@v6 with: python-version: ${{matrix.python-version}} @@ -97,6 +103,10 @@ jobs: run: | pyright --version pyright -p pyproject.toml --pythonversion ${{ matrix.python-version }} + - if: matrix.spec-from-main + run: | + pip uninstall -y bioimageio.spec + pip install git+https://github.com/bioimage-io/spec-bioimage-io.git@main - name: Restore bioimageio cache ${{needs.populate-cache.outputs.cache-key}} uses: actions/cache/restore@v4 with: @@ -123,7 +133,7 @@ jobs: include-hidden-files: true coverage: - needs: [test] + needs: test runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -138,13 +148,11 @@ jobs: ls -la .coverage* coverage combine coverage xml -o coverage.xml - - uses: orgoro/coverage@v3.2 + - uses: actions/upload-artifact@v4 with: - coverageFile: coverage.xml - token: ${{ secrets.GITHUB_TOKEN }} - thresholdAll: 0.7 - thresholdNew: 0.9 - thresholdModified: 0.6 + name: coverage.xml + path: coverage.xml + retention-days: 1 - name: generate coverage badge and html report run: | pip install genbadge[coverage] @@ -154,6 +162,23 @@ jobs: with: name: coverage-summary path: dist + retention-days: 1 + + coverage-comment: + needs: coverage + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: coverage.xml + - uses: orgoro/coverage@v3.2 + with: + coverageFile: coverage.xml + token: ${{ secrets.GITHUB_TOKEN }} + thresholdAll: 0.7 + thresholdNew: 0.9 + thresholdModified: 0.6 conda-build: needs: [populate-cache, test] # only so we run tests even if the pinned bioimageio.spec version is not yet published on conda-forge @@ -190,34 +215,6 @@ jobs: env: BIOIMAGEIO_CACHE_PATH: bioimageio_cache - docs: - needs: [coverage, test] - if: github.ref == 'refs/heads/main' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - name: coverage-summary - path: dist - - uses: actions/setup-python@v6 - with: - python-version: '3.12' - cache: 'pip' - - run: pip install -e .[dev,partners] - - name: Generate developer docs - run: ./scripts/pdoc/run.sh - - run: cp README.md ./dist/README.md - - name: copy rendered presentations - run: | - mkdir ./dist/presentations - cp -r ./presentations/*.html ./dist/presentations/ - - name: Deploy to gh-pages πŸš€ - uses: JamesIves/github-pages-deploy-action@v4 - with: - branch: gh-pages - folder: dist - build: runs-on: ubuntu-latest steps: @@ -235,21 +232,36 @@ jobs: path: dist/ name: dist - publish: - needs: [test, build, conda-build, docs] + docs: + needs: coverage runs-on: ubuntu-latest - environment: - name: release - url: https://pypi.org/project/bioimageio.core/ permissions: contents: write # required for tag creation - id-token: write # required for pypi publish action + outputs: + new-version: ${{ steps.get-new-version.outputs.new-version }} steps: - - name: Check out the repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: - fetch-depth: 2 + fetch-depth: 0 fetch-tags: true + - uses: ./.github/actions/max_disk_space + - uses: actions/download-artifact@v4 + with: + name: coverage-summary + path: dist + - uses: actions/setup-python@v6 + with: + python-version: '3.12' + cache: 'pip' + - run: pip install -e .[dev,docs,partners] + - name: Check doc scripts + run: pyright scripts/generate_api_doc_pages.py + - name: Get branch name to deploy to + id: get_branch + shell: bash + run: | + if [[ -n '${{ github.event.pull_request.head.ref }}' ]]; then branch=gh-pages-${{ github.event.pull_request.head.ref }}; else branch=gh-pages; fi + echo "branch=$branch" >> $GITHUB_OUTPUT - name: Get parent commit if: inputs.force-publish != 'true' id: get-parent-commit @@ -258,7 +270,6 @@ jobs: - id: get-existing-tag if: inputs.force-publish == 'true' run: echo "existing-tag=$(git tag --points-at HEAD 'v[0-9]*.[0-9]*.[0-9]*')" >> $GITHUB_OUTPUT - - name: Detect new version from last commit and create tag id: tag-version if: github.ref == 'refs/heads/main' && steps.get-parent-commit.outputs.sha && inputs.force-publish != 'true' @@ -273,8 +284,6 @@ jobs: import os from pathlib import Path - - if "${{ inputs.force-publish }}" == "true": existing_tag = "${{ steps.get-existing-tag.outputs.existing-tag }}" valid = existing_tag.count("v") == 1 and existing_tag.count(".") == 2 and all(part.isdigit() for part in existing_tag.lstrip("v").split(".")) @@ -291,23 +300,52 @@ jobs: with open(os.environ['GITHUB_OUTPUT'], 'a') as f: print(f"new-version={new_version}", file=f) + - name: Configure Git Credentials + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + - name: Generate developer docs + run: mike deploy --push --branch ${{ steps.get_branch.outputs.branch }} --update-aliases ${{ steps.get-new-version.outputs.new-version || 'dev'}} ${{ steps.get-new-version.outputs.new-version && 'latest' || ' '}} + - name: copy rendered presentations + run: | + mkdir ./dist/presentations + cp -r ./presentations/*.html ./dist/presentations/ + - name: Deploy to gh-pages πŸš€ + uses: JamesIves/github-pages-deploy-action@v4 + with: + branch: gh-pages + folder: dist + clean: true + clean-exclude: | + .nojekyll + index.html + versions.json + latest/ + dev/ + v0.*/ + publish: + needs: [test, coverage, build, conda-build, docs] + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' && needs.docs.outputs.new-version + environment: + name: release + url: https://pypi.org/project/bioimageio.core/ + permissions: + contents: write # required to create a github release (release drafter) + id-token: write # required for pypi publish action + steps: - uses: actions/download-artifact@v4 - if: github.ref == 'refs/heads/main' && steps.get-new-version.outputs.new-version with: name: dist path: dist - name: Publish package on PyPI - if: github.ref == 'refs/heads/main' && steps.get-new-version.outputs.new-version uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: dist/ - - name: Publish the release notes - if: github.ref == 'refs/heads/main' uses: release-drafter/release-drafter@v6.0.0 with: - publish: "${{ steps.get-new-version.outputs.new-version != '' }}" - tag: '${{ steps.get-new-version.outputs.new-version }}' + tag: '${{ needs.docs.outputs.new-version }}' env: GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' diff --git a/.gitignore b/.gitignore index 688e4a889..a4991c3cd 100644 --- a/.gitignore +++ b/.gitignore @@ -6,12 +6,14 @@ __pycache__/ *.egg-info/ *.pyc **/tmp +bioimageio_cache/ bioimageio_unzipped_tf_weights/ build/ cache coverage.xml dist/ -docs/ +docs/api/ dogfood/ +pkgs/ +site/ typings/pooch/ -bioimageio_cache/ diff --git a/README.md b/README.md index da957da35..102bcb8cc 100644 --- a/README.md +++ b/README.md @@ -8,342 +8,39 @@ # bioimageio.core -Python specific core utilities for bioimage.io resources (in particular DL models). - -## Get started - -To get started we recommend installing bioimageio.core with conda together with a deep -learning framework, e.g. pytorch, and run a few `bioimageio` commands to see what -bioimage.core has to offer: - -1. install with conda (for more details on conda environments, [checkout the conda docs](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html)) - - ```console - conda install -c conda-forge bioimageio.core pytorch - ``` - -1. test a model - - ```console - $ bioimageio test powerful-chipmunk - ... - ``` - -
- (Click to expand output) - - ```console - - - βœ”οΈ bioimageio validation passed - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - source https://uk1s3.embassy.ebi.ac.uk/public-datasets/bioimage.io/powerful-chipmunk/1/files/rdf.yaml - format version model 0.4.10 - bioimageio.spec 0.5.3post4 - bioimageio.core 0.6.8 - - - - ❓ location detail - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - βœ”οΈ initialized ModelDescr to describe model 0.4.10 - - βœ”οΈ bioimageio.spec format validation model 0.4.10 - πŸ” context.perform_io_checks True - πŸ” context.root https://uk1s3.embassy.ebi.ac.uk/public-datasets/bioimage.io/powerful-chipmunk/1/files - πŸ” context.known_files.weights.pt 3bd9c518c8473f1e35abb7624f82f3aa92f1015e66fb1f6a9d08444e1f2f5698 - πŸ” context.known_files.weights-torchscript.pt 4e568fd81c0ffa06ce13061327c3f673e1bac808891135badd3b0fcdacee086b - πŸ” context.warning_level error - - βœ”οΈ Reproduce test outputs from test inputs - - βœ”οΈ Reproduce test outputs from test inputs - ``` - -
- - or - - ```console - $ bioimageio test impartial-shrimp - ... - ``` - -
(Click to expand output) - - ```console - βœ”οΈ bioimageio validation passed - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - source https://uk1s3.embassy.ebi.ac.uk/public-datasets/bioimage.io/impartial-shrimp/1.1/files/rdf.yaml - format version model 0.5.3 - bioimageio.spec 0.5.3.2 - bioimageio.core 0.6.9 - - - ❓ location detail - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - βœ”οΈ initialized ModelDescr to describe model 0.5.3 - - - βœ”οΈ bioimageio.spec format validation model 0.5.3 - - πŸ” context.perform_io_checks False - πŸ” context.warning_level error - - βœ”οΈ Reproduce test outputs from test inputs (pytorch_state_dict) - - - βœ”οΈ Run pytorch_state_dict inference for inputs with batch_size: 1 and size parameter n: - - 0 - - βœ”οΈ Run pytorch_state_dict inference for inputs with batch_size: 2 and size parameter n: - - 0 - - βœ”οΈ Run pytorch_state_dict inference for inputs with batch_size: 1 and size parameter n: - - 1 - - βœ”οΈ Run pytorch_state_dict inference for inputs with batch_size: 2 and size parameter n: - - 1 - - βœ”οΈ Run pytorch_state_dict inference for inputs with batch_size: 1 and size parameter n: - - 2 - - βœ”οΈ Run pytorch_state_dict inference for inputs with batch_size: 2 and size parameter n: - - 2 - - βœ”οΈ Reproduce test outputs from test inputs (torchscript) - - - βœ”οΈ Run torchscript inference for inputs with batch_size: 1 and size parameter n: 0 - - - βœ”οΈ Run torchscript inference for inputs with batch_size: 2 and size parameter n: 0 - - - βœ”οΈ Run torchscript inference for inputs with batch_size: 1 and size parameter n: 1 - - - βœ”οΈ Run torchscript inference for inputs with batch_size: 2 and size parameter n: 1 - - - βœ”οΈ Run torchscript inference for inputs with batch_size: 1 and size parameter n: 2 - - - βœ”οΈ Run torchscript inference for inputs with batch_size: 2 and size parameter n: 2 - ``` - -
-1. run prediction on your data - -- display the `bioimageio-predict` command help to get an overview: - - ```console - $ bioimageio predict --help - ... - ``` - -
- (Click to expand output) - - ```console - usage: bioimageio predict [-h] [--inputs Sequence[Union[str,Annotated[Tuple[str,...],MinLenmin_length=1]]]] - [--outputs {str,Tuple[str,...]}] [--overwrite bool] [--blockwise bool] [--stats Path] - [--preview bool] - [--weight_format {typing.Literal['keras_hdf5','onnx','pytorch_state_dict','tensorflow_js','tensorflow_saved_model_bundle','torchscript'],any}] - [--example bool] - SOURCE - - bioimageio-predict - Run inference on your data with a bioimage.io model. - - positional arguments: - SOURCE Url/path to a bioimageio.yaml/rdf.yaml file - or a bioimage.io resource identifier, e.g. 'affable-shark' - - optional arguments: - -h, --help show this help message and exit - --inputs Sequence[Union[str,Annotated[Tuple[str,...],MinLen(min_length=1)]]] - Model input sample paths (for each input tensor) - - The input paths are expected to have shape... - - (n_samples,) or (n_samples,1) for models expecting a single input tensor - - (n_samples,) containing the substring '{input_id}', or - - (n_samples, n_model_inputs) to provide each input tensor path explicitly. - - All substrings that are replaced by metadata from the model description: - - '{model_id}' - - '{input_id}' - - Example inputs to process sample 'a' and 'b' - for a model expecting a 'raw' and a 'mask' input tensor: - --inputs="[["a_raw.tif","a_mask.tif"],["b_raw.tif","b_mask.tif"]]" - (Note that JSON double quotes need to be escaped.) - - Alternatively a `bioimageio-cli.yaml` (or `bioimageio-cli.json`) file - may provide the arguments, e.g.: - ```yaml - inputs: - - [a_raw.tif, a_mask.tif] - - [b_raw.tif, b_mask.tif] - ``` - - `.npy` and any file extension supported by imageio are supported. - Aavailable formats are listed at - https://imageio.readthedocs.io/en/stable/formats/index.html#all-formats. - Some formats have additional dependencies. - - β€Š (default: ('{input_id}/001.tif',)) - --outputs {str,Tuple[str,...]} - Model output path pattern (per output tensor) - - All substrings that are replaced: - - '{model_id}' (from model description) - - '{output_id}' (from model description) - - '{sample_id}' (extracted from input paths) - - β€Š (default: outputs_{model_id}/{output_id}/{sample_id}.tif) - --overwrite bool allow overwriting existing output files (default: False) - --blockwise bool process inputs blockwise (default: False) - --stats Path path to dataset statistics - (will be written if it does not exist, - but the model requires statistical dataset measures) - β€Š (default: dataset_statistics.json) - --preview bool preview which files would be processed - and what outputs would be generated. (default: False) - --weight_format {typing.Literal['keras_hdf5','onnx','pytorch_state_dict','tensorflow_js','tensorflow_saved_model_bundle','torchscript'],any} - The weight format to use. (default: any) - --example bool generate and run an example - - 1. downloads example model inputs - 2. creates a `{model_id}_example` folder - 3. writes input arguments to `{model_id}_example/bioimageio-cli.yaml` - 4. executes a preview dry-run - 5. executes prediction with example input - - β€Š (default: False) - ``` - -
- -- create an example and run prediction locally! - - ```console - $ bioimageio predict impartial-shrimp --example=True - ... - ``` - -
- (Click to expand output) - - ```console - πŸ›ˆ bioimageio prediction preview structure: - {'{sample_id}': {'inputs': {'{input_id}': ''}, - 'outputs': {'{output_id}': ''}}} - πŸ”Ž bioimageio prediction preview output: - {'1': {'inputs': {'input0': 'impartial-shrimp_example/input0/001.tif'}, - 'outputs': {'output0': 'impartial-shrimp_example/outputs/output0/1.tif'}}} - predict with impartial-shrimp: 100%|β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 1/1 [00:21<00:00, 21.76s/sample] - πŸŽ‰ Sucessfully ran example prediction! - To predict the example input using the CLI example config file impartial-shrimp_example\bioimageio-cli.yaml, execute `bioimageio predict` from impartial-shrimp_example: - $ cd impartial-shrimp_example - $ bioimageio predict "impartial-shrimp" - - Alternatively run the following command in the current workind directory, not the example folder: - $ bioimageio predict --preview=False --overwrite=True --stats="impartial-shrimp_example/dataset_statistics.json" --inputs="[[\"impartial-shrimp_example/input0/001.tif\"]]" --outputs="impartial-shrimp_example/outputs/{output_id}/{sample_id}.tif" "impartial-shrimp" - (note that a local 'bioimageio-cli.json' or 'bioimageio-cli.yaml' may interfere with this) - ``` - -
- -## Installation - -### Via Conda - -The `bioimageio.core` package can be installed from conda-forge via - -```console -conda install -c conda-forge bioimageio.core -``` - -If you do not install any additional deep learning libraries, you will only be able to use general convenience -functionality, but not any functionality depending on model prediction. -To install additional deep learning libraries add `pytorch`, `onnxruntime`, `keras` or `tensorflow`. - -Deeplearning frameworks to consider installing alongside `bioimageio.core`: - -- [Pytorch/Torchscript](https://pytorch.org/get-started/locally/) -- [TensorFlow](https://www.tensorflow.org/install) -- [ONNXRuntime](https://onnxruntime.ai/docs/install/#python-installs) - -### Via pip - -The package is also available via pip -(e.g. with recommended extras `onnx` and `pytorch`): - -```console -pip install "bioimageio.core[onnx,pytorch]" -``` - -## 🐍 Use in Python - -`bioimageio.core` is a python package that implements prediction with bioimageio models +`bioimageio.core` is a python package that implements prediction with bioimage.io models including standardized pre- and postprocessing operations. -These models are described by---and can be loaded with---the bioimageio.spec package. - -In addition bioimageio.core provides functionality to convert model weight formats. +Such models are represented as [bioimageio.spec](https://bioimage-io.github.io/spec-bioimage-io) resource descriptions. -### Documentation +In addition bioimageio.core provides functionality to convert model weight formats +and compute selected dataset statistics used for preprocessing. -[Here you find the bioimageio.core documentation.](https://bioimage-io.github.io/core-bioimage-io-python/bioimageio/core.html) +## Documentation -#### Presentations - -- [Create a model from scratch](https://bioimage-io.github.io/core-bioimage-io-python/presentations/create_ambitious_sloth.slides.html) ([source](https://github.com/bioimage-io/core-bioimage-io-python/tree/main/presentations)) +[Here you find the bioimageio.core documentation.](https://bioimage-io.github.io/core-bioimage-io-python) #### Examples -
-
Notebooks that save and load resource descriptions and validate their format (using bioimageio.spec, a dependency of bioimageio.core)
-
load_model_and_create_your_own.ipynb +Notebooks that save and load resource descriptions and validate their format (using bioimageio.spec, a dependency of bioimageio.core) +
-
dataset_creation.ipynb + +
  • dataset_creation.ipynb Open In Colab -
    • -
    Use the described resources in Python with bioimageio.core
    -
    model_usage.ipynb - Open In Colab -
    - -## πŸ’» Use the Command Line Interface + + -`bioimageio.core` installs a command line interface (CLI) for testing models and other functionality. -You can list all the available commands via: - -```console -bioimageio -``` - -For examples see [Get started](#get-started). - -### CLI inputs from file +Use the described resources in Python with bioimageio.core + -For convenience the command line options (not arguments) may be given in a `bioimageio-cli.json` -or `bioimageio-cli.yaml` file, e.g.: +#### Presentations -```yaml -# bioimageio-cli.yaml -inputs: inputs/*_{tensor_id}.h5 -outputs: outputs_{model_id}/{sample_id}_{tensor_id}.h5 -overwrite: true -blockwise: true -stats: inputs/dataset_statistics.json -``` +- [Create a model from scratch](https://bioimage-io.github.io/core-bioimage-io-python/presentations/create_ambitious_sloth.slides.html) ([source](https://github.com/bioimage-io/core-bioimage-io-python/tree/main/presentations)) ## Set up Development Environment diff --git a/changelog.md b/changelog.md index 1ce18e61c..605be0597 100644 --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,12 @@ +### 0.9.6 + +- bump bioimageio.spec library version to 0.5.7.4 +- increase default reprducibility tolerance +- unify quantile (vs percentile) variable names +- add quantile computation method parameter +- accept `SampleQuantile` or `DatasetQuantile` as `min`/`max` arguments to `proc_ops.Clip` +- save actual output during model testing only if an explicit working directory was specified to produce less clutter + ### 0.9.5 - bump bioimageio.spec library version to 0.5.6.0 diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index b4ffb39a4..2552cf93d 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -40,7 +40,7 @@ requirements: {% endif %} {% endfor %} {% for dep in pyproject['project']['optional-dependencies']['onnx'] %} - - {{ dep.lower() }} + - {{ dep.replace(';python_version<"3.10"', '').lower().replace('_', '-') }} {% endfor %} {% for dep in pyproject['project']['optional-dependencies']['tensorflow'] %} - {{ dep.lower() }} @@ -55,7 +55,7 @@ test: requires: {% for dep in pyproject['project']['optional-dependencies']['dev'] %} {% if 'torch' not in dep %} # can't install pytorch>=2.8 from conda-forge smh - - {{ dep.lower().replace('_', '-') }} + - {{ dep.replace(';python_version<"3.10"', '').lower().replace('_', '-') }} {% endif %} {% endfor %} commands: diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 000000000..a38b0a23c --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1,5 @@ +--- +title: Changelog +--- + +--8<-- "changelog.md" diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 000000000..3db09484d --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,23 @@ +## bioimageio Command Line Interface + +`bioimageio.core` installs a command line interface (CLI) for testing models and other functionality. +You can list all the available commands via: + +```bash exec="1" source="console" result="ansi" width="200" +bioimageio --help +``` + +For concrete examples see [Get started](get-started.md). + +### CLI inputs from file + +For convenience the command line options (not arguments) may be given in a `bioimageio-cli.json` or `bioimageio-cli.yaml` file, e.g.: + +```yaml +# bioimageio-cli.yaml +inputs: inputs/*_{tensor_id}.tiff +outputs: outputs_{model_id}/{sample_id}_{tensor_id}.tiff +overwrite: true +blockwise: true +stats: inputs/dataset_statistics.json +``` diff --git a/docs/compatibility.md b/docs/compatibility.md new file mode 100644 index 000000000..5143266ee --- /dev/null +++ b/docs/compatibility.md @@ -0,0 +1,7 @@ +# Compatibility with bioimage.io resources + +bioimageio.core is used on [bioimage.io](https://bioimage.io) to test resources during and after the upload process. +Results are reported as "Test reports" (bioimageio.core deployed in a generic Python environment) +as well as the bioimageio.core tool compatibility (testing a resource with bioimageio.core in a dedicated Python environment). + +An overview of the latter is available [as part of the collection documentation](https://bioimage-io.github.io/collection/latest/reports_overview/). diff --git a/docs/get_started.md b/docs/get_started.md new file mode 100644 index 000000000..258933409 --- /dev/null +++ b/docs/get_started.md @@ -0,0 +1,122 @@ +## Finding a compatible Python environment + +For model inference you need a Python environment with the `bioimageio.core` package and model (framework) specific dependencies installed. +You may choose to install `bioimageio.core` alongside (a) suitable framework(s) as optional dependencies with pip, e.g.: + +```bash +pip install bioimageio.core[pytorch,onnx] +``` + +If you are not sure which framework you want to use this model with or the model comes with custom dependencies, +you may choose to have the bioimageio Command Line Interface (CLI) create a suitable environment for a specific model, +using [mini-forge](https://github.com/conda-forge/miniforge) (or your favorite conda distribution). +For more details on conda environments, [checkout the conda docs](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html). +First create/use any conda environment with `bioimageio.core>0.9.6` in it: + +```bash +conda create -n bioimageio -c conda-forge "bioimageio.core>0.9.6" +conda activate bioimageio +``` + +Choose a model source, e.g. a bioimage.io model id like "affable-shark" or a path/url to a bioimageio.yaml (often named rdf.yaml). +Then use the bioimageio CLI (or [bioimageio.core.test_description][]) to test the model. +Use runtime-env=as-described to test each available weight format in the recommended conda environment that is installed on the fly if necessary: + + + +```bash +bioimageio test affable-shark --runtime-env=as-described +``` + +The resulting report shows details of the tests performed in the respective conda environments. +Inspecting the report, choose a conda environment that passed all tests. +The conda environments will be named by the SHA-256 value of the generated conda environment.yaml, e.g. "95227f474ca45b024cf315edb4101e4919199d0a79ef5ff1eb474dc8ce1ec4d8". + +You may want to rename or clone your chosen conda environment: + +```bash +conda activate base +conda rename -n 95227f474ca45b024cf315edb4101e4919199d0a79ef5ff1eb474dc8ce1ec4d8 bioimageio-affable-shark +conda activate bioimageio-affable-shark +``` + +## Test model+environment + +Test a bioimageio compatible model, e.g. "affable-shark" in an active Python environment: + +```bash exec="1" source="console" result="ansi" width="200" +bioimageio test affable-shark +``` + +To test your model replace the already published model identifier 'affabl-shark' with a local folder or path to a bioimageio.yaml file. +Check out the [bioimageio.spec documentation](https://bioimage-io.github.io/spec-bioimage-io) for more information on the bioimage.io metadata description format. + +The Python equivalent would be: + +```python exec="1" souce="console" result="ansi" width="300" +from bioimageio.core import test_description + +summary = test_description("affable-shark") +summary.display() +``` + +## CLI: bioimageio predict + +You can use the `bioimageio` Command Line Interface (CLI) provided by the `bioimageio.core` package to run prediction with a bioimageio compatible model in a [suitable Python environment](#finding-a-compatible-python-environment). + +```bash exec="1" source="console" result="ansi" width="200" +bioimageio predict --help +``` + +Create a local example and run prediction locally: + +```bash exec="1" source="console" result="ansi" width="200" +bioimageio predict affable-shark --example +``` + +## Python: bioimageio.core.predict + +Here is a code snippet to get started deploying a model in Python using the test sample provided by the model description: + +```python +from bioimageio.core import load_model_description, predict +from bioimageio.core.digest_spec import get_test_input_sample + +model_descr = load_model_description("") +input_sample = get_test_input_sample(model_descr) +output_sample = predict(model=model_descr, inputs=input_sample) +``` + +### Python: predict your own data + +```python +from bioimageio.core.digest_spec import create_sample_for_model + +input_sample = create_sample_for_model( + model_descr, + inputs={{"raw": ""}} +) +output_sample = predict(model=model_descr, inputs=input_sample) +``` + +### Python: prediction options + +For model inference from within Python these options are available: + +- [bioimageio.core.predict][] to run inference on a single sample/image. +- [bioimageio.core.predict_many][] to run inference on a set of samples. +- [bioimageio.core.create_prediction_pipeline][] for reusing the instatiated model and more fine-grain control over the inference process this function creates a suitable [bioimageio.core.PredictionPipeline][] for more advanced use. + +## Other bioimageio.core functionality + +### CLI: bioimageio commands + +To get an overview of available commands: + +```bash exec="1" source="console" result="ansi" width="200" +bioimageio --help +``` + +### Python: API docs + +See [bioimageio.core][]. diff --git a/docs/images/bioimage-io-icon.png b/docs/images/bioimage-io-icon.png new file mode 100644 index 000000000..9bfd47655 Binary files /dev/null and b/docs/images/bioimage-io-icon.png differ diff --git a/docs/images/favicon.ico b/docs/images/favicon.ico new file mode 100644 index 000000000..e0b01e666 Binary files /dev/null and b/docs/images/favicon.ico differ diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..612c7a5e0 --- /dev/null +++ b/docs/index.md @@ -0,0 +1 @@ +--8<-- "README.md" diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 000000000..6aff16487 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,32 @@ +## Via Conda + +The `bioimageio.core` package can be installed from conda-forge via + +```console +conda install -c conda-forge bioimageio.core +``` + +If you do not install any additional deep learning (DL) libraries, you will only be able to use general convenience +functionality, but model inference will be unavailable. +To install additional deep learning libraries add `pytorch`, `onnxruntime`, `keras` or `tensorflow`. + +Deeplearning frameworks to consider installing alongside `bioimageio.core`: + +- [Pytorch/Torchscript](https://pytorch.org/get-started/locally/) +- [TensorFlow](https://www.tensorflow.org/install) +- [ONNXRuntime](https://onnxruntime.ai/docs/install/#python-installs) + +Example for installing bioimageio.core via conda with additional DL frameworks: + +```console +conda install -c conda-forge bioimageio.core pytorch torchvision onnxruntime +``` + +## Via pip + +The package is also available via pip +(e.g. with recommended extras `onnx` and `pytorch`): + +```console +pip install "bioimageio.core[pytorch,onnx]" +``` diff --git a/docs/use_in_python.md b/docs/use_in_python.md new file mode 100644 index 000000000..a70360ba0 --- /dev/null +++ b/docs/use_in_python.md @@ -0,0 +1,9 @@ +Here you can find recommendations for using bioimageio.core in your Python package or scripts. + +See [API reference](api/index.html) for details beyond this brief orientation. + +## Run inference + +The [bioimageio.core.predict][] and [predict_many][bioimageio.core.predict_many] aim to provide an easy-to-use interface +to run inference with a bioimage.io model. +See [Compatibility](compatibility.md) for a list of compatible models or browser the Model Zoo at [https://bioimage.io](https://bioimage.io). diff --git a/mkdocs.yaml b/mkdocs.yaml new file mode 100644 index 000000000..95f8211ca --- /dev/null +++ b/mkdocs.yaml @@ -0,0 +1,183 @@ +site_name: 'bioimageio.core' +site_url: 'https://bioimage-io.github.io/core-bioimage-io-python' +site_author: Fynn BeuttenmΓΌller +site_description: 'Python specific core utilities for bioimage.io resources (in particular DL models).' + +repo_name: bioimage-io/core-bioimage-io-python +repo_url: https://github.com/bioimage-io/core-bioimage-io-python +edit_uri: edit/main/docs/ + +theme: + name: material + language: en + features: + - announce.dismiss + - content.action.edit + - content.action.view + - content.code.annotate + - content.code.copy + - content.code.select + - content.footnote.tooltips + - content.tabs.link + - content.tooltips + - header.autohide + - navigation.expand + - navigation.footer + - navigation.indexes + - navigation.instant + - navigation.instant.prefetch + - navigation.instant.preview + - navigation.instant.progress + - navigation.path + - navigation.prune + - navigation.sections + - navigation.tabs + - navigation.tabs.sticky + - navigation.top + - navigation.tracking + - search.highlight + - search.share + - search.suggest + - toc.follow + + palette: + - media: '(prefers-color-scheme)' + primary: 'indigo' + accent: 'orange' + toggle: + icon: material/brightness-auto + name: 'Switch to light mode' + - media: '(prefers-color-scheme: light)' + scheme: default + primary: 'indigo' + accent: 'orange' + toggle: + icon: material/brightness-7 + name: 'Switch to dark mode' + - media: '(prefers-color-scheme: dark)' + scheme: slate + primary: 'indigo' + accent: 'orange' + toggle: + icon: material/brightness-4 + name: 'Switch to system preference' + + font: + text: Roboto + code: Roboto Mono + + logo: images/bioimage-io-icon.png + favicon: images/favicon.ico + +plugins: + - autorefs + - coverage: + html_report_dir: dist/coverage + - gen-files: + scripts: + - scripts/generate_api_doc_pages.py + - markdown-exec: + ansi: required + - mkdocstrings: + enable_inventory: true + default_handler: python + locale: en + handlers: + python: + inventories: + - https://docs.pydantic.dev/latest/objects.inv + - https://bioimage-io.github.io/spec-bioimage-io/v0.5.7.4/objects.inv + options: + annotations_path: source + backlinks: tree + docstring_options: + ignore_init_summary: true + returns_multiple_items: false + returns_named_value: false + trim_doctest_flags: true + # docstring_section_style: spacy + docstring_style: google + filters: public + heading_level: 1 + inherited_members: true + merge_init_into_class: true + parameter_headings: true + preload_modules: [pydantic, bioimageio.spec] + scoped_crossrefs: true + separate_signature: true + show_docstring_examples: true + show_if_no_docstring: true + show_inheritance_diagram: true + show_root_full_path: false + show_root_heading: true + show_signature_annotations: true + show_signature_type_parameters: true + show_source: true + show_submodules: false + show_symbol_type_heading: true + show_symbol_type_toc: true + signature_crossrefs: true + summary: true + unwrap_annotated: false + extensions: + - griffe_pydantic: + schema: true + - griffe_inherited_docstrings + - griffe_public_redundant_aliases + - mike: + alias_type: symlink + canonical_version: latest + version_selector: true + - literate-nav: + nav_file: SUMMARY.md + - search + - section-index + +markdown_extensions: + - attr_list + - admonition + - callouts: + strip_period: false + - footnotes + - pymdownx.details + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + - pymdownx.highlight: + pygments_lang_class: true + - pymdownx.magiclink + - pymdownx.snippets: + base_path: [!relative $config_dir] + check_paths: true + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + slugify: !!python/object/apply:pymdownx.slugs.slugify + kwds: + case: lower + - pymdownx.tasklist: + custom_checkbox: true + - pymdownx.tilde + - toc: + permalink: 'Β€' + permalink_title: Anchor link to this section for reference + toc_depth: 2 + +nav: + - Home: + - index.md + - Get started: get_started.md + - Installation: installation.md + - bioimageio CLI: cli.md + - Use in Python: use_in_python.md + - Compatibility: compatibility.md + - API Reference: api/ + - Changelog: changelog.md + - Coverage report: coverage.md + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/bioimage-io + version: + provider: mike diff --git a/pyproject.toml b/pyproject.toml index ed7e9aa39..b9830339a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,8 +6,7 @@ requires-python = ">=3.9" readme = "README.md" dynamic = ["version"] dependencies = [ - "bioimageio.spec ==0.5.6.0", - "h5py", + "bioimageio.spec ==0.5.7.4", "imagecodecs", "imageio>=2.10", "loguru", @@ -40,39 +39,58 @@ Documentation = "https://bioimage-io.github.io/core-bioimage-io-python/bioimagei Source = "https://github.com/bioimage-io/core-bioimage-io-python" [project.optional-dependencies] -onnx = ["onnxruntime", "onnxscript"] +onnx = [ + "onnxruntime", + "onnxscript", + 'onnx_ir!=0.1.14;python_version<"3.10"', # uses typing.Concatentate which requires py>=3.10 +] pytorch = ["torch>=1.6,<3", "torchvision>=0.21", "keras>=3.0,<4"] -tensorflow = ["tensorflow", "keras>=2.15,<4"] +tensorflow = ["tensorflow", "keras>=2.15,<4", "h5py"] partners = [ # "biapy", # pins core exactly "careamics", # "stardist", # for model testing and stardist postprocessing # TODO: add updated stardist to partners env ] dev = [ - "cellpose", # for model testing + "cellpose", # for model testing "crick", "httpx", "jupyter", "keras>=3.0,<4", "matplotlib", - "monai", # for model testing + "monai", # for model testing "numpy", "onnx", "onnxruntime", "onnxscript", + 'onnx_ir!=0.1.14;python_version<"3.10"', # uses typing.Concatentate which requires py>=3.10 "packaging>=17.0", - "pdoc", "pre-commit", "pyright==1.1.407", "pytest-cov", "pytest", "python-dotenv", - "segment-anything", # for model testing + "segment-anything", # for model testing "tensorflow", - "timm", # for model testing + "timm", # for model testing "torch>=1.6,<3", "torchvision>=0.21", ] +docs = [ + "griffe-pydantic", + "griffe-inherited-docstrings", + "griffe-public-redundant-aliases", + "markdown-callouts", + "markdown-exec[ansi]", + "markdown-pycon", + "mike", + "mkdocs-api-autonav", + "mkdocs-coverage", + "mkdocs-gen-files", + "mkdocs-literate-nav", + "mkdocs-material", + "mkdocs-section-index", +] [build-system] requires = ["pip", "setuptools>=61.0"] @@ -86,12 +104,12 @@ version = { attr = "bioimageio.core.__version__" } [tool.pyright] exclude = [ + "**/.*", "**/__pycache__", "**/node_modules", "dogfood", "presentations", - "scripts/pdoc/original.py", - "scripts/pdoc/patched.py", + "scripts/generate_api_doc_pages.py", "tests/old_*", ] include = ["src", "scripts", "tests"] @@ -120,7 +138,7 @@ typeCheckingMode = "strict" useLibraryCodeForTypes = true [tool.pytest.ini_options] -addopts = "--doctest-modules --failed-first --ignore dogfood --ignore src/bioimageio/core/backends --ignore src/bioimageio/core/weight_converters" +addopts = "--doctest-modules --failed-first --ignore dogfood --ignore src/bioimageio/core/backends --ignore src/bioimageio/core/weight_converters --ignore scripts/generate_api_doc_pages.py" testpaths = ["src", "tests"] [tool.ruff] @@ -135,5 +153,11 @@ exclude = [ [tool.ruff.lint] select = ["NPY201"] +[tool.ruff.lint.isort] +known-first-party = ["bioimageio"] + [tool.coverage.report] exclude_also = ["if TYPE_CHECKING:", "assert_never\\("] + +[tool.coverage.run] +patch = ["subprocess"] diff --git a/scripts/generate_api_doc_pages.py b/scripts/generate_api_doc_pages.py new file mode 100644 index 000000000..becf0f37e --- /dev/null +++ b/scripts/generate_api_doc_pages.py @@ -0,0 +1,81 @@ +"""Generate the code reference pages. +(adapted from https://mkdocstrings.github.io/recipes/#bind-pages-to-sections-themselves) +""" + +from pathlib import Path + +import mkdocs_gen_files + +nav = mkdocs_gen_files.nav.Nav() + +root = Path(__file__).parent.parent +src = root / "src" + +# Track flat nav entries we have added +added_nav_labels: set[str] = set() + +for path in sorted(src.rglob("*.py")): + module_path = path.relative_to(src).with_suffix("") + nav_path = path.relative_to(src).with_suffix(".md") + full_doc_path = Path("api", nav_path) + + parts = tuple(module_path.parts) + + # Skip if this is just the bioimageio namespace package + if parts == ("bioimageio",): + continue + + # Skip private submodules prefixed with '_' + if any( + part.startswith("_") and part not in ("__init__", "__main__") for part in parts + ): + continue + + if parts[-1] == "__init__": + parts = parts[:-1] + nav_path = nav_path.with_name("index.md") + full_doc_path = full_doc_path.with_name("index.md") + elif parts[-1] == "__main__": + continue + + if not parts: # Skip if parts is empty + continue + + # Build a flat nav for API Reference: one entry for bioimageio.core and + # one entry per top-level submodule under bioimageio.core. No subsections. + assert parts[0:2] == ("bioimageio", "core") + if len(parts) == 2: + # Landing page for bioimageio.core at api/index.md + full_doc_path = Path("api", "index.md") + nav_target = Path("index.md") + module_name = ".".join(parts) + if module_name not in added_nav_labels: + nav[(module_name,)] = nav_target.as_posix() + added_nav_labels.add(module_name) + + else: + # Top-level submodule/package directly under bioimageio.x + top = ".".join(parts[:3]) + + if top not in added_nav_labels: + pkg_init = src / "/".join(parts) / "__init__.py" + if pkg_init.exists(): + nav_target = Path("/".join(parts[:3])) / "index.md" + else: + nav_target = Path("/".join(parts[:2])) / f"{parts[2]}.md" + + nav[(top,)] = nav_target.as_posix() + added_nav_labels.add(top) + + with mkdocs_gen_files.open(full_doc_path, "w") as fd: + # Reconstruct the full identifier from the original module_path + ident = ".".join(module_path.parts) + if ident.endswith(".__init__"): + ident = ident[:-9] # Remove .__init__ + fd.write(f"::: {ident}") + print(f"Written {full_doc_path}") + + mkdocs_gen_files.set_edit_path(full_doc_path, path.relative_to(root)) + +with mkdocs_gen_files.open("api/SUMMARY.md", "w") as nav_file: + nav_file.writelines(nav.build_literate_nav()) diff --git a/scripts/pdoc/create_pydantic_patch.sh b/scripts/pdoc/create_pydantic_patch.sh deleted file mode 100755 index 05b6da6bd..000000000 --- a/scripts/pdoc/create_pydantic_patch.sh +++ /dev/null @@ -1,25 +0,0 @@ -pydantic_root=$(python -c "import pydantic;from pathlib import Path;print(Path(pydantic.__file__).parent)") -main=$pydantic_root'/main.py' -original="$(dirname "$0")/original.py" -patched="$(dirname "$0")/patched.py" - -if [ -e $original ] -then - echo "found existing $original" -else - cp --verbose $main $original -fi - -if [ -e $patched ] -then - echo "found existing $patched" -else - cp --verbose $main $patched - echo "Please update $patched, then press enter to continue" - read -fi - -patch_file="$(dirname "$0")/mark_pydantic_attrs_private.patch" -diff -au $original $patched > $patch_file -echo "content of $patch_file:" -cat $patch_file diff --git a/scripts/pdoc/mark_pydantic_attrs_private.patch b/scripts/pdoc/mark_pydantic_attrs_private.patch deleted file mode 100644 index 722d4fbb9..000000000 --- a/scripts/pdoc/mark_pydantic_attrs_private.patch +++ /dev/null @@ -1,28 +0,0 @@ ---- ./original.py 2024-11-08 15:18:37.493768700 +0100 -+++ ./patched.py 2024-11-08 15:13:54.288887700 +0100 -@@ -121,14 +121,14 @@ - # `GenerateSchema.model_schema` to work for a plain `BaseModel` annotation. - - model_config: ClassVar[ConfigDict] = ConfigDict() -- """ -+ """@private - Configuration for the model, should be a dictionary conforming to [`ConfigDict`][pydantic.config.ConfigDict]. - """ - - # Because `dict` is in the local namespace of the `BaseModel` class, we use `Dict` for annotations. - # TODO v3 fallback to `dict` when the deprecated `dict` method gets removed. - model_fields: ClassVar[Dict[str, FieldInfo]] = {} # noqa: UP006 -- """ -+ """@private - Metadata about the fields defined on the model, - mapping of field names to [`FieldInfo`][pydantic.fields.FieldInfo] objects. - -@@ -136,7 +136,7 @@ - """ - - model_computed_fields: ClassVar[Dict[str, ComputedFieldInfo]] = {} # noqa: UP006 -- """A dictionary of computed field names and their corresponding `ComputedFieldInfo` objects.""" -+ """@private A dictionary of computed field names and their corresponding `ComputedFieldInfo` objects.""" - - __class_vars__: ClassVar[set[str]] - """The names of the class variables defined on the model.""" diff --git a/scripts/pdoc/run.sh b/scripts/pdoc/run.sh deleted file mode 100755 index 74981aa5f..000000000 --- a/scripts/pdoc/run.sh +++ /dev/null @@ -1,16 +0,0 @@ -cd "$(dirname "$0")" # cd to folder this script is in - -# patch pydantic to hide pydantic attributes that somehow show up in the docs -# (not even as inherited, but as if the documented class itself would define them) -pydantic_main=$(python -c "import pydantic;from pathlib import Path;print(Path(pydantic.__file__).parent / 'main.py')") - -patch --verbose --forward -p1 $pydantic_main < mark_pydantic_attrs_private.patch - -cd ../.. # cd to repo root -pdoc \ - --docformat google \ - --logo "https://bioimage.io/static/img/bioimage-io-logo.svg" \ - --logo-link "https://bioimage.io/" \ - --favicon "https://bioimage.io/static/img/bioimage-io-icon-small.svg" \ - --footer-text "bioimageio.core $(python -c 'import bioimageio.core;print(bioimageio.core.__version__)')" \ - -o ./dist bioimageio.core bioimageio.spec # generate bioimageio.spec as well for references diff --git a/src/bioimageio/core/__init__.py b/src/bioimageio/core/__init__.py index ac51907d5..ccbbf0d4e 100644 --- a/src/bioimageio/core/__init__.py +++ b/src/bioimageio/core/__init__.py @@ -1,62 +1,85 @@ -""" -.. include:: ../../README.md +"""bioimageio.core --- core functionality for BioImage.IO resources + +The main focus on this library is to provide functionality to run prediction with +BioImage.IO models, including standardized pre- and postprocessing operations. +The BioImage.IO models (and other resources) are described by---and can be loaded with---the bioimageio.spec package. + +See `predict` and `predict_many` for straight-forward model inference +and `create_prediction_pipeline` for finer control of the inference process. + +Other notable bioimageio.core functionalities include: +- Testing BioImage.IO resources beyond format validation, e.g. by generating model outputs from test inputs. + See `test_model` or for arbitrary resource types `test_description`. +- Extending available model weight formats by converting existing ones, see `add_weights`. +- Creating and manipulating `Sample`s consisting of tensors with associated statistics. +- Computing statistics on datasets (represented as sequences of samples), see `compute_dataset_measures`. """ # ruff: noqa: E402 -__version__ = "0.9.5" +__version__ = "0.9.6" from loguru import logger logger.disable("bioimageio.core") -from bioimageio.spec import ( - ValidationSummary, - build_description, - dump_description, - load_dataset_description, - load_description, - load_description_and_validate_format_only, - load_model_description, - save_bioimageio_package, - save_bioimageio_package_as_folder, - save_bioimageio_yaml_only, - validate_format, -) +import bioimageio.spec -from . import ( - axis, - block_meta, - cli, - commands, - common, - digest_spec, - io, - model_adapters, - prediction, - proc_ops, - proc_setup, - sample, - stat_calculators, - stat_measures, - tensor, +from . import axis as axis +from . import backends as backends +from . import block_meta as block_meta +from . import cli as cli +from . import commands as commands +from . import common as common +from . import digest_spec as digest_spec +from . import io as io +from . import model_adapters as model_adapters +from . import prediction as prediction +from . import proc_ops as proc_ops +from . import proc_setup as proc_setup +from . import sample as sample +from . import stat_calculators as stat_calculators +from . import stat_measures as stat_measures +from . import tensor as tensor +from . import weight_converters as weight_converters +from ._prediction_pipeline import PredictionPipeline as PredictionPipeline +from ._prediction_pipeline import ( + create_prediction_pipeline as create_prediction_pipeline, ) -from ._prediction_pipeline import PredictionPipeline, create_prediction_pipeline -from ._resource_tests import ( - enable_determinism, - load_description_and_test, - test_description, - test_model, +from ._resource_tests import enable_determinism as enable_determinism +from ._resource_tests import load_description_and_test as load_description_and_test +from ._resource_tests import test_description as test_description +from ._resource_tests import test_model as test_model +from ._settings import Settings as Settings +from ._settings import settings as settings + +# reexports from bioimageio.spec +build_description = bioimageio.spec.build_description +dump_description = bioimageio.spec.dump_description +load_dataset_description = bioimageio.spec.load_dataset_description +load_description = bioimageio.spec.load_description +load_description_and_validate_format_only = ( + bioimageio.spec.load_description_and_validate_format_only ) -from ._settings import settings -from .axis import Axis, AxisId -from .backends import create_model_adapter -from .block_meta import BlockMeta -from .common import MemberId -from .prediction import predict, predict_many -from .sample import Sample -from .stat_calculators import compute_dataset_measures -from .stat_measures import Stat -from .tensor import Tensor -from .weight_converters import add_weights +load_model_description = bioimageio.spec.load_model_description +save_bioimageio_package = bioimageio.spec.save_bioimageio_package +save_bioimageio_package_as_folder = bioimageio.spec.save_bioimageio_package_as_folder +save_bioimageio_yaml_only = bioimageio.spec.save_bioimageio_yaml_only +validate_format = bioimageio.spec.validate_format +ValidationSummary = bioimageio.spec.ValidationSummary + + +# reexports from bioimageio.core submodules +add_weights = weight_converters.add_weights +Axis = axis.Axis +AxisId = axis.AxisId +BlockMeta = block_meta.BlockMeta +compute_dataset_measures = stat_calculators.compute_dataset_measures +create_model_adapter = backends.create_model_adapter +MemberId = common.MemberId +predict = prediction.predict +predict_many = prediction.predict_many +Sample = sample.Sample +Stat = stat_measures.Stat +Tensor = tensor.Tensor # aliases test_resource = test_description @@ -65,55 +88,3 @@ """alias of `load_description`""" load_model = load_model_description """alias of `load_model_description`""" - -__all__ = [ - "__version__", - "add_weights", - "axis", - "Axis", - "AxisId", - "block_meta", - "BlockMeta", - "build_description", - "cli", - "commands", - "common", - "compute_dataset_measures", - "create_model_adapter", - "create_prediction_pipeline", - "digest_spec", - "dump_description", - "enable_determinism", - "io", - "load_dataset_description", - "load_description_and_test", - "load_description_and_validate_format_only", - "load_description", - "load_model_description", - "load_model", - "load_resource", - "MemberId", - "model_adapters", - "predict_many", - "predict", - "prediction", - "PredictionPipeline", - "proc_ops", - "proc_setup", - "sample", - "Sample", - "save_bioimageio_package_as_folder", - "save_bioimageio_package", - "save_bioimageio_yaml_only", - "settings", - "stat_calculators", - "stat_measures", - "Stat", - "tensor", - "Tensor", - "test_description", - "test_model", - "test_resource", - "validate_format", - "ValidationSummary", -] diff --git a/src/bioimageio/core/__main__.py b/src/bioimageio/core/__main__.py index ed7c32808..123b6a9c9 100644 --- a/src/bioimageio/core/__main__.py +++ b/src/bioimageio/core/__main__.py @@ -1,6 +1,7 @@ import sys from loguru import logger +from pydantic_settings import CliApp logger.enable("bioimageio") @@ -17,8 +18,7 @@ def main(): - cli = Bioimageio() # pyright: ignore[reportCallIssue] - cli.run() + _ = CliApp.run(Bioimageio) if __name__ == "__main__": diff --git a/src/bioimageio/core/_prediction_pipeline.py b/src/bioimageio/core/_prediction_pipeline.py index 0b7717aa5..0cad757e7 100644 --- a/src/bioimageio/core/_prediction_pipeline.py +++ b/src/bioimageio/core/_prediction_pipeline.py @@ -66,7 +66,7 @@ def __init__( default_blocksize_parameter: BlocksizeParameter = 10, default_batch_size: int = 1, ) -> None: - """Use `create_prediction_pipeline` to create a `PredictionPipeline`""" + """Consider using `create_prediction_pipeline` to create a `PredictionPipeline` with sensible defaults.""" super().__init__() default_blocksize_parameter = default_ns or default_blocksize_parameter if default_ns is not None: diff --git a/src/bioimageio/core/_resource_tests.py b/src/bioimageio/core/_resource_tests.py index c4572929d..6a58cbbbd 100644 --- a/src/bioimageio/core/_resource_tests.py +++ b/src/bioimageio/core/_resource_tests.py @@ -4,6 +4,7 @@ import subprocess import sys import warnings +from contextlib import nullcontext from io import StringIO from itertools import product from pathlib import Path @@ -24,6 +25,10 @@ ) import numpy as np +from loguru import logger +from numpy.typing import NDArray +from typing_extensions import NotRequired, TypedDict, Unpack, assert_never, get_args + from bioimageio.spec import ( AnyDatasetDescr, AnyModelDescr, @@ -61,18 +66,14 @@ ValidationSummary, WarningEntry, ) -from loguru import logger -from numpy.typing import NDArray -from typing_extensions import NotRequired, TypedDict, Unpack, assert_never, get_args - -from bioimageio.core import __version__ -from bioimageio.core.io import save_tensor +from . import __version__ from ._prediction_pipeline import create_prediction_pipeline from ._settings import settings from .axis import AxisId, BatchSize from .common import MemberId, SupportedWeightsFormat from .digest_spec import get_test_input_sample, get_test_output_sample +from .io import save_tensor from .sample import Sample CONDA_CMD = "conda.bat" if platform.system() == "Windows" else "conda" @@ -211,6 +212,7 @@ def test_description( Literal["currently-active", "as-described"], Path, BioimageioCondaEnv ] = ("currently-active"), run_command: Callable[[Sequence[str]], None] = default_run_command, + working_dir: Optional[Union[os.PathLike[str], str]] = None, **deprecated: Unpack[DeprecatedKwargs], ) -> ValidationSummary: """Test a bioimage.io resource dynamically, @@ -236,6 +238,9 @@ def test_description( run_command: (Experimental feature!) Function to execute (conda) terminal commands in a subprocess. The function should raise an exception if the command fails. **run_command** is ignored if **runtime_env** is `"currently-active"`. + working_dir: (for debugging) directory to save any temporary files + (model packages, conda environments, test summaries). + Defaults to a temporary directory. """ if runtime_env == "currently-active": rd = load_description_and_test( @@ -247,6 +252,7 @@ def test_description( expected_type=expected_type, sha256=sha256, stop_early=stop_early, + working_dir=working_dir, **deprecated, ) return rd.validation_summary @@ -260,19 +266,26 @@ def test_description( else: assert_never(runtime_env) - try: - run_command(["thiscommandshouldalwaysfail", "please"]) - except Exception: - pass - else: - raise RuntimeError( - "given run_command does not raise an exception for a failing command" + if run_command is not default_run_command: + try: + run_command(["thiscommandshouldalwaysfail", "please"]) + except Exception: + pass + else: + raise RuntimeError( + "given run_command does not raise an exception for a failing command" + ) + + verbose = working_dir is not None + if working_dir is None: + td_kwargs: Dict[str, Any] = ( + dict(ignore_cleanup_errors=True) if sys.version_info >= (3, 10) else {} ) + working_dir_ctxt = TemporaryDirectory(**td_kwargs) + else: + working_dir_ctxt = nullcontext(working_dir) - td_kwargs: Dict[str, Any] = ( - dict(ignore_cleanup_errors=True) if sys.version_info >= (3, 10) else {} - ) - with TemporaryDirectory(**td_kwargs) as _d: + with working_dir_ctxt as _d: working_dir = Path(_d) if isinstance(source, ResourceDescrBase): @@ -307,6 +320,7 @@ def test_description( sha256=sha256, stop_early=stop_early, run_command=run_command, + verbose=verbose, **deprecated, ) @@ -326,6 +340,7 @@ def _test_in_env( stop_early: bool, expected_type: Optional[str], sha256: Optional[Sha256], + verbose: bool, **deprecated: Unpack[DeprecatedKwargs], ): """Test a bioimage.io resource in a given conda environment. @@ -356,6 +371,7 @@ def _test_in_env( expected_type=expected_type, sha256=sha256, stop_early=stop_early, + verbose=verbose, **deprecated, ) @@ -391,7 +407,7 @@ def _test_in_env( test_loc = () - # remove name as we crate a name based on the env description hash value + # remove name as we create a name based on the env description hash value conda_env.name = None dumped_env = conda_env.model_dump(mode="json", exclude_none=True) @@ -447,6 +463,22 @@ def _test_in_env( ) ) return + else: + descr.validation_summary.add_detail( + ValidationDetail( + name=f"Created conda environment '{env_name}'", + status="passed", + loc=test_loc, + ) + ) + else: + descr.validation_summary.add_detail( + ValidationDetail( + name=f"Found existing conda environment '{env_name}'", + status="passed", + loc=test_loc, + ) + ) working_dir.mkdir(parents=True, exist_ok=True) summary_path = working_dir / "summary.json" @@ -468,6 +500,7 @@ def _test_in_env( f"--{summary_path_arg_name}={summary_path.as_posix()}", f"--determinism={determinism}", ] + + ([f"--weight-format={weight_format}"] if weight_format else []) + ([f"--expected-type={expected_type}"] if expected_type else []) + (["--stop-early"] if stop_early else []) ) @@ -500,7 +533,7 @@ def _test_in_env( # add relevant details from command summary command_summary = ValidationSummary.load_json(summary_path) for detail in command_summary.details: - if detail.loc[: len(test_loc)] == test_loc: + if detail.loc[: len(test_loc)] == test_loc or detail.status == "failed": descr.validation_summary.add_detail(detail) @@ -515,6 +548,7 @@ def load_description_and_test( expected_type: Literal["model"], sha256: Optional[Sha256] = None, stop_early: bool = True, + working_dir: Optional[Union[os.PathLike[str], str]] = None, **deprecated: Unpack[DeprecatedKwargs], ) -> Union[ModelDescr, InvalidDescr]: ... @@ -530,6 +564,7 @@ def load_description_and_test( expected_type: Literal["dataset"], sha256: Optional[Sha256] = None, stop_early: bool = True, + working_dir: Optional[Union[os.PathLike[str], str]] = None, **deprecated: Unpack[DeprecatedKwargs], ) -> Union[DatasetDescr, InvalidDescr]: ... @@ -545,6 +580,7 @@ def load_description_and_test( expected_type: Optional[str] = None, sha256: Optional[Sha256] = None, stop_early: bool = True, + working_dir: Optional[Union[os.PathLike[str], str]] = None, **deprecated: Unpack[DeprecatedKwargs], ) -> Union[LatestResourceDescr, InvalidDescr]: ... @@ -560,6 +596,7 @@ def load_description_and_test( expected_type: Literal["model"], sha256: Optional[Sha256] = None, stop_early: bool = True, + working_dir: Optional[Union[os.PathLike[str], str]] = None, **deprecated: Unpack[DeprecatedKwargs], ) -> Union[AnyModelDescr, InvalidDescr]: ... @@ -575,6 +612,7 @@ def load_description_and_test( expected_type: Literal["dataset"], sha256: Optional[Sha256] = None, stop_early: bool = True, + working_dir: Optional[Union[os.PathLike[str], str]] = None, **deprecated: Unpack[DeprecatedKwargs], ) -> Union[AnyDatasetDescr, InvalidDescr]: ... @@ -590,6 +628,7 @@ def load_description_and_test( expected_type: Optional[str] = None, sha256: Optional[Sha256] = None, stop_early: bool = True, + working_dir: Optional[Union[os.PathLike[str], str]] = None, **deprecated: Unpack[DeprecatedKwargs], ) -> Union[ResourceDescr, InvalidDescr]: ... @@ -604,6 +643,7 @@ def load_description_and_test( expected_type: Optional[str] = None, sha256: Optional[Sha256] = None, stop_early: bool = True, + working_dir: Optional[Union[os.PathLike[str], str]] = None, **deprecated: Unpack[DeprecatedKwargs], ) -> Union[ResourceDescr, InvalidDescr]: """Test a bioimage.io resource dynamically, @@ -676,7 +716,15 @@ def load_description_and_test( enable_determinism(determinism, weight_formats=weight_formats) for w in weight_formats: - _test_model_inference(rd, w, devices, stop_early=stop_early, **deprecated) + _test_model_inference( + rd, + w, + devices, + stop_early=stop_early, + working_dir=working_dir, + verbose=working_dir is not None, + **deprecated, + ) if stop_early and rd.validation_summary.status != "passed": break @@ -710,7 +758,7 @@ def _get_tolerance( if wf == weights_format: applicable = v0_5.ReproducibilityTolerance( relative_tolerance=test_kwargs.get("relative_tolerance", 1e-3), - absolute_tolerance=test_kwargs.get("absolute_tolerance", 1e-4), + absolute_tolerance=test_kwargs.get("absolute_tolerance", 1e-3), ) break @@ -739,7 +787,7 @@ def _get_tolerance( mismatched_tol = 0 else: # use given (deprecated) test kwargs - atol = deprecated.get("absolute_tolerance", 1e-5) + atol = deprecated.get("absolute_tolerance", 1e-3) rtol = deprecated.get("relative_tolerance", 1e-3) mismatched_tol = 0 @@ -751,6 +799,9 @@ def _test_model_inference( weight_format: SupportedWeightsFormat, devices: Optional[Sequence[str]], stop_early: bool, + *, + working_dir: Optional[Union[os.PathLike[str], str]], + verbose: bool, **deprecated: Unpack[DeprecatedKwargs], ) -> None: test_name = f"Reproduce test outputs from test inputs ({weight_format})" @@ -834,15 +885,20 @@ def add_warning_entry(msg: str): if not mismatched_elements: continue - actual_output_path = Path(f"actual_output_{m}_{weight_format}.npy") - try: - save_tensor(actual_output_path, actual) - except Exception as e: - logger.error( - "Failed to save actual output tensor to {}: {}", - actual_output_path, - e, + if working_dir is not None and verbose: + actual_output_path = ( + Path(working_dir) / f"actual_output_{m}_{weight_format}.npy" ) + try: + save_tensor(actual_output_path, actual) + except Exception as e: + logger.error( + "Failed to save actual output tensor to {}: {}", + actual_output_path, + e, + ) + else: + actual_output_path = None mismatched_ppm = mismatched_elements / expected_np.size * 1e6 abs_diff[~mismatched] = 0 # ignore non-mismatched elements @@ -874,13 +930,15 @@ def add_warning_entry(msg: str): f"Output '{m}' disagrees with {mismatched_elements} of" + f" {expected_np.size} expected values" + f" ({mismatched_ppm:.1f} ppm)." - + f"\n Max relative difference: {r_max:.2e}" + + f"\n Max relative difference not accounted for by absolute tolerance ({atol:.2e}): {r_max:.2e}" + rf" (= \|{r_actual:.2e} - {r_expected:.2e}\|/\|{r_expected:.2e} + 1e-6\|)" + f" at {dict(zip(dims, r_max_idx))}" - + f"\n Max absolute difference not accounted for by relative tolerance: {a_max:.2e}" + + f"\n Max absolute difference not accounted for by relative tolerance ({rtol:.2e}): {a_max:.2e}" + rf" (= \|{a_actual:.7e} - {a_expected:.7e}\|) at {dict(zip(dims, a_max_idx))}" - + f"\n Saved actual output to {actual_output_path}." ) + if actual_output_path is not None: + msg += f"\n Saved actual output to {actual_output_path}." + if mismatched_ppm > mismatched_tol: add_error_entry(msg) if stop_early: diff --git a/src/bioimageio/core/backends/keras_backend.py b/src/bioimageio/core/backends/keras_backend.py index b11fb7181..a2e95d3e6 100644 --- a/src/bioimageio/core/backends/keras_backend.py +++ b/src/bioimageio/core/backends/keras_backend.py @@ -4,7 +4,6 @@ from tempfile import TemporaryDirectory from typing import Any, Optional, Sequence, Union -import h5py # pyright: ignore[reportMissingTypeStubs] from keras.src.legacy.saving import ( # pyright: ignore[reportMissingTypeStubs] legacy_h5_format, ) @@ -76,6 +75,8 @@ def __init__( weight_reader = model_description.weights.keras_hdf5.get_reader() if weight_reader.suffix in (".h5", "hdf5"): + import h5py # pyright: ignore[reportMissingTypeStubs] + h5_file = h5py.File(weight_reader, mode="r") self._network = legacy_h5_format.load_model_from_hdf5(h5_file) else: diff --git a/src/bioimageio/core/block_meta.py b/src/bioimageio/core/block_meta.py index 688463dec..f2849e02c 100644 --- a/src/bioimageio/core/block_meta.py +++ b/src/bioimageio/core/block_meta.py @@ -51,24 +51,25 @@ class BlockMeta: The inner slice (thin) is expanded by a halo in both dimensions on both sides. The outer slice reaches from the sample member origin (0, 0) to the right halo point. - ```terminal + ``` + first block (at the sample origin) β”Œ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─┐ β•· halo(left) β•· - β•· β•· + β•· padding outside the sample β•· β•· (0, 0)β”β”β”β”β”β”β”β”β”β”β”β”β”β”β”β”β”β”β”―β”β”β”β”β”β”β”β”β”β”―β”β”β”βž” β•· ┃ β”‚ β•· sample member - β•· ┃ inner β”‚ β•· - β•· ┃ (and outer) β”‚ outer β•· - β•· ┃ slice β”‚ slice β•· + β•· ┃ inner β”‚ outer β•· + β•· ┃ region β”‚ region β•· + β•· ┃ /slice β”‚ /slice β•· β•· ┃ β”‚ β•· β•· β”£β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β•· - β•· ┃ outer slice β•· + β•· ┃ outer region/slice β•· β•· ┃ halo(right) β•· β”” ─ ─ ─ ─┃─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ β”€β”˜ ⬇ ``` - note: + Note: - Inner and outer slices are specified in sample member coordinates. - The outer_slice of a block at the sample edge may overlap by more than the halo with the neighboring block (the inner slices will not overlap though). @@ -178,7 +179,7 @@ def tagged_shape(self) -> PerAxis[int]: return self.shape @property - def inner_slice_wo_overlap(self): + def inner_slice_wo_overlap(self) -> PerAxis[SliceInfo]: """subslice of the inner slice, such that all `inner_slice_wo_overlap` can be stiched together trivially to form the original sample. diff --git a/src/bioimageio/core/cli.py b/src/bioimageio/core/cli.py index ff24f1ece..8fc4ff668 100644 --- a/src/bioimageio/core/cli.py +++ b/src/bioimageio/core/cli.py @@ -16,6 +16,7 @@ from pathlib import Path from pprint import pformat, pprint from typing import ( + Annotated, Any, Dict, Iterable, @@ -32,9 +33,17 @@ import rich.markdown from loguru import logger -from pydantic import AliasChoices, BaseModel, Field, model_validator +from pydantic import ( + AliasChoices, + BaseModel, + Field, + PlainSerializer, + WithJsonSchema, + model_validator, +) from pydantic_settings import ( BaseSettings, + CliApp, CliPositionalArg, CliSettingsSource, CliSubCommand, @@ -64,7 +73,12 @@ from bioimageio.spec.dataset import DatasetDescr from bioimageio.spec.model import ModelDescr, v0_4, v0_5 from bioimageio.spec.notebook import NotebookDescr -from bioimageio.spec.utils import ensure_description_is_model, get_reader, write_yaml +from bioimageio.spec.utils import ( + empty_cache, + ensure_description_is_model, + get_reader, + write_yaml, +) from .commands import WeightFormatArgAll, WeightFormatArgAny, package, test from .common import MemberId, SampleId, SupportedWeightsFormat @@ -160,7 +174,7 @@ class ValidateFormatCmd(CmdBase, WithSource, WithSummaryLogging): def descr(self): return load_description(self.source, perform_io_checks=self.perform_io_checks) - def run(self): + def cli_cmd(self): self.log(self.descr) sys.exit( 0 @@ -195,6 +209,9 @@ class TestCmd(CmdBase, WithSource, WithSummaryLogging): Note: The `bioimageio.core` dependency will be added automatically if not present. """ + working_dir: Optional[Path] = Field(None, alias="working-dir") + """(for debugging) Directory to save any temporary files.""" + determinism: Literal["seed_only", "full"] = "seed_only" """Modes to improve reproducibility of test outputs.""" @@ -212,7 +229,7 @@ class TestCmd(CmdBase, WithSource, WithSummaryLogging): - '0.4', '0.5', ...: Use the specified format version (may trigger auto updating) """ - def run(self): + def cli_cmd(self): sys.exit( test( self.descr, @@ -222,6 +239,7 @@ def run(self): runtime_env=self.runtime_env, determinism=self.determinism, format_version=self.format_version, + working_dir=self.working_dir, ) ) @@ -241,7 +259,7 @@ class PackageCmd(CmdBase, WithSource, WithSummaryLogging): ) """The weight format to include in the package (for model descriptions only).""" - def run(self): + def cli_cmd(self): if isinstance(self.descr, InvalidDescr): self.log(self.descr) raise ValueError(f"Invalid {self.descr.type} description.") @@ -314,7 +332,7 @@ class UpdateCmdBase(CmdBase, WithSource, ABC): def updated(self) -> Union[ResourceDescr, InvalidDescr]: raise NotImplementedError - def run(self): + def cli_cmd(self): original_yaml = open_bioimageio_yaml(self.source).unparsed_content assert isinstance(original_yaml, str) stream = StringIO() @@ -450,7 +468,11 @@ class PredictCmd(CmdBase, WithSource): blockwise: bool = False """process inputs blockwise""" - stats: Path = Path("dataset_statistics.json") + stats: Annotated[ + Path, + WithJsonSchema({"type": "string"}), + PlainSerializer(lambda p: p.as_posix(), return_type=str), + ] = Path("dataset_statistics.json") """path to dataset statistics (will be written if it does not exist, but the model requires statistical dataset measures) @@ -574,7 +596,7 @@ def get_example_command(preview: bool, escape: bool = False): + f"\n(note that a local '{JSON_FILE}' or '{YAML_FILE}' may interfere with this)" ) - def run(self): + def cli_cmd(self): if self.example: return self._example() @@ -655,7 +677,7 @@ def expand_outputs(): ) for s in sample_ids ] - + # check for distinctness and correct number within each output sample for i, out in enumerate(outputs, start=1): if len(set(out)) < len(out): raise ValueError( @@ -667,6 +689,14 @@ def expand_outputs(): f"[output sample #{i}] Expected {len(output_ids)} outputs {output_ids}, got {out}" ) + # check for distinctness across all output samples + all_output_paths = [p for out in outputs for p in out] + if len(set(all_output_paths)) < len(all_output_paths): + raise ValueError( + "Output paths are not distinct across samples. " + + f"Make sure to include '{{sample_id}}' in the output path pattern." + ) + return outputs outputs = expand_outputs() @@ -742,6 +772,8 @@ def input_dataset(stat: Stat): class AddWeightsCmd(CmdBase, WithSource, WithSummaryLogging): + """Add additional weights to a model description by converting from available formats.""" + output: CliPositionalArg[Path] """The path to write the updated model package to.""" @@ -758,7 +790,7 @@ class AddWeightsCmd(CmdBase, WithSource, WithSummaryLogging): """Allow tracing when converting pytorch_state_dict to torchscript (still uses scripting if possible).""" - def run(self): + def cli_cmd(self): model_descr = ensure_description_is_model(self.descr) if isinstance(model_descr, v0_4.ModelDescr): raise TypeError( @@ -776,6 +808,13 @@ def run(self): self.log(updated_model_descr) +class EmptyCache(CmdBase): + """Empty the bioimageio cache directory.""" + + def cli_cmd(self): + empty_cache() + + JSON_FILE = "bioimageio-cli.json" YAML_FILE = "bioimageio-cli.yaml" @@ -814,8 +853,10 @@ class Bioimageio( """Create a bioimageio.yaml description with updated file hashes.""" add_weights: CliSubCommand[AddWeightsCmd] = Field(alias="add-weights") - """Add additional weights to the model descriptions converted from available - formats to improve deployability.""" + """Add additional weights to a model description by converting from available formats.""" + + empty_cache: CliSubCommand[EmptyCache] = Field(alias="empty-cache") + """Empty the bioimageio cache directory.""" @classmethod def settings_customise_sources( @@ -849,22 +890,12 @@ def _log(cls, data: Any): ) return data - def run(self): + def cli_cmd(self) -> None: logger.info( "executing CLI command:\n{}", pformat({k: v for k, v in self.model_dump().items() if v is not None}), ) - cmd = ( - self.add_weights - or self.package - or self.predict - or self.test - or self.update_format - or self.update_hashes - or self.validate_format - ) - assert cmd is not None - cmd.run() + _ = CliApp.run_subcommand(self) assert isinstance(Bioimageio.__doc__, str) diff --git a/src/bioimageio/core/commands.py b/src/bioimageio/core/commands.py index 61d0bd4be..025f177ba 100644 --- a/src/bioimageio/core/commands.py +++ b/src/bioimageio/core/commands.py @@ -52,10 +52,11 @@ def test( ] = "currently-active", determinism: Literal["seed_only", "full"] = "seed_only", format_version: Union[FormatVersionPlaceholder, str] = "discover", + working_dir: Optional[Path] = None, ) -> int: """Test a bioimageio resource. - Arguments as described in `bioimageio.core.cli.TestCmd` + Arguments as described in [bioimageio.core.cli.TestCmd][] """ if isinstance(descr, InvalidDescr): test_summary = descr.validation_summary @@ -67,6 +68,7 @@ def test( devices=[devices] if isinstance(devices, str) else devices, runtime_env=runtime_env, determinism=determinism, + working_dir=working_dir, ) _ = test_summary.log(summary) @@ -102,7 +104,7 @@ def package( Args: descr: a bioimageio resource description path: output path - weight-format: include only this single weight-format (if not 'all'). + weight_format: include only this single weight-format (if not 'all'). """ if isinstance(descr, InvalidDescr): logged = descr.validation_summary.save() diff --git a/src/bioimageio/core/common.py b/src/bioimageio/core/common.py index 9f939061c..2d9426512 100644 --- a/src/bioimageio/core/common.py +++ b/src/bioimageio/core/common.py @@ -25,6 +25,30 @@ "torchscript", ] +QuantileMethod = Literal[ + "inverted_cdf", + # "averaged_inverted_cdf", + # "closest_observation", + # "interpolated_inverted_cdf", + # "hazen", + # "weibull", + "linear", + # "median_unbiased", + # "normal_unbiased", +] +"""Methods to use when the desired quantile lies between two data points. +See https://numpy.org/devdocs/reference/generated/numpy.quantile.html#numpy-quantile for details. + +Note: + Only relevant for `SampleQuantile` measures, as `DatasetQuantile` measures computed by [bioimageio.core.stat_calculators.][] are approximations (and use the "linear" method for each sample quantiles) + +!!! warning + Limited choices to map more easily to bioimageio.spec descriptions. + Current implementations: + - [bioimageio.spec.model.v0_5.ClipKwargs][] implies "inverted_cdf" for sample quantiles and "linear" (numpy's default) for dataset quantiles. + - [bioimageio.spec.model.v0_5.ScaleRangeKwargs][] implies "linear" (numpy's default) + +""" DTypeStr = Literal[ "bool", diff --git a/src/bioimageio/core/io.py b/src/bioimageio/core/io.py index 55a87bdad..6f6d8deb5 100644 --- a/src/bioimageio/core/io.py +++ b/src/bioimageio/core/io.py @@ -1,24 +1,21 @@ import collections.abc import warnings import zipfile -from pathlib import Path, PurePosixPath +from pathlib import Path from shutil import copyfileobj from typing import ( Any, Mapping, Optional, Sequence, - Tuple, TypeVar, Union, ) -import h5py # pyright: ignore[reportMissingTypeStubs] from imageio.v3 import imread, imwrite # type: ignore from loguru import logger from numpy.typing import NDArray from pydantic import BaseModel, ConfigDict, TypeAdapter -from typing_extensions import assert_never from bioimageio.spec._internal.io import get_reader, interprete_file_source from bioimageio.spec._internal.type_guards import is_ndarray @@ -32,17 +29,12 @@ ) from bioimageio.spec.utils import download, load_array, save_array -from .axis import AxisLike +from .axis import AxisId, AxisLike from .common import PerMember from .sample import Sample from .stat_measures import DatasetMeasure, MeasureValue from .tensor import Tensor -DEFAULT_H5_DATASET_PATH = "data" - - -SUFFIXES_WITH_DATAPATH = (".h5", ".hdf", ".hdf5") - def load_image( source: Union[ZipPath, PermissiveFileSource], is_volume: Optional[bool] = None @@ -62,51 +54,14 @@ def load_image( parsed_source = interprete_file_source(source) if isinstance(parsed_source, RelativeFilePath): - src = parsed_source.absolute() - else: - src = parsed_source - - if isinstance(src, Path): - file_source, suffix, subpath = _split_dataset_path(src) - elif isinstance(src, HttpUrl): - file_source, suffix, subpath = _split_dataset_path(src) - elif isinstance(src, ZipPath): - file_source, suffix, subpath = _split_dataset_path(src) - else: - assert_never(src) - - if suffix == ".npy": - if subpath is not None: - logger.warning( - "Unexpected subpath {} for .npy source {}", subpath, file_source - ) - - image = load_array(file_source) - elif suffix in SUFFIXES_WITH_DATAPATH: - if subpath is None: - dataset_path = DEFAULT_H5_DATASET_PATH - else: - dataset_path = str(subpath) - - reader = download(file_source) + parsed_source = parsed_source.absolute() - with h5py.File(reader, "r") as f: - h5_dataset = f.get( # pyright: ignore[reportUnknownVariableType] - dataset_path - ) - if not isinstance(h5_dataset, h5py.Dataset): - raise ValueError( - f"{file_source} did not load as {h5py.Dataset}, but has type " - + str( - type(h5_dataset) # pyright: ignore[reportUnknownArgumentType] - ) - ) - image: NDArray[Any] - image = h5_dataset[:] # pyright: ignore[reportUnknownVariableType] + if parsed_source.suffix == ".npy": + image = load_array(parsed_source) else: - reader = download(file_source) + reader = download(parsed_source) image = imread( # pyright: ignore[reportUnknownVariableType] - reader.read(), extension=suffix + reader.read(), extension=parsed_source.suffix ) assert is_ndarray(image) @@ -127,62 +82,6 @@ def load_tensor( Suffix = str -def _split_dataset_path( - source: _SourceT, -) -> Tuple[_SourceT, Suffix, Optional[PurePosixPath]]: - """Split off subpath (e.g. internal h5 dataset path) - from a file path following a file extension. - - Examples: - >>> _split_dataset_path(Path("my_file.h5/dataset")) - (...Path('my_file.h5'), '.h5', PurePosixPath('dataset')) - - >>> _split_dataset_path(Path("my_plain_file")) - (...Path('my_plain_file'), '', None) - - """ - if isinstance(source, RelativeFilePath): - src = source.absolute() - else: - src = source - - del source - - def separate_pure_path(path: PurePosixPath): - for p in path.parents: - if p.suffix in SUFFIXES_WITH_DATAPATH: - return p, p.suffix, PurePosixPath(path.relative_to(p)) - - return path, path.suffix, None - - if isinstance(src, HttpUrl): - file_path, suffix, data_path = separate_pure_path(PurePosixPath(src.path or "")) - - if data_path is None: - return src, suffix, None - - return ( - HttpUrl(str(file_path).replace(f"/{data_path}", "")), - suffix, - data_path, - ) - - if isinstance(src, ZipPath): - file_path, suffix, data_path = separate_pure_path(PurePosixPath(str(src))) - - if data_path is None: - return src, suffix, None - - return ( - ZipPath(str(file_path).replace(f"/{data_path}", "")), - suffix, - data_path, - ) - - file_path, suffix, data_path = separate_pure_path(PurePosixPath(src)) - return Path(file_path), suffix, data_path - - def save_tensor(path: Union[Path, str], tensor: Tensor) -> None: # TODO: save axis meta data @@ -190,32 +89,26 @@ def save_tensor(path: Union[Path, str], tensor: Tensor) -> None: tensor.data.to_numpy() ) assert is_ndarray(data) - file_path, suffix, subpath = _split_dataset_path(Path(path)) - if not suffix: + path = Path(path) + if not path.suffix: raise ValueError(f"No suffix (needed to decide file format) found in {path}") - file_path.parent.mkdir(exist_ok=True, parents=True) - if file_path.suffix == ".npy": - if subpath is not None: - raise ValueError(f"Unexpected subpath {subpath} found in .npy path {path}") - save_array(file_path, data) - elif suffix in (".h5", ".hdf", ".hdf5"): - if subpath is None: - dataset_path = DEFAULT_H5_DATASET_PATH - else: - dataset_path = str(subpath) - - with h5py.File(file_path, "a") as f: - if dataset_path in f: - del f[dataset_path] - - _ = f.create_dataset(dataset_path, data=data, chunks=True) + extension = path.suffix.lower() + path.parent.mkdir(exist_ok=True, parents=True) + if extension == ".npy": + save_array(path, data) + elif extension in (".h5", ".hdf", ".hdf5"): + raise NotImplementedError("Saving to h5 with dataset path is not implemented.") else: - # if singleton_axes := [a for a, s in tensor.tagged_shape.items() if s == 1]: - # tensor = tensor[{a: 0 for a in singleton_axes}] - # singleton_axes_msg = f"(without singleton axes {singleton_axes}) " - # else: - singleton_axes_msg = "" + if ( + extension in (".tif", ".tiff") + and tensor.tagged_shape.get(ba := AxisId("batch")) == 1 + ): + # remove singleton batch axis for saving + tensor = tensor[{ba: 0}] + singleton_axes_msg = f"(without singleton batch axes) " + else: + singleton_axes_msg = "" logger.debug( "writing tensor {} {}to {}", @@ -223,7 +116,7 @@ def save_tensor(path: Union[Path, str], tensor: Tensor) -> None: singleton_axes_msg, path, ) - imwrite(path, data) + imwrite(path, data, extension=extension) def save_sample( @@ -309,19 +202,6 @@ def ensure_unzipped( return out_path -def get_suffix(source: Union[ZipPath, FileSource]) -> str: - if isinstance(source, Path): - return source.suffix - elif isinstance(source, ZipPath): - return source.suffix - if isinstance(source, RelativeFilePath): - return source.path.suffix - elif isinstance(source, ZipPath): - return source.suffix - elif isinstance(source, HttpUrl): - if source.path is None: - return "" - else: - return PurePosixPath(source.path).suffix - else: - assert_never(source) +def get_suffix(source: Union[ZipPath, FileSource]) -> Suffix: + """DEPRECATED: use source.suffix instead.""" + return source.suffix diff --git a/src/bioimageio/core/proc_ops.py b/src/bioimageio/core/proc_ops.py index 65c4975aa..d50d7070e 100644 --- a/src/bioimageio/core/proc_ops.py +++ b/src/bioimageio/core/proc_ops.py @@ -1,6 +1,7 @@ import collections.abc from abc import ABC, abstractmethod from dataclasses import InitVar, dataclass, field +from functools import partial from typing import ( Collection, Literal, @@ -32,7 +33,7 @@ from .stat_measures import ( DatasetMean, DatasetMeasure, - DatasetPercentile, + DatasetQuantile, DatasetStd, MeanMeasure, Measure, @@ -230,19 +231,75 @@ def from_proc_descr( @dataclass class Clip(_SimpleOperator): - min: Optional[float] = None + min: Optional[Union[float, SampleQuantile, DatasetQuantile]] = None """minimum value for clipping""" - max: Optional[float] = None + max: Optional[Union[float, SampleQuantile, DatasetQuantile]] = None """maximum value for clipping""" def __post_init__(self): - assert self.min is not None or self.max is not None, "missing min or max value" - assert self.min is None or self.max is None or self.min < self.max, ( - f"expected min < max, but {self.min} !< {self.max}" - ) + if self.min is None and self.max is None: + raise ValueError("missing min or max value") + + if ( + isinstance(self.min, float) + and isinstance(self.max, float) + and self.min >= self.max + ): + raise ValueError(f"expected min < max, but {self.min} >= {self.max}") + + if isinstance(self.min, (SampleQuantile, DatasetQuantile)) and isinstance( + self.max, (SampleQuantile, DatasetQuantile) + ): + if self.min.axes != self.max.axes: + raise NotImplementedError( + f"expected min and max quantiles with same axes, but got {self.min.axes} and {self.max.axes}" + ) + if self.min.q >= self.max.q: + raise ValueError( + f"expected min quantile < max quantile, but {self.min.q} >= {self.max.q}" + ) + + @property + def required_measures(self): + return { + arg + for arg in (self.min, self.max) + if isinstance(arg, (SampleQuantile, DatasetQuantile)) + } def _apply(self, x: Tensor, stat: Stat) -> Tensor: - return x.clip(self.min, self.max) + if isinstance(self.min, (SampleQuantile, DatasetQuantile)): + min_value = stat[self.min] + if isinstance(min_value, (int, float)): + # use clip for scalar value + min_clip_arg = min_value + else: + # clip does not support non-scalar values + x = Tensor.from_xarray( + x.data.where(x.data >= min_value.data, min_value.data) + ) + min_clip_arg = None + else: + min_clip_arg = self.min + + if isinstance(self.max, (SampleQuantile, DatasetQuantile)): + max_value = stat[self.max] + if isinstance(max_value, (int, float)): + # use clip for scalar value + max_clip_arg = max_value + else: + # clip does not support non-scalar values + x = Tensor.from_xarray( + x.data.where(x.data <= max_value.data, max_value.data) + ) + max_clip_arg = None + else: + max_clip_arg = self.max + + if min_clip_arg is not None or max_clip_arg is not None: + x = x.clip(min_clip_arg, max_clip_arg) + + return x def get_output_shape( self, input_shape: Mapping[AxisId, int] @@ -253,11 +310,46 @@ def get_output_shape( def from_proc_descr( cls, descr: Union[v0_4.ClipDescr, v0_5.ClipDescr], member_id: MemberId ) -> Self: + if isinstance(descr, v0_5.ClipDescr): + dataset_mode, axes = _get_axes(descr.kwargs) + if dataset_mode: + Quantile = DatasetQuantile + else: + Quantile = partial(SampleQuantile, method="inverted_cdf") + + if descr.kwargs.min is not None: + min_arg = descr.kwargs.min + elif descr.kwargs.min_percentile is not None: + min_arg = Quantile( + q=descr.kwargs.min_percentile / 100, + axes=axes, + member_id=member_id, + ) + else: + min_arg = None + + if descr.kwargs.max is not None: + max_arg = descr.kwargs.max + elif descr.kwargs.max_percentile is not None: + max_arg = Quantile( + q=descr.kwargs.max_percentile / 100, + axes=axes, + member_id=member_id, + ) + else: + max_arg = None + + elif isinstance(descr, v0_4.ClipDescr): + min_arg = descr.kwargs.min + max_arg = descr.kwargs.max + else: + assert_never(descr) + return cls( input=member_id, output=member_id, - min=descr.kwargs.min, - max=descr.kwargs.max, + min=min_arg, + max=max_arg, ) @@ -404,6 +496,7 @@ def _get_axes( v0_5.ScaleRangeKwargs, v0_4.ScaleMeanVarianceKwargs, v0_5.ScaleMeanVarianceKwargs, + v0_5.ClipKwargs, ], ) -> Tuple[bool, Optional[Tuple[AxisId, ...]]]: if kwargs.axes is None: @@ -420,28 +513,28 @@ def _get_axes( @dataclass class ScaleRange(_SimpleOperator): - lower_percentile: InitVar[Optional[Union[SampleQuantile, DatasetPercentile]]] = None - upper_percentile: InitVar[Optional[Union[SampleQuantile, DatasetPercentile]]] = None - lower: Union[SampleQuantile, DatasetPercentile] = field(init=False) - upper: Union[SampleQuantile, DatasetPercentile] = field(init=False) + lower_quantile: InitVar[Optional[Union[SampleQuantile, DatasetQuantile]]] = None + upper_quantile: InitVar[Optional[Union[SampleQuantile, DatasetQuantile]]] = None + lower: Union[SampleQuantile, DatasetQuantile] = field(init=False) + upper: Union[SampleQuantile, DatasetQuantile] = field(init=False) eps: float = 1e-6 def __post_init__( self, - lower_percentile: Optional[Union[SampleQuantile, DatasetPercentile]], - upper_percentile: Optional[Union[SampleQuantile, DatasetPercentile]], + lower_quantile: Optional[Union[SampleQuantile, DatasetQuantile]], + upper_quantile: Optional[Union[SampleQuantile, DatasetQuantile]], ): - if lower_percentile is None: - tid = self.input if upper_percentile is None else upper_percentile.member_id - self.lower = DatasetPercentile(q=0.0, member_id=tid) + if lower_quantile is None: + tid = self.input if upper_quantile is None else upper_quantile.member_id + self.lower = DatasetQuantile(q=0.0, member_id=tid) else: - self.lower = lower_percentile + self.lower = lower_quantile - if upper_percentile is None: - self.upper = DatasetPercentile(q=1.0, member_id=self.lower.member_id) + if upper_quantile is None: + self.upper = DatasetQuantile(q=1.0, member_id=self.lower.member_id) else: - self.upper = upper_percentile + self.upper = upper_quantile assert self.lower.member_id == self.upper.member_id assert self.lower.q < self.upper.q @@ -470,18 +563,22 @@ def from_proc_descr( ) dataset_mode, axes = _get_axes(descr.kwargs) if dataset_mode: - Percentile = DatasetPercentile + Quantile = DatasetQuantile else: - Percentile = SampleQuantile + Quantile = partial(SampleQuantile, method="linear") return cls( input=member_id, output=member_id, - lower_percentile=Percentile( - q=kwargs.min_percentile / 100, axes=axes, member_id=ref_tensor + lower_quantile=Quantile( + q=kwargs.min_percentile / 100, + axes=axes, + member_id=ref_tensor, ), - upper_percentile=Percentile( - q=kwargs.max_percentile / 100, axes=axes, member_id=ref_tensor + upper_quantile=Quantile( + q=kwargs.max_percentile / 100, + axes=axes, + member_id=ref_tensor, ), ) diff --git a/src/bioimageio/core/stat_calculators.py b/src/bioimageio/core/stat_calculators.py index c9ae2d838..c85c2f175 100644 --- a/src/bioimageio/core/stat_calculators.py +++ b/src/bioimageio/core/stat_calculators.py @@ -22,19 +22,20 @@ import numpy as np import xarray as xr -from bioimageio.spec.model.v0_5 import BATCH_AXIS_ID from loguru import logger from numpy.typing import NDArray from typing_extensions import assert_never +from bioimageio.spec.model.v0_5 import BATCH_AXIS_ID + from .axis import AxisId, PerAxis -from .common import MemberId +from .common import MemberId, QuantileMethod from .sample import Sample from .stat_measures import ( DatasetMean, DatasetMeasure, DatasetMeasureBase, - DatasetPercentile, + DatasetQuantile, DatasetStd, DatasetVar, Measure, @@ -208,33 +209,39 @@ def finalize( } -class SamplePercentilesCalculator: - """to calculate sample percentiles""" +class SampleQuantilesCalculator: + """to calculate sample quantiles""" def __init__( self, member_id: MemberId, axes: Optional[Sequence[AxisId]], qs: Collection[float], + method: QuantileMethod = "linear", ): super().__init__() assert all(0.0 <= q <= 1.0 for q in qs) self._qs = sorted(set(qs)) self._axes = None if axes is None else tuple(axes) self._member_id = member_id + self._method: QuantileMethod = method def compute(self, sample: Sample) -> Dict[SampleQuantile, MeasureValue]: tensor = sample.members[self._member_id] - ps = tensor.quantile(self._qs, dim=self._axes) + ps = tensor.quantile(self._qs, dim=self._axes, method=self._method) return { - SampleQuantile(q=q, axes=self._axes, member_id=self._member_id): p + SampleQuantile( + q=q, axes=self._axes, member_id=self._member_id, method=self._method + ): p for q, p in zip(self._qs, ps) } -class MeanPercentilesCalculator: - """to calculate dataset percentiles heuristically by averaging across samples - **note**: the returned dataset percentiles are an estiamte and **not mathematically correct** +class MeanQuantilesCalculator: + """to calculate dataset quantiles heuristically by averaging across samples + + Note: + The returned dataset quantiles are an estiamte and **not mathematically correct** """ def __init__( @@ -253,9 +260,9 @@ def __init__( def update(self, sample: Sample): tensor = sample.members[self._member_id] - sample_estimates = tensor.quantile(self._qs, dim=self._axes).astype( - "float64", copy=False - ) + sample_estimates = tensor.quantile( + self._qs, dim=self._axes, method="linear" + ).astype("float64", copy=False) # reduced voxel count n = int(tensor.size / np.prod(sample_estimates.shape_tuple[1:])) @@ -271,7 +278,7 @@ def update(self, sample: Sample): self._n += n - def finalize(self) -> Dict[DatasetPercentile, MeasureValue]: + def finalize(self) -> Dict[DatasetQuantile, MeasureValue]: if self._estimates is None: return {} else: @@ -279,13 +286,13 @@ def finalize(self) -> Dict[DatasetPercentile, MeasureValue]: "Computed dataset percentiles naively by averaging percentiles of samples." ) return { - DatasetPercentile(q=q, axes=self._axes, member_id=self._member_id): e + DatasetQuantile(q=q, axes=self._axes, member_id=self._member_id): e for q, e in zip(self._qs, self._estimates) } -class CrickPercentilesCalculator: - """to calculate dataset percentiles with the experimental [crick libray](https://github.com/dask/crick)""" +class CrickQuantilesCalculator: + """to calculate dataset quantiles with the experimental [crick libray](https://github.com/dask/crick)""" def __init__( self, @@ -293,12 +300,10 @@ def __init__( axes: Optional[Sequence[AxisId]], qs: Collection[float], ): - warnings.warn( - "Computing dataset percentiles with experimental 'crick' library." - ) + warnings.warn("Computing dataset quantiles with experimental 'crick' library.") super().__init__() assert all(0.0 <= q <= 1.0 for q in qs) - assert axes is None or "_percentiles" not in axes + assert axes is None or "_quantiles" not in axes self._qs = sorted(set(qs)) self._axes = None if axes is None else tuple(axes) self._member_id = member_id @@ -310,7 +315,7 @@ def __init__( def _initialize(self, tensor_sizes: PerAxis[int]): assert crick is not None out_sizes: OrderedDict[AxisId, int] = collections.OrderedDict( - _percentiles=len(self._qs) + _quantiles=len(self._qs) ) if self._axes is not None: for d, s in tensor_sizes.items(): @@ -329,7 +334,7 @@ def update(self, part: Sample): if isinstance(part, Sample) else part.members[self._member_id].data ) - assert "_percentiles" not in tensor.dims + assert "_quantiles" not in tensor.dims if self._digest is None: self._initialize(tensor.tagged_shape) @@ -339,7 +344,7 @@ def update(self, part: Sample): for i, idx in enumerate(self._indices): self._digest[i].update(tensor[dict(zip(self._dims[1:], idx))]) - def finalize(self) -> Dict[DatasetPercentile, MeasureValue]: + def finalize(self) -> Dict[DatasetQuantile, MeasureValue]: if self._digest is None: return {} else: @@ -350,7 +355,7 @@ def finalize(self) -> Dict[DatasetPercentile, MeasureValue]: [[d.quantile(q) for d in self._digest] for q in self._qs] ).reshape(self._shape) return { - DatasetPercentile( + DatasetQuantile( q=q, axes=self._axes, member_id=self._member_id ): Tensor(v, dims=self._dims[1:]) for q, v in zip(self._qs, vs) @@ -358,11 +363,11 @@ def finalize(self) -> Dict[DatasetPercentile, MeasureValue]: if crick is None: - DatasetPercentilesCalculator: Type[ - Union[MeanPercentilesCalculator, CrickPercentilesCalculator] - ] = MeanPercentilesCalculator + DatasetQuantilesCalculator: Type[ + Union[MeanQuantilesCalculator, CrickQuantilesCalculator] + ] = MeanQuantilesCalculator else: - DatasetPercentilesCalculator = CrickPercentilesCalculator + DatasetQuantilesCalculator = CrickQuantilesCalculator class NaiveSampleMeasureCalculator: @@ -380,11 +385,11 @@ def compute(self, sample: Sample) -> Dict[SampleMeasure, MeasureValue]: SampleMeasureCalculator = Union[ MeanCalculator, MeanVarStdCalculator, - SamplePercentilesCalculator, + SampleQuantilesCalculator, NaiveSampleMeasureCalculator, ] DatasetMeasureCalculator = Union[ - MeanCalculator, MeanVarStdCalculator, DatasetPercentilesCalculator + MeanCalculator, MeanVarStdCalculator, DatasetQuantilesCalculator ] @@ -493,10 +498,10 @@ def get_measure_calculators( required_dataset_mean_var_std: Set[Union[DatasetMean, DatasetVar, DatasetStd]] = ( set() ) - required_sample_percentiles: Dict[ - Tuple[MemberId, Optional[Tuple[AxisId, ...]]], Set[float] + required_sample_quantiles: Dict[ + Tuple[MemberId, Optional[Tuple[AxisId, ...]], QuantileMethod], Set[float] ] = {} - required_dataset_percentiles: Dict[ + required_dataset_quantiles: Dict[ Tuple[MemberId, Optional[Tuple[AxisId, ...]]], Set[float] ] = {} @@ -522,11 +527,11 @@ def get_measure_calculators( ) assert rm in required_dataset_mean_var_std elif isinstance(rm, SampleQuantile): - required_sample_percentiles.setdefault((rm.member_id, rm.axes), set()).add( - rm.q - ) - elif isinstance(rm, DatasetPercentile): - required_dataset_percentiles.setdefault((rm.member_id, rm.axes), set()).add( + required_sample_quantiles.setdefault( + (rm.member_id, rm.axes, rm.method), set() + ).add(rm.q) + elif isinstance(rm, DatasetQuantile): + required_dataset_quantiles.setdefault((rm.member_id, rm.axes), set()).add( rm.q ) else: @@ -556,14 +561,14 @@ def get_measure_calculators( MeanVarStdCalculator(member_id=rm.member_id, axes=rm.axes) ) - for (tid, axes), qs in required_sample_percentiles.items(): + for (tid, axes, m), qs in required_sample_quantiles.items(): sample_calculators.append( - SamplePercentilesCalculator(member_id=tid, axes=axes, qs=qs) + SampleQuantilesCalculator(member_id=tid, axes=axes, qs=qs, method=m) ) - for (tid, axes), qs in required_dataset_percentiles.items(): + for (tid, axes), qs in required_dataset_quantiles.items(): dataset_calculators.append( - DatasetPercentilesCalculator(member_id=tid, axes=axes, qs=qs) + DatasetQuantilesCalculator(member_id=tid, axes=axes, qs=qs) ) return sample_calculators, dataset_calculators diff --git a/src/bioimageio/core/stat_measures.py b/src/bioimageio/core/stat_measures.py index 609207897..1a96cd6bf 100644 --- a/src/bioimageio/core/stat_measures.py +++ b/src/bioimageio/core/stat_measures.py @@ -23,7 +23,7 @@ from typing_extensions import Annotated from .axis import AxisId -from .common import MemberId, PerMember +from .common import MemberId, PerMember, QuantileMethod from .tensor import Tensor @@ -157,19 +157,23 @@ def model_post_init(self, __context: Any): class SampleQuantile(_Quantile, SampleMeasureBase, frozen=True): - """The `n`th percentile of a single tensor""" + """The `q`th quantile of a single tensor""" + + method: QuantileMethod = "linear" + """Method to use when the desired quantile lies between two data points. + See https://numpy.org/devdocs/reference/generated/numpy.quantile.html#numpy-quantile for details.""" def compute(self, sample: SampleLike) -> MeasureValue: tensor = sample.members[self.member_id] - return tensor.quantile(self.q, dim=self.axes) + return tensor.quantile(self.q, dim=self.axes, method=self.method) def model_post_init(self, __context: Any): super().model_post_init(__context) assert self.axes is None or AxisId("batch") not in self.axes -class DatasetPercentile(_Quantile, DatasetMeasureBase, frozen=True): - """The `n`th percentile across multiple samples""" +class DatasetQuantile(_Quantile, DatasetMeasureBase, frozen=True): + """The `q`th quantile across multiple samples""" def model_post_init(self, __context: Any): super().model_post_init(__context) @@ -180,7 +184,7 @@ def model_post_init(self, __context: Any): Union[SampleMean, SampleStd, SampleVar, SampleQuantile], Discriminator("name") ] DatasetMeasure = Annotated[ - Union[DatasetMean, DatasetStd, DatasetVar, DatasetPercentile], Discriminator("name") + Union[DatasetMean, DatasetStd, DatasetVar, DatasetQuantile], Discriminator("name") ] Measure = Annotated[Union[SampleMeasure, DatasetMeasure], Discriminator("scope")] Stat = Dict[Measure, MeasureValue] @@ -188,7 +192,7 @@ def model_post_init(self, __context: Any): MeanMeasure = Union[SampleMean, DatasetMean] StdMeasure = Union[SampleStd, DatasetStd] VarMeasure = Union[SampleVar, DatasetVar] -PercentileMeasure = Union[SampleQuantile, DatasetPercentile] +PercentileMeasure = Union[SampleQuantile, DatasetQuantile] MeanMeasureT = TypeVar("MeanMeasureT", bound=MeanMeasure) StdMeasureT = TypeVar("StdMeasureT", bound=StdMeasure) VarMeasureT = TypeVar("VarMeasureT", bound=VarMeasure) diff --git a/src/bioimageio/core/tensor.py b/src/bioimageio/core/tensor.py index 17358b002..248a29884 100644 --- a/src/bioimageio/core/tensor.py +++ b/src/bioimageio/core/tensor.py @@ -19,11 +19,12 @@ import numpy as np import xarray as xr -from bioimageio.spec.model import v0_5 from loguru import logger from numpy.typing import DTypeLike, NDArray from typing_extensions import Self, assert_never +from bioimageio.spec.model import v0_5 + from ._magic_tensor_ops import MagicTensorOpsMixin from .axis import AxisId, AxisInfo, AxisLike, PerAxis from .common import ( @@ -33,6 +34,7 @@ PadWhere, PadWidth, PadWidthLike, + QuantileMethod, SliceInfo, ) @@ -177,11 +179,11 @@ def from_numpy( Args: array: the nd numpy array - axes: A description of the array's axes, + dims: A description of the array's axes, if None axes are guessed (which might fail and raise a ValueError.) Raises: - ValueError: if `axes` is None and axes guessing fails. + ValueError: if `dims` is None and dims guessing fails. """ if dims is None: @@ -388,6 +390,7 @@ def quantile( self, q: Union[float, Sequence[float]], dim: Optional[Union[AxisId, Sequence[AxisId]]] = None, + method: QuantileMethod = "linear", ) -> Self: assert ( isinstance(q, (float, int)) @@ -404,7 +407,9 @@ def quantile( assert dim is None or ( (quantile_dim := AxisId("quantile")) != dim and quantile_dim not in set(dim) ) - return self.__class__.from_xarray(self._data.quantile(q, dim=dim)) + return self.__class__.from_xarray( + self._data.quantile(q, dim=dim, method=method) + ) def resize_to( self, diff --git a/src/bioimageio/core/weight_converters/_add_weights.py b/src/bioimageio/core/weight_converters/_add_weights.py index cc9156192..255aa7b2d 100644 --- a/src/bioimageio/core/weight_converters/_add_weights.py +++ b/src/bioimageio/core/weight_converters/_add_weights.py @@ -30,8 +30,8 @@ def add_weights( Default: choose automatically from any available. target_format: convert to a specific weights format. Default: attempt to convert to any missing format. - devices: Devices that may be used during conversion. verbose: log more (error) output + allow_tracing: allow conversion to torchscript by tracing if scripting fails. Returns: A (potentially invalid) model copy stored at `output_path` with added weights if any conversion was possible. diff --git a/tests/test_add_weights.py b/tests/test_add_weights.py index 836353c74..31932f4d5 100644 --- a/tests/test_add_weights.py +++ b/tests/test_add_weights.py @@ -1,48 +1,43 @@ -# TODO: update add weights tests -# import os - - -# def _test_add_weights(model, tmp_path, base_weights, added_weights, **kwargs): -# from bioimageio.core.build_spec import add_weights - -# rdf = load_raw_resource_description(model) -# assert base_weights in rdf.weights -# assert added_weights in rdf.weights - -# weight_path = load_description(model).weights[added_weights].source -# assert weight_path.exists() - -# drop_weights = set(rdf.weights.keys()) - {base_weights} -# for drop in drop_weights: -# rdf.weights.pop(drop) -# assert tuple(rdf.weights.keys()) == (base_weights,) - -# in_path = tmp_path / "model1.zip" -# export_resource_package(rdf, output_path=in_path) - -# out_path = tmp_path / "model2.zip" -# add_weights(in_path, weight_path, weight_type=added_weights, output_path=out_path, **kwargs) - -# assert out_path.exists() -# new_rdf = load_description(out_path) -# assert set(new_rdf.weights.keys()) == {base_weights, added_weights} -# for weight in new_rdf.weights.values(): -# assert weight.source.exists() - -# test_res = _test_model(out_path, added_weights) -# failed = [s for s in test_res if s["status"] != "passed"] -# assert not failed, failed -# test_res = _test_model(out_path) -# failed = [s for s in test_res if s["status"] != "passed"] -# assert not failed, failed - -# # make sure the weights were cleaned from the cwd -# assert not os.path.exists(os.path.split(weight_path)[1]) - - -# def test_add_torchscript(unet2d_nuclei_broad_model, tmp_path): -# _test_add_weights(unet2d_nuclei_broad_model, tmp_path, "pytorch_state_dict", "torchscript") - - -# def test_add_onnx(unet2d_nuclei_broad_model, tmp_path): -# _test_add_weights(unet2d_nuclei_broad_model, tmp_path, "pytorch_state_dict", "onnx", opset_version=12) +from pathlib import Path + +import pytest + +from bioimageio.core import add_weights, load_model_description +from bioimageio.spec import InvalidDescr +from bioimageio.spec.model.v0_5 import WeightsFormat + + +@pytest.mark.parametrize( + ("source_format", "target_format"), + [ + ("pytorch_state_dict", "torchscript"), + ("pytorch_state_dict", "onnx"), + ("torchscript", "onnx"), + ], +) +def test_add_weights( + source_format: WeightsFormat, + target_format: WeightsFormat, + unet2d_nuclei_broad_model: str, + tmp_path: Path, + request: pytest.FixtureRequest, +): + model = load_model_description(unet2d_nuclei_broad_model, format_version="latest") + assert source_format in model.weights.available_formats, ( + "source format not found in model" + ) + if target_format in model.weights.available_formats: + setattr(model.weights, target_format, None) + + out_path = tmp_path / "converted.zip" + converted = add_weights( + model, + output_path=out_path, + source_format=source_format, + target_format=target_format, + ) + assert not isinstance(converted, InvalidDescr), ( + "conversion resulted in invalid descr", + converted.validation_summary.display(), + ) + assert target_format in converted.weights.available_formats diff --git a/tests/test_cli.py b/tests/test_cli.py index 203677ecf..a034ad9c7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,9 +2,12 @@ from pathlib import Path from typing import Any, List, Sequence +import numpy as np import pytest from pydantic import FilePath +from bioimageio.spec import load_description, settings + def run_subprocess( commands: Sequence[str], **kwargs: Any @@ -21,6 +24,8 @@ def run_subprocess( @pytest.mark.parametrize( "args", [ + ["--help"], + ["add-weights", "unet2d_nuclei_broad_model", "tmp_path"], [ "package", "unet2d_nuclei_broad_model", @@ -29,6 +34,7 @@ def run_subprocess( "pytorch_state_dict", ], ["package", "unet2d_nuclei_broad_model", "output.zip"], + ["predict", "--example", "unet2d_nuclei_broad_model"], [ "test", "unet2d_nuclei_broad_model", @@ -36,11 +42,9 @@ def run_subprocess( "pytorch_state_dict", ], ["test", "unet2d_nuclei_broad_model"], - ["predict", "--example", "unet2d_nuclei_broad_model"], ["update-format", "unet2d_nuclei_broad_model_old"], - ["add-weights", "unet2d_nuclei_broad_model", "tmp_path"], - ["update-hashes", "unet2d_nuclei_broad_model_old"], ["update-hashes", "unet2d_nuclei_broad_model_old", "--output=stdout"], + ["update-hashes", "unet2d_nuclei_broad_model_old"], ], ) def test_cli( @@ -67,6 +71,27 @@ def test_cli( assert ret.returncode == 0, ret.stdout +def test_empty_cache(tmp_path: Path, unet2d_nuclei_broad_model: str): + from bioimageio.spec.utils import empty_cache + + origingal_cache_path = settings.cache_path + try: + settings.cache_path = tmp_path / "cache" + assert not settings.cache_path.exists() + _ = load_description(unet2d_nuclei_broad_model, perform_io_checks=False) + assert ( + len([fn for fn in settings.cache_path.iterdir() if fn.suffix != ".lock"]) + == 1 + ) + empty_cache() + assert ( + len([fn for fn in settings.cache_path.iterdir() if fn.suffix != ".lock"]) + == 0 + ) + finally: + settings.cache_path = origingal_cache_path + + @pytest.mark.parametrize("args", [["test", "stardist_wrong_shape"]]) def test_cli_fails(args: List[str], stardist_wrong_shape: FilePath): resolved_args = [ @@ -77,83 +102,90 @@ def test_cli_fails(args: List[str], stardist_wrong_shape: FilePath): assert ret.returncode == 1, ret.stdout -# TODO: update CLI test -# def _test_cli_predict_image(model: Path, tmp_path: Path, extra_cmd_args: Optional[List[str]] = None): -# spec = load_description(model) -# in_path = spec.test_inputs[0] - -# out_path = tmp_path.with_suffix(".npy") -# cmd = ["bioimageio", "predict-image", model, "--input", str(in_path), "--output", str(out_path)] -# if extra_cmd_args is not None: -# cmd.extend(extra_cmd_args) -# ret = run_subprocess(cmd) -# assert ret.returncode == 0, ret.stdout -# assert out_path.exists() - - -# def test_cli_predict_image(unet2d_nuclei_broad_model: Path, tmp_path: Path): -# _test_cli_predict_image(unet2d_nuclei_broad_model, tmp_path) - - -# def test_cli_predict_image_with_weight_format(unet2d_nuclei_broad_model: Path, tmp_path: Path): -# _test_cli_predict_image(unet2d_nuclei_broad_model, tmp_path, ["--weight-format", "pytorch_state_dict"]) - - -# def _test_cli_predict_images(model: Path, tmp_path: Path, extra_cmd_args: Optional[List[str]] = None): -# n_images = 3 -# shape = (1, 1, 128, 128) -# expected_shape = (1, 1, 128, 128) - -# in_folder = tmp_path / "inputs" -# in_folder.mkdir() -# out_folder = tmp_path / "outputs" -# out_folder.mkdir() - -# expected_outputs: List[Path] = [] -# for i in range(n_images): -# path = in_folder / f"im-{i}.npy" -# im = np.random.randint(0, 255, size=shape).astype("uint8") -# np.save(path, im) -# expected_outputs.append(out_folder / f"im-{i}.npy") - -# input_pattern = str(in_folder / "*.npy") -# cmd = ["bioimageio", "predict-images", str(model), input_pattern, str(out_folder)] -# if extra_cmd_args is not None: -# cmd.extend(extra_cmd_args) -# ret = run_subprocess(cmd) -# assert ret.returncode == 0, ret.stdout +def _test_cli_predict_single( + model_source: str, tmp_path: Path, extra_cmd_args: Sequence[str] = () +): + from bioimageio.spec import load_model_description + + model = load_model_description(model_source, format_version="latest") + assert model.inputs[0].test_tensor is not None + in_path = tmp_path / "in.npy" + _ = in_path.write_bytes(model.inputs[0].test_tensor.get_reader().read()) + out_path = tmp_path / "out.npy" + cmd = [ + "bioimageio", + "predict", + str(model_source), + "--inputs", + str(in_path), + "--outputs", + str(out_path), + ] + list(extra_cmd_args) + ret = run_subprocess(cmd) + assert ret.returncode == 0, ret.stdout + assert out_path.exists() -# for out_path in expected_outputs: -# assert out_path.exists() -# assert np.load(out_path).shape == expected_shape +def test_cli_predict_single(unet2d_nuclei_broad_model: Path, tmp_path: Path): + _test_cli_predict_single(str(unet2d_nuclei_broad_model), tmp_path) -# def test_cli_predict_images(unet2d_nuclei_broad_model: Path, tmp_path: Path): -# _test_cli_predict_images(unet2d_nuclei_broad_model, tmp_path) +def test_cli_predict_single_with_weight_format( + unet2d_nuclei_broad_model: Path, tmp_path: Path +): + _test_cli_predict_single( + str(unet2d_nuclei_broad_model), + tmp_path, + ["--weight-format", "pytorch_state_dict"], + ) -# def test_cli_predict_images_with_weight_format(unet2d_nuclei_broad_model: Path, tmp_path: Path): -# _test_cli_predict_images(unet2d_nuclei_broad_model, tmp_path, ["--weight-format", "pytorch_state_dict"]) +def _test_cli_predict_multiple( + model_source: str, tmp_path: Path, extra_cmd_args: Sequence[str] = () +): + n_images = 3 + shape = (1, 1, 128, 128) + expected_shape = (1, 1, 128, 128) + + in_folder = tmp_path / "inputs" + in_folder.mkdir() + out_folder = tmp_path / "outputs" + out_folder.mkdir() + out_file_pattern = "im-{sample_id}.npy" + inputs: List[str] = [] + expected_outputs: List[Path] = [] + for i in range(n_images): + input_path = in_folder / f"im-{i}.npy" + im = np.random.randint(0, 255, size=shape).astype("uint8") + np.save(input_path, im) + inputs.extend(["--inputs", str(input_path)]) + expected_outputs.append(out_folder / out_file_pattern.format(sample_id=i)) + + cmd = [ + "bioimageio", + "predict", + model_source, + *inputs, + "--outputs", + str(out_folder / out_file_pattern), + ] + list(extra_cmd_args) + ret = run_subprocess(cmd) + assert ret.returncode == 0, ret.stdout -# def test_torch_to_torchscript(unet2d_nuclei_broad_model: Path, tmp_path: Path): -# out_path = tmp_path.with_suffix(".pt") -# ret = run_subprocess( -# ["bioimageio", "convert-torch-weights-to-torchscript", str(unet2d_nuclei_broad_model), str(out_path)] -# ) -# assert ret.returncode == 0, ret.stdout -# assert out_path.exists() + for out_path in expected_outputs: + assert out_path.exists() + assert np.load(out_path).shape == expected_shape -# def test_torch_to_onnx(convert_to_onnx: Path, tmp_path: Path): -# out_path = tmp_path.with_suffix(".onnx") -# ret = run_subprocess(["bioimageio", "convert-torch-weights-to-onnx", str(convert_to_onnx), str(out_path)]) -# assert ret.returncode == 0, ret.stdout -# assert out_path.exists() +def test_cli_predict_multiple(unet2d_nuclei_broad_model: Path, tmp_path: Path): + _test_cli_predict_multiple(str(unet2d_nuclei_broad_model), tmp_path) -# def test_keras_to_tf(unet2d_keras: Path, tmp_path: Path): -# out_path = tmp_path / "weights.zip" -# ret = run_subprocess(["bioimageio", "convert-keras-weights-to-tensorflow", str(unet2d_keras), str(out_path)]) -# assert ret.returncode == 0, ret.stdout -# assert out_path.exists() +def test_cli_predict_multiple_with_weight_format( + unet2d_nuclei_broad_model: Path, tmp_path: Path +): + _test_cli_predict_multiple( + str(unet2d_nuclei_broad_model), + tmp_path, + ["--weight-format", "pytorch_state_dict"], + ) diff --git a/tests/test_io.py b/tests/test_io.py index a45dfe51a..3ff166936 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -10,8 +10,6 @@ [ "img.png", "img.tiff", - "img.h5", - "img.h5/img", "img.npy", ], ) @@ -25,7 +23,7 @@ (5, 3, 4), ], ) -def test_image_io(name: str, shape: Tuple[int, ...], tmp_path: Path): +def test_tensor_io(name: str, shape: Tuple[int, ...], tmp_path: Path): from bioimageio.core import Tensor from bioimageio.core.io import load_tensor, save_tensor diff --git a/tests/test_prediction.py b/tests/test_prediction.py index 04779158d..61ef8642a 100644 --- a/tests/test_prediction.py +++ b/tests/test_prediction.py @@ -117,7 +117,7 @@ def test_predict_with_fixed_blocking(prep: Prep): def test_predict_save_output(prep: Prep, tmp_path: Path): - save_path = tmp_path / "{member_id}_{sample_id}.h5" + save_path = tmp_path / "{member_id}_{sample_id}.tiff" out = predict( model=prep.prediction_pipeline, inputs=prep.input_sample, diff --git a/tests/test_proc_ops.py b/tests/test_proc_ops.py index 7ab2eaa00..0e800ad51 100644 --- a/tests/test_proc_ops.py +++ b/tests/test_proc_ops.py @@ -189,6 +189,51 @@ def test_clip(tid: MemberId): xr.testing.assert_equal(expected, sample.members[tid].data) +def test_clip_percentiles(): + from bioimageio.core.proc_ops import Clip + from bioimageio.core.stat_measures import SampleQuantile + from bioimageio.spec.model.v0_5 import AxisId, ClipDescr, ClipKwargs + + descr = ClipDescr( + kwargs=ClipKwargs(min_percentile=30, max_percentile=70, axes=(AxisId("x"),)) + ) + op = Clip.from_proc_descr( + descr, + member_id=MemberId("data"), + ) + assert op.required_measures == { + SampleQuantile( + member_id=MemberId("data"), + scope="sample", + name="quantile", + q=0.3, + axes=(AxisId("x"),), + method="inverted_cdf", + ), + SampleQuantile( + member_id=MemberId("data"), + scope="sample", + name="quantile", + q=0.7, + axes=(AxisId("x"),), + method="inverted_cdf", + ), + } + + data = xr.DataArray(np.arange(15).reshape(3, 5), dims=("channel", "x")) + sample = Sample( + members={MemberId("data"): Tensor.from_xarray(data)}, stat={}, id=None + ) + sample.stat = compute_measures(op.required_measures, [sample]) + + expected = xr.DataArray( + np.array([[1, 1, 2, 3, 3], [6, 6, 7, 8, 8], [11, 11, 12, 13, 13]]), + dims=("channel", "x"), + ) + op(sample) + xr.testing.assert_equal(expected, sample.members[MemberId("data")].data) + + def test_combination_of_op_steps_with_dims_specified(tid: MemberId): from bioimageio.core.proc_ops import ZeroMeanUnitVariance @@ -292,7 +337,7 @@ def test_scale_mean_variance_per_channel(tid: MemberId, axes_str: Optional[str]) sample.stat = compute_measures(op.required_measures, [sample]) op(sample) - if axes is not None and AxisId("c") not in axes: + if axes is not None and AxisId("channel") not in axes: # mean,std per channel should match exactly xr.testing.assert_allclose( ref_data, sample.members[tid].data, rtol=1e-5, atol=1e-7 @@ -330,10 +375,10 @@ def test_scale_range_axes(tid: MemberId): eps = 1.0e-6 lower_quantile = SampleQuantile( - member_id=tid, q=0.1, axes=(AxisId("x"), AxisId("y")) + member_id=tid, q=0.1, axes=(AxisId("x"), AxisId("y")), method="linear" ) upper_quantile = SampleQuantile( - member_id=tid, q=0.9, axes=(AxisId("x"), AxisId("y")) + member_id=tid, q=0.9, axes=(AxisId("x"), AxisId("y")), method="linear" ) op = ScaleRange(tid, tid, lower_quantile, upper_quantile, eps=eps) diff --git a/tests/test_resource_tests.py b/tests/test_resource_tests.py index f4eca96bc..5b09176b7 100644 --- a/tests/test_resource_tests.py +++ b/tests/test_resource_tests.py @@ -1,3 +1,7 @@ +from pathlib import Path + +import numpy as np + from bioimageio.spec import InvalidDescr, ValidationContext @@ -42,3 +46,31 @@ def test_loading_description_multiple_times(unet2d_nuclei_broad_model: str): # load again, which some users might end up doing model_descr = load_description(model_descr) # pyright: ignore[reportArgumentType] assert not isinstance(model_descr, InvalidDescr) + + +def test_test_description_runtime_env(unet2d_nuclei_broad_model: str): + from bioimageio.core._resource_tests import test_description + + summary = test_description(unet2d_nuclei_broad_model, runtime_env="as-described") + + assert summary.status == "passed", summary.display() + + +def test_failed_reproducibility(unet2d_nuclei_broad_model: str, tmp_path: str): + from bioimageio.core import load_model + from bioimageio.core._resource_tests import test_model + from bioimageio.spec.common import FileDescr + from bioimageio.spec.utils import load_array, save_array + + model = load_model(unet2d_nuclei_broad_model, format_version="latest") + + # use corrupted test input to fail the reproducibility test + test_array_path = Path(tmp_path) / "input.npy" + assert model.inputs[0].test_tensor is not None + test_array = load_array(model.inputs[0].test_tensor) + save_array(test_array_path, np.zeros_like(test_array)) + model.inputs[0].test_tensor = FileDescr(source=test_array_path) + + summary = test_model(model) + + assert summary.status == "valid-format" diff --git a/tests/test_stat_measures.py b/tests/test_stat_measures.py index 49c876098..e4cd9d926 100644 --- a/tests/test_stat_measures.py +++ b/tests/test_stat_measures.py @@ -1,5 +1,5 @@ from itertools import product -from typing import Optional, Tuple +from typing import Literal, Optional, Tuple import numpy as np import pytest @@ -10,7 +10,7 @@ from bioimageio.core.common import MemberId from bioimageio.core.sample import Sample from bioimageio.core.stat_calculators import ( - SamplePercentilesCalculator, + SampleQuantilesCalculator, get_measure_calculators, ) from bioimageio.core.stat_measures import SampleQuantile @@ -42,23 +42,28 @@ def test_individual_normal_measure( xr.testing.assert_allclose(expected.data, actual.data) +@pytest.mark.parametrize("method", ["inverted_cdf", "linear"]) @pytest.mark.parametrize("axes", [None, (AxisId("x"), AxisId("y"))]) -def test_individual_percentile_measure(axes: Optional[Tuple[AxisId, ...]]): +def test_individual_percentile_measure( + axes: Optional[Tuple[AxisId, ...]], method: Literal["inverted_cdf", "linear"] +): qs = [0, 0.1, 0.5, 1.0] tid = MemberId("tensor") - measures = [SampleQuantile(member_id=tid, axes=axes, q=q) for q in qs] + measures = [ + SampleQuantile(member_id=tid, axes=axes, q=q, method=method) for q in qs + ] calcs, _ = get_measure_calculators(measures) assert len(calcs) == 1 calc = calcs[0] - assert isinstance(calc, SamplePercentilesCalculator) + assert isinstance(calc, SampleQuantilesCalculator) data = Tensor( np.random.random((5, 6, 3)), dims=(AxisId("x"), AxisId("y"), AxisId("c")) ) actual = calc.compute(Sample(members={tid: data}, stat={}, id=None)) for m in measures: - expected = data.quantile(q=m.q, dim=m.axes) + expected = data.quantile(q=m.q, dim=m.axes, method=m.method) actual_data = actual[m] if isinstance(actual_data, Tensor): actual_data = actual_data.data