From 84534f429a087660c01d4b5c824d940c1ed84b9c Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Tue, 14 Nov 2023 14:41:46 +0100 Subject: [PATCH 001/403] Setup from copier_template --- packages/essnmx/.copier-answers.yml | 13 + packages/essnmx/.github/dependabot.yml | 13 + packages/essnmx/.github/workflows/ci.yml | 57 ++++ packages/essnmx/.github/workflows/docs.yml | 69 +++++ .../.github/workflows/nightly_at_main.yml | 36 +++ .../.github/workflows/nightly_at_release.yml | 43 +++ .../.github/workflows/python-version-ci | 1 + packages/essnmx/.github/workflows/release.yml | 116 ++++++++ packages/essnmx/.github/workflows/test.yml | 64 +++++ .../essnmx/.github/workflows/unpinned.yml | 43 +++ packages/essnmx/.gitignore | 18 ++ packages/essnmx/.pre-commit-config.yaml | 58 ++++ packages/essnmx/CODE_OF_CONDUCT.md | 134 ++++++++++ packages/essnmx/CONTRIBUTING.md | 20 ++ packages/essnmx/LICENSE | 29 ++ packages/essnmx/MANIFEST.in | 1 + packages/essnmx/README.md | 16 ++ packages/essnmx/conda/meta.yaml | 44 ++++ .../essnmx/docs/_static/anaconda-logo.svg | 103 ++++++++ packages/essnmx/docs/_static/css/custom.css | 21 ++ .../essnmx/docs/_templates/class-template.rst | 31 +++ .../essnmx/docs/_templates/doc_version.html | 2 + .../docs/_templates/module-template.rst | 66 +++++ packages/essnmx/docs/about/index.md | 26 ++ packages/essnmx/docs/api-reference/index.md | 29 ++ packages/essnmx/docs/conf.py | 210 +++++++++++++++ .../docs/developer/coding-conventions.md | 117 +++++++++ .../docs/developer/dependency-management.md | 13 + .../essnmx/docs/developer/getting-started.md | 91 +++++++ packages/essnmx/docs/developer/index.md | 16 ++ packages/essnmx/docs/index.md | 16 ++ packages/essnmx/pyproject.toml | 80 ++++++ packages/essnmx/requirements/base.in | 7 + packages/essnmx/requirements/base.txt | 88 +++++++ packages/essnmx/requirements/ci.in | 4 + packages/essnmx/requirements/ci.txt | 56 ++++ packages/essnmx/requirements/dev.in | 10 + packages/essnmx/requirements/dev.txt | 101 +++++++ packages/essnmx/requirements/docs.in | 12 + packages/essnmx/requirements/docs.txt | 248 ++++++++++++++++++ packages/essnmx/requirements/make_base.py | 52 ++++ packages/essnmx/requirements/mypy.in | 2 + packages/essnmx/requirements/mypy.txt | 14 + packages/essnmx/requirements/nightly.in | 7 + packages/essnmx/requirements/nightly.txt | 84 ++++++ packages/essnmx/requirements/static.in | 1 + packages/essnmx/requirements/static.txt | 28 ++ packages/essnmx/requirements/test.in | 2 + packages/essnmx/requirements/test.txt | 18 ++ packages/essnmx/requirements/wheels.in | 1 + packages/essnmx/requirements/wheels.txt | 21 ++ packages/essnmx/setup.cfg | 4 + packages/essnmx/src/ess/nmx/__init__.py | 12 + packages/essnmx/src/ess/nmx/py.typed | 0 packages/essnmx/tests/package_test.py | 7 + packages/essnmx/tox.ini | 69 +++++ 56 files changed, 2444 insertions(+) create mode 100644 packages/essnmx/.copier-answers.yml create mode 100644 packages/essnmx/.github/dependabot.yml create mode 100644 packages/essnmx/.github/workflows/ci.yml create mode 100644 packages/essnmx/.github/workflows/docs.yml create mode 100644 packages/essnmx/.github/workflows/nightly_at_main.yml create mode 100644 packages/essnmx/.github/workflows/nightly_at_release.yml create mode 100644 packages/essnmx/.github/workflows/python-version-ci create mode 100644 packages/essnmx/.github/workflows/release.yml create mode 100644 packages/essnmx/.github/workflows/test.yml create mode 100644 packages/essnmx/.github/workflows/unpinned.yml create mode 100644 packages/essnmx/.gitignore create mode 100644 packages/essnmx/.pre-commit-config.yaml create mode 100644 packages/essnmx/CODE_OF_CONDUCT.md create mode 100644 packages/essnmx/CONTRIBUTING.md create mode 100644 packages/essnmx/LICENSE create mode 100644 packages/essnmx/MANIFEST.in create mode 100644 packages/essnmx/README.md create mode 100644 packages/essnmx/conda/meta.yaml create mode 100644 packages/essnmx/docs/_static/anaconda-logo.svg create mode 100644 packages/essnmx/docs/_static/css/custom.css create mode 100644 packages/essnmx/docs/_templates/class-template.rst create mode 100644 packages/essnmx/docs/_templates/doc_version.html create mode 100644 packages/essnmx/docs/_templates/module-template.rst create mode 100644 packages/essnmx/docs/about/index.md create mode 100644 packages/essnmx/docs/api-reference/index.md create mode 100644 packages/essnmx/docs/conf.py create mode 100644 packages/essnmx/docs/developer/coding-conventions.md create mode 100644 packages/essnmx/docs/developer/dependency-management.md create mode 100644 packages/essnmx/docs/developer/getting-started.md create mode 100644 packages/essnmx/docs/developer/index.md create mode 100644 packages/essnmx/docs/index.md create mode 100644 packages/essnmx/pyproject.toml create mode 100644 packages/essnmx/requirements/base.in create mode 100644 packages/essnmx/requirements/base.txt create mode 100644 packages/essnmx/requirements/ci.in create mode 100644 packages/essnmx/requirements/ci.txt create mode 100644 packages/essnmx/requirements/dev.in create mode 100644 packages/essnmx/requirements/dev.txt create mode 100644 packages/essnmx/requirements/docs.in create mode 100644 packages/essnmx/requirements/docs.txt create mode 100644 packages/essnmx/requirements/make_base.py create mode 100644 packages/essnmx/requirements/mypy.in create mode 100644 packages/essnmx/requirements/mypy.txt create mode 100644 packages/essnmx/requirements/nightly.in create mode 100644 packages/essnmx/requirements/nightly.txt create mode 100644 packages/essnmx/requirements/static.in create mode 100644 packages/essnmx/requirements/static.txt create mode 100644 packages/essnmx/requirements/test.in create mode 100644 packages/essnmx/requirements/test.txt create mode 100644 packages/essnmx/requirements/wheels.in create mode 100644 packages/essnmx/requirements/wheels.txt create mode 100644 packages/essnmx/setup.cfg create mode 100644 packages/essnmx/src/ess/nmx/__init__.py create mode 100644 packages/essnmx/src/ess/nmx/py.typed create mode 100644 packages/essnmx/tests/package_test.py create mode 100644 packages/essnmx/tox.ini diff --git a/packages/essnmx/.copier-answers.yml b/packages/essnmx/.copier-answers.yml new file mode 100644 index 00000000..04f10eed --- /dev/null +++ b/packages/essnmx/.copier-answers.yml @@ -0,0 +1,13 @@ +# Changes here will be overwritten by Copier; NEVER EDIT MANUALLY +_commit: 3ad29de +_src_path: gh:scipp/copier_template +description: Data reduction for NMX at the European Spallation Source +max_python: '3.11' +min_python: '3.8' +namespace_package: ess +nightly_deps: scipp,sciline,scippnexus,plopp +orgname: scipp +prettyname: Essnmx +projectname: essnmx +related_projects: Scipp,Sciline,Plopp,ScippNexus +year: 2023 diff --git a/packages/essnmx/.github/dependabot.yml b/packages/essnmx/.github/dependabot.yml new file mode 100644 index 00000000..c8076bb1 --- /dev/null +++ b/packages/essnmx/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + # Note: We are not listing package-ecosystem: "github-actions". This causes + # noise in all template instances. Instead dependabot.yml in scipp/copier_template + # triggers updates of github-actions in the *template*. We then use `copier update` + # in template instances. + - package-ecosystem: "pip" + directory: "/requirements" + schedule: + interval: "daily" + allow: + - dependency-name: "scipp" + dependency-type: "direct" diff --git a/packages/essnmx/.github/workflows/ci.yml b/packages/essnmx/.github/workflows/ci.yml new file mode 100644 index 00000000..0d294d17 --- /dev/null +++ b/packages/essnmx/.github/workflows/ci.yml @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) + +name: CI + +on: + push: + branches: + - main + - release + pull_request: + +jobs: + formatting: + name: Formatting and static analysis + runs-on: 'ubuntu-20.04' + outputs: + min_python: ${{ steps.vars.outputs.min_python }} + min_tox_env: ${{ steps.vars.outputs.min_tox_env }} + steps: + - uses: actions/checkout@v4 + - name: Get Python version for other CI jobs + id: vars + run: | + echo "min_python=$(cat .github/workflows/python-version-ci)" >> $GITHUB_OUTPUT + echo "min_tox_env=py$(cat .github/workflows/python-version-ci | sed 's/\.//g')" >> $GITHUB_OUTPUT + - uses: actions/setup-python@v4 + with: + python-version-file: '.github/workflows/python-version-ci' + - run: python -m pip install --upgrade pip + - run: python -m pip install -r requirements/ci.txt + - run: tox -e static + - uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: Apply automatic formatting + + tests: + name: Tests + needs: formatting + strategy: + matrix: + os: ['ubuntu-20.04'] + python: + - version: '${{needs.formatting.outputs.min_python}}' + tox-env: '${{needs.formatting.outputs.min_tox_env}}' + uses: ./.github/workflows/test.yml + with: + os-variant: ${{ matrix.os }} + python-version: ${{ matrix.python.version }} + tox-env: ${{ matrix.python.tox-env }} + + docs: + needs: tests + uses: ./.github/workflows/docs.yml + with: + publish: false + branch: ${{ github.head_ref == '' && github.ref_name || github.head_ref }} diff --git a/packages/essnmx/.github/workflows/docs.yml b/packages/essnmx/.github/workflows/docs.yml new file mode 100644 index 00000000..9dd22f19 --- /dev/null +++ b/packages/essnmx/.github/workflows/docs.yml @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) + +name: Docs + +on: + workflow_dispatch: + inputs: + publish: + default: false + type: boolean + version: + default: '' + required: false + type: string + branch: + description: 'Branch/tag with documentation source. If not set, the current branch will be used.' + default: '' + required: false + type: string + workflow_call: + inputs: + publish: + default: false + type: boolean + version: + default: '' + required: false + type: string + branch: + description: 'Branch/tag with documentation source. If not set, the current branch will be used.' + default: '' + required: false + type: string + +env: + VERSION: ${{ inputs.version }} + +jobs: + docs: + name: Build documentation + runs-on: 'ubuntu-20.04' + steps: + - run: sudo apt install --yes graphviz pandoc + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch == '' && github.ref_name || inputs.branch }} + fetch-depth: 0 # history required so cmake can determine version + - uses: actions/setup-python@v4 + with: + python-version-file: '.github/workflows/python-version-ci' + - run: python -m pip install --upgrade pip + - run: python -m pip install -r requirements/ci.txt + - run: tox -e releasedocs -- ${VERSION} + if: ${{ inputs.version != '' }} + - run: tox -e docs + if: ${{ inputs.version == '' }} + - uses: actions/upload-artifact@v3 + with: + name: docs_html + path: html/ + + - uses: JamesIves/github-pages-deploy-action@v4.4.3 + if: ${{ inputs.publish }} + with: + branch: gh-pages + folder: html + single-commit: true + ssh-key: ${{ secrets.GH_PAGES_DEPLOY_KEY }} diff --git a/packages/essnmx/.github/workflows/nightly_at_main.yml b/packages/essnmx/.github/workflows/nightly_at_main.yml new file mode 100644 index 00000000..f83c0c36 --- /dev/null +++ b/packages/essnmx/.github/workflows/nightly_at_main.yml @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) + +name: Nightly test at main branch + +on: + workflow_dispatch: + schedule: + - cron: '30 1 * * 1-5' + +jobs: + setup: + name: Setup variables + runs-on: 'ubuntu-20.04' + outputs: + min_python: ${{ steps.vars.outputs.min_python }} + steps: + - uses: actions/checkout@v4 + - name: Get Python version for other CI jobs + id: vars + run: echo "min_python=$(cat .github/workflows/python-version-ci)" >> $GITHUB_OUTPUT + + tests: + name: Tests + needs: setup + strategy: + matrix: + os: ['ubuntu-20.04'] + python: + - version: '${{needs.setup.outputs.min_python}}' + tox-env: 'nightly' + uses: ./.github/workflows/test.yml + with: + os-variant: ${{ matrix.os }} + python-version: ${{ matrix.python.version }} + tox-env: ${{ matrix.python.tox-env }} diff --git a/packages/essnmx/.github/workflows/nightly_at_release.yml b/packages/essnmx/.github/workflows/nightly_at_release.yml new file mode 100644 index 00000000..f9d811a0 --- /dev/null +++ b/packages/essnmx/.github/workflows/nightly_at_release.yml @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) + +name: Nightly tests at latest release + +on: + workflow_dispatch: + schedule: + - cron: '0 1 * * 1-5' + +jobs: + setup: + name: Setup variables + runs-on: 'ubuntu-20.04' + outputs: + min_python: ${{ steps.vars.outputs.min_python }} + release_tag: ${{ steps.release.outputs.release_tag }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # history required so we can determine latest release tag + - name: Get last release tag from git + id: release + run: echo "release_tag=$(git describe --tags --abbrev=0 --match '[0-9]*.[0-9]*.[0-9]*')" >> $GITHUB_OUTPUT + - name: Get Python version for other CI jobs + id: vars + run: echo "min_python=$(cat .github/workflows/python-version-ci)" >> $GITHUB_OUTPUT + + tests: + name: Tests + needs: setup + strategy: + matrix: + os: ['ubuntu-20.04'] + python: + - version: '${{needs.setup.outputs.min_python}}' + tox-env: 'nightly' + uses: ./.github/workflows/test.yml + with: + os-variant: ${{ matrix.os }} + python-version: ${{ matrix.python.version }} + tox-env: ${{ matrix.python.tox-env }} + checkout_ref: ${{ needs.setup.outputs.release_tag }} diff --git a/packages/essnmx/.github/workflows/python-version-ci b/packages/essnmx/.github/workflows/python-version-ci new file mode 100644 index 00000000..cc1923a4 --- /dev/null +++ b/packages/essnmx/.github/workflows/python-version-ci @@ -0,0 +1 @@ +3.8 diff --git a/packages/essnmx/.github/workflows/release.yml b/packages/essnmx/.github/workflows/release.yml new file mode 100644 index 00000000..223dd5af --- /dev/null +++ b/packages/essnmx/.github/workflows/release.yml @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) + +name: Release + +on: + release: + types: [published] + workflow_dispatch: + +defaults: + run: + shell: bash -l {0} # required for conda env + +jobs: + build_conda: + name: Conda build + runs-on: 'ubuntu-20.04' + + steps: + - uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 # history required so setuptools_scm can determine version + + - uses: mamba-org/setup-micromamba@v1 + with: + environment-name: build-env + create-args: >- + conda-build + boa + - run: conda mambabuild --channel conda-forge --channel scipp --no-anaconda-upload --override-channels --output-folder conda/package conda + + - uses: actions/upload-artifact@v3 + with: + name: conda-package-noarch + path: conda/package/noarch/*.tar.bz2 + + build_wheels: + name: Wheels + runs-on: 'ubuntu-20.04' + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 # history required so setuptools_scm can determine version + + - uses: actions/setup-python@v4 + with: + python-version-file: '.github/workflows/python-version-ci' + + - run: python -m pip install --upgrade pip + - run: python -m pip install -r requirements/wheels.txt + + - name: Build wheels + run: python -m build + + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: dist + path: dist + + upload_pypi: + name: Deploy PyPI + needs: [build_wheels, build_conda] + runs-on: 'ubuntu-20.04' + environment: release + permissions: + id-token: write + if: github.event_name == 'release' && github.event.action == 'published' + steps: + - uses: actions/download-artifact@v3 + - uses: pypa/gh-action-pypi-publish@v1.8.10 + + upload_conda: + name: Deploy Conda + needs: [build_wheels, build_conda] + runs-on: 'ubuntu-20.04' + if: github.event_name == 'release' && github.event.action == 'published' + + steps: + - uses: actions/download-artifact@v3 + - uses: mamba-org/setup-micromamba@v1 + with: + environment-name: upload-env + # frozen python due to breaking removal of 'imp' in 3.12 + create-args: >- + anaconda-client + python=3.11 + - run: anaconda --token ${{ secrets.ANACONDATOKEN }} upload --user scipp --label main $(ls conda-package-noarch/*.tar.bz2) + + docs: + needs: [upload_conda, upload_pypi] + uses: ./.github/workflows/docs.yml + with: + publish: ${{ github.event_name == 'release' && github.event.action == 'published' }} + secrets: inherit + + assets: + name: Upload docs + needs: docs + runs-on: 'ubuntu-20.04' + permissions: + contents: write # This is needed so that the action can upload the asset + steps: + - uses: actions/download-artifact@v3 + - name: Zip documentation + run: | + mv docs_html documentation-${{ github.ref_name }} + zip -r documentation-${{ github.ref_name }}.zip documentation-${{ github.ref_name }} + - name: Upload release assets + uses: svenstaro/upload-release-action@v2 + with: + file: ./documentation-${{ github.ref_name }}.zip + overwrite: false diff --git a/packages/essnmx/.github/workflows/test.yml b/packages/essnmx/.github/workflows/test.yml new file mode 100644 index 00000000..8ce71be0 --- /dev/null +++ b/packages/essnmx/.github/workflows/test.yml @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) + +name: Test + +on: + workflow_dispatch: + inputs: + os-variant: + default: 'ubuntu-20.04' + type: string + python-version: + type: string + tox-env: + default: 'test' + type: string + pip-recipe: + default: 'requirements/ci.txt' + type: string + coverage-report: + default: false + type: boolean + checkout_ref: + default: '' + type: string + workflow_call: + inputs: + os-variant: + default: 'ubuntu-20.04' + type: string + python-version: + type: string + tox-env: + default: 'test' + type: string + pip-recipe: + default: 'requirements/ci.txt' + type: string + coverage-report: + default: false + type: boolean + checkout_ref: + default: '' + type: string + +jobs: + test: + runs-on: ${{ inputs.os-variant }} + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.checkout_ref }} + - uses: actions/setup-python@v3 + with: + python-version: ${{ inputs.python-version }} + - run: python -m pip install --upgrade pip + - run: python -m pip install -r ${{ inputs.pip-recipe }} + - run: tox -e ${{ inputs.tox-env }} + - uses: actions/upload-artifact@v3 + if: ${{ inputs.coverage-report }} + with: + name: CoverageReport + path: coverage_html/ diff --git a/packages/essnmx/.github/workflows/unpinned.yml b/packages/essnmx/.github/workflows/unpinned.yml new file mode 100644 index 00000000..dbb546f4 --- /dev/null +++ b/packages/essnmx/.github/workflows/unpinned.yml @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) + +name: Unpinned tests at latest release + +on: + workflow_dispatch: + schedule: + - cron: '0 2 * * 1' + +jobs: + setup: + name: Setup variables + runs-on: 'ubuntu-20.04' + outputs: + min_python: ${{ steps.vars.outputs.min_python }} + release_tag: ${{ steps.release.outputs.release_tag }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # history required so we can determine latest release tag + - name: Get last release tag from git + id: release + run: echo "release_tag=$(git describe --tags --abbrev=0 --match '[0-9]*.[0-9]*.[0-9]*')" >> $GITHUB_OUTPUT + - name: Get Python version for other CI jobs + id: vars + run: echo "min_python=$(cat .github/workflows/python-version-ci)" >> $GITHUB_OUTPUT + + tests: + name: Tests + needs: setup + strategy: + matrix: + os: ['ubuntu-20.04'] + python: + - version: '${{needs.setup.outputs.min_python}}' + tox-env: 'unpinned' + uses: ./.github/workflows/test.yml + with: + os-variant: ${{ matrix.os }} + python-version: ${{ matrix.python.version }} + tox-env: ${{ matrix.python.tox-env }} + checkout_ref: ${{ needs.setup.outputs.release_tag }} diff --git a/packages/essnmx/.gitignore b/packages/essnmx/.gitignore new file mode 100644 index 00000000..9529e0f3 --- /dev/null +++ b/packages/essnmx/.gitignore @@ -0,0 +1,18 @@ +dist +html +.tox +src/essnmx.egg-info + +*.sw? + +.clangd/ +.idea/ +.vscode/ +*.ipynb_checkpoints +__pycache__/ +.vs/ +.virtual_documents +.hypothesis +.pytest_cache +.mypy_cache +docs/generated/ diff --git a/packages/essnmx/.pre-commit-config.yaml b/packages/essnmx/.pre-commit-config.yaml new file mode 100644 index 00000000..6a86994b --- /dev/null +++ b/packages/essnmx/.pre-commit-config.yaml @@ -0,0 +1,58 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-added-large-files + - id: check-json + exclude: asv.conf.json + - id: check-toml + - id: check-yaml + exclude: conda/meta.yaml + - id: detect-private-key + - id: trailing-whitespace + args: [ --markdown-linebreak-ext=md ] + exclude: '\.svg' + - repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + name: isort (python) + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 23.11.0 + hooks: + - id: black + - repo: https://github.com/kynan/nbstripout + rev: 0.6.0 + hooks: + - id: nbstripout + types: [ "jupyter" ] + args: [ "--drop-empty-cells", + "--extra-keys 'metadata.language_info.version cell.metadata.jp-MarkdownHeadingCollapsed cell.metadata.pycharm'" ] + - repo: https://github.com/pycqa/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + types: ["python"] + additional_dependencies: ["flake8-bugbear==23.9.16"] + - repo: https://github.com/pycqa/bandit + rev: 1.7.5 + hooks: + - id: bandit + additional_dependencies: ["bandit[toml]"] + args: ["-c", "pyproject.toml"] + - repo: https://github.com/codespell-project/codespell + rev: v2.2.6 + hooks: + - id: codespell + additional_dependencies: + - tomli + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: python-no-eval + - id: python-no-log-warn + - id: python-use-type-annotations + - id: rst-backticks + - id: rst-directive-colons + - id: rst-inline-touching-normal + - id: text-unicode-replacement-char diff --git a/packages/essnmx/CODE_OF_CONDUCT.md b/packages/essnmx/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..1c3746e4 --- /dev/null +++ b/packages/essnmx/CODE_OF_CONDUCT.md @@ -0,0 +1,134 @@ + +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +scipp[at]ess.eu. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations + diff --git a/packages/essnmx/CONTRIBUTING.md b/packages/essnmx/CONTRIBUTING.md new file mode 100644 index 00000000..bcd9e616 --- /dev/null +++ b/packages/essnmx/CONTRIBUTING.md @@ -0,0 +1,20 @@ +## Contributing to Essnmx + +Welcome to the developer side of Essnmx! + +Contributions are always welcome. +This includes reporting bugs or other issues, submitting pull requests, requesting new features, etc. + +If you need help with using Essnmx or contributing to it, have a look at the GitHub [discussions](https://github.com/scipp/essnmx/discussions) and start a new [Q&A discussion](https://github.com/scipp/essnmx/discussions/categories/q-a) if you can't find what you are looking for. + +For bug reports and other problems, please open an [issue](https://github.com/scipp/essnmx/issues/new) in GitHub. + +You are welcome to submit pull requests at any time. +But to avoid having to make large modifications during review or even have your PR rejected, please first open an issue first to discuss your idea! + +Check out the subsections of the [Developer documentation](https://scipp.github.io/essnmx/developer/index.html) for details on how Essnmx is developed. + +## Code of conduct + +This project is a community effort, and everyone is welcome to contribute. +Everyone within the community is expected to abide by our [code of conduct](https://github.com/scipp/essnmx/blob/main/CODE_OF_CONDUCT.md). diff --git a/packages/essnmx/LICENSE b/packages/essnmx/LICENSE new file mode 100644 index 00000000..b402aa64 --- /dev/null +++ b/packages/essnmx/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2023, Scipp contributors (https://github.com/scipp) +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/packages/essnmx/MANIFEST.in b/packages/essnmx/MANIFEST.in new file mode 100644 index 00000000..1aba38f6 --- /dev/null +++ b/packages/essnmx/MANIFEST.in @@ -0,0 +1 @@ +include LICENSE diff --git a/packages/essnmx/README.md b/packages/essnmx/README.md new file mode 100644 index 00000000..f33d2aef --- /dev/null +++ b/packages/essnmx/README.md @@ -0,0 +1,16 @@ +[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](CODE_OF_CONDUCT.md) +[![PyPI badge](http://img.shields.io/pypi/v/essnmx.svg)](https://pypi.python.org/pypi/essnmx) +[![Anaconda-Server Badge](https://anaconda.org/scipp/essnmx/badges/version.svg)](https://anaconda.org/scipp/essnmx) +[![License: BSD 3-Clause](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](LICENSE) + +# essnmx + +## About + +Data reduction for NMX at the European Spallation Source + +## Installation + +```sh +python -m pip install essnmx +``` diff --git a/packages/essnmx/conda/meta.yaml b/packages/essnmx/conda/meta.yaml new file mode 100644 index 00000000..0e0aecaf --- /dev/null +++ b/packages/essnmx/conda/meta.yaml @@ -0,0 +1,44 @@ +package: + name: essnmx + + version: {{ GIT_DESCRIBE_TAG }} + +source: + path: .. + +requirements: + build: + - setuptools + - setuptools_scm + run: + - dask + - python>=3.8 + - python-graphviz + - plopp + - sciline>=23.9.1 + - scipp>=23.8.0 + - scippnexus>=23.9.0 + +test: + imports: + - essnmx + requires: + - pytest + source_files: + - pyproject.toml + - tests/ + commands: + - python -m pytest tests + +build: + noarch: python + script: + - pip install . + +about: + home: https://github.com/scipp/essnmx + license: BSD-3-Clause + summary: Data reduction for NMX at the European Spallation Source + description: Data reduction for NMX at the European Spallation Source + dev_url: https://github.com/scipp/essnmx + doc_url: https://scipp.github.io/essnmx diff --git a/packages/essnmx/docs/_static/anaconda-logo.svg b/packages/essnmx/docs/_static/anaconda-logo.svg new file mode 100644 index 00000000..67bd68d9 --- /dev/null +++ b/packages/essnmx/docs/_static/anaconda-logo.svg @@ -0,0 +1,103 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/essnmx/docs/_static/css/custom.css b/packages/essnmx/docs/_static/css/custom.css new file mode 100644 index 00000000..d7fabbbb --- /dev/null +++ b/packages/essnmx/docs/_static/css/custom.css @@ -0,0 +1,21 @@ +html[data-theme="light"] { + /* match pst-color-text-muted (used in header buttons) */ + --scipp-filter-header-icon: saturate(0) brightness(0.696); +} + +html[data-theme="dark"] { + /* match pst-color-text-muted (used in header buttons) */ + --scipp-filter-header-icon: saturate(0) brightness(1.161); +} + +/* This selects custom icon links in the header but not the builtin ones. + * Currently, this is only the anaconda logo and the filters are adjusted to it. + */ +.bd-header .navbar-nav li a.nav-link .icon-link-image { + filter: var(--scipp-filter-header-icon); +} + +.bd-header .navbar-nav li a.nav-link .icon-link-image:hover { + /* match primary color */ + filter: hue-rotate(85deg) saturate(0.829) brightness(0.945); +} diff --git a/packages/essnmx/docs/_templates/class-template.rst b/packages/essnmx/docs/_templates/class-template.rst new file mode 100644 index 00000000..0200267d --- /dev/null +++ b/packages/essnmx/docs/_templates/class-template.rst @@ -0,0 +1,31 @@ +{{ fullname | escape | underline }} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }} + :members: + :special-members: __getitem__ + + {% block methods %} + .. automethod:: __init__ + + {% if methods %} + .. rubric:: {{ _('Methods') }} + + .. autosummary:: + {% for item in methods %} + ~{{ name }}.{{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Attributes') }} + + .. autosummary:: + {% for item in attributes %} + ~{{ name }}.{{ item }} + {%- endfor %} + {% endif %} + {% endblock %} diff --git a/packages/essnmx/docs/_templates/doc_version.html b/packages/essnmx/docs/_templates/doc_version.html new file mode 100644 index 00000000..7fd881ac --- /dev/null +++ b/packages/essnmx/docs/_templates/doc_version.html @@ -0,0 +1,2 @@ + +Current {{ project }} version: {{ version }} (older versions). diff --git a/packages/essnmx/docs/_templates/module-template.rst b/packages/essnmx/docs/_templates/module-template.rst new file mode 100644 index 00000000..6fee8d77 --- /dev/null +++ b/packages/essnmx/docs/_templates/module-template.rst @@ -0,0 +1,66 @@ +{{ fullname | escape | underline}} + +.. automodule:: {{ fullname }} + + {% block attributes %} + {% if attributes %} + .. rubric:: {{ _('Module Attributes') }} + + .. autosummary:: + :toctree: + {% for item in attributes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block functions %} + {% if functions %} + .. rubric:: {{ _('Functions') }} + + .. autosummary:: + :toctree: + {% for item in functions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block classes %} + {% if classes %} + .. rubric:: {{ _('Classes') }} + + .. autosummary:: + :toctree: + :template: class-template.rst + {% for item in classes %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block exceptions %} + {% if exceptions %} + .. rubric:: {{ _('Exceptions') }} + + .. autosummary:: + :toctree: + {% for item in exceptions %} + {{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + +{% block modules %} +{% if modules %} +.. rubric:: Modules + +.. autosummary:: + :toctree: + :template: module-template.rst + :recursive: +{% for item in modules %} + {{ item }} +{%- endfor %} +{% endif %} +{% endblock %} diff --git a/packages/essnmx/docs/about/index.md b/packages/essnmx/docs/about/index.md new file mode 100644 index 00000000..98182ed2 --- /dev/null +++ b/packages/essnmx/docs/about/index.md @@ -0,0 +1,26 @@ +# About Essnmx + +## Development + +Essnmx is an open source project by the [European Spallation Source ERIC](https://europeanspallationsource.se/) (ESS). + +## License + +Essnmx is available as open source under the [BSD-3 license](https://opensource.org/licenses/BSD-3-Clause). + +## Citing Essnmx + +Please cite the following: + +[![DOI](https://zenodo.org/badge/FIXME.svg)](https://zenodo.org/doi/10.5281/zenodo.FIXME) + +To cite a specific version of Essnmx, select the desired version on Zenodo to get the corresponding DOI. + +## Older versions of the documentation + +Older versions of the documentation pages can be found under the assets of each [release](https://github.com/scipp/essnmx/releases). +Simply download the archive, unzip and view locally in a web browser. + +## Source code and development + +Essnmx is hosted and developed [on GitHub](https://github.com/scipp/essnmx). diff --git a/packages/essnmx/docs/api-reference/index.md b/packages/essnmx/docs/api-reference/index.md new file mode 100644 index 00000000..2cb58f2e --- /dev/null +++ b/packages/essnmx/docs/api-reference/index.md @@ -0,0 +1,29 @@ +# API Reference + +## Classes + +```{eval-rst} +.. currentmodule:: essnmx + +.. autosummary:: + :toctree: ../generated/classes + :template: class-template.rst + :recursive: +``` + +## Top-level functions + +```{eval-rst} +.. autosummary:: + :toctree: ../generated/functions + :recursive: +``` + +## Submodules + +```{eval-rst} +.. autosummary:: + :toctree: ../generated/modules + :template: module-template.rst + :recursive: +``` \ No newline at end of file diff --git a/packages/essnmx/docs/conf.py b/packages/essnmx/docs/conf.py new file mode 100644 index 00000000..09af0b33 --- /dev/null +++ b/packages/essnmx/docs/conf.py @@ -0,0 +1,210 @@ +# -*- coding: utf-8 -*- + +import doctest +import os +import sys + +from ess import nmx + +sys.path.insert(0, os.path.abspath('.')) + +# General information about the project. +project = u'Essnmx' +copyright = u'2023 Scipp contributors' +author = u'Scipp contributors' + +html_show_sourcelink = True + +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.doctest', + 'sphinx.ext.githubpages', + 'sphinx.ext.intersphinx', + 'sphinx.ext.mathjax', + 'sphinx.ext.napoleon', + 'sphinx_autodoc_typehints', + 'sphinx_copybutton', + "sphinx_design", + 'nbsphinx', + 'myst_parser', +] + +myst_enable_extensions = [ + "amsmath", + "colon_fence", + "deflist", + "dollarmath", + "fieldlist", + "html_admonition", + "html_image", + "replacements", + "smartquotes", + "strikethrough", + "substitution", + "tasklist", +] + +myst_heading_anchors = 3 + +autodoc_type_aliases = { + 'array_like': 'array_like', +} + +intersphinx_mapping = { + 'python': ('https://docs.python.org/3', None), + 'numpy': ('https://numpy.org/doc/stable/', None), +} + +# autodocs includes everything, even irrelevant API internals. autosummary +# looks more suitable in the long run when the API grows. +# For a nice example see how xarray handles its API documentation. +autosummary_generate = True + +napoleon_google_docstring = False +napoleon_numpy_docstring = True +napoleon_use_param = True +napoleon_use_rtype = False +napoleon_preprocess_types = True +napoleon_type_aliases = { + # objects without namespace: numpy + "ndarray": "~numpy.ndarray", +} +typehints_defaults = 'comma' +typehints_use_rtype = False + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +source_suffix = ['.rst', '.md'] +html_sourcelink_suffix = '' # Avoid .ipynb.txt extensions in sources + +# The master toctree document. +master_doc = 'index' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# + +# The short X.Y version. +version = nmx.__version__ +# The full version, including alpha/beta/rc tags. +release = nmx.__version__ + +warning_is_error = True + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = "en" + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '**.ipynb_checkpoints'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + +# -- Options for HTML output ---------------------------------------------- + +html_theme = "pydata_sphinx_theme" +html_theme_options = { + "primary_sidebar_end": ["edit-this-page", "sourcelink"], + "secondary_sidebar_items": [], + "show_nav_level": 1, + # Adjust this to ensure external links are moved to "Move" menu + "header_links_before_dropdown": 4, + "pygment_light_style": "github-light-high-contrast", + "pygment_dark_style": "github-dark-high-contrast", + "logo": { + "image_light": "_static/logo.svg", + "image_dark": "_static/logo-dark.svg", + }, + "external_links": [ + {"name": "Plopp", "url": "https://scipp.github.io/plopp"}, + {"name": "Sciline", "url": "https://scipp.github.io/sciline"}, + {"name": "Scipp", "url": "https://scipp.github.io"}, + {"name": "ScippNexus", "url": "https://scipp.github.io/scippnexus"}, + ], + "icon_links": [ + { + "name": "GitHub", + "url": "https://github.com/scipp/essnmx", + "icon": "fa-brands fa-github", + "type": "fontawesome", + }, + { + "name": "PyPI", + "url": "https://pypi.org/project/essnmx/", + "icon": "fa-brands fa-python", + "type": "fontawesome", + }, + { + "name": "Conda", + "url": "https://anaconda.org/conda-forge/essnmx", + "icon": "_static/anaconda-logo.svg", + "type": "local", + }, + ], + "footer_start": ["copyright", "sphinx-version"], + "footer_end": ["doc_version", "theme-version"], +} +html_context = { + "doc_path": "docs", +} +html_sidebars = { + "**": ["sidebar-nav-bs", "page-toc"], +} + +html_title = "Essnmx" +html_logo = "_static/logo.svg" +html_favicon = "_static/favicon.ico" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] +html_css_files = ["css/custom.css"] + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'essnmxdoc' + +# -- Options for Matplotlib in notebooks ---------------------------------- + +nbsphinx_execute_arguments = [ + "--Session.metadata=scipp_docs_build=True", +] + +# -- Options for doctest -------------------------------------------------- + +doctest_global_setup = ''' +import numpy as np +''' + +# Using normalize whitespace because many __str__ functions in scipp produce +# extraneous empty lines and it would look strange to include them in the docs. +doctest_default_flags = ( + doctest.ELLIPSIS + | doctest.IGNORE_EXCEPTION_DETAIL + | doctest.DONT_ACCEPT_TRUE_FOR_1 + | doctest.NORMALIZE_WHITESPACE +) + +# -- Options for linkcheck ------------------------------------------------ + +linkcheck_ignore = [ + # Specific lines in Github blobs cannot be found by linkcheck. + r'https?://github\.com/.*?/blob/[a-f0-9]+/.+?#', +] diff --git a/packages/essnmx/docs/developer/coding-conventions.md b/packages/essnmx/docs/developer/coding-conventions.md new file mode 100644 index 00000000..b23c0eb4 --- /dev/null +++ b/packages/essnmx/docs/developer/coding-conventions.md @@ -0,0 +1,117 @@ +# Coding conventions + +## Code formatting + +There are no explicit code formatting conventions since we use `black` to enforce a format. + +## Docstring format + +We use the [NumPy docstring format](https://www.sphinx-doc.org/en/master/usage/extensions/example_numpy.html). +We use `sphinx-autodocs-typehints` to automatically insert type hints into the docstrings. +Our format thus deviates from the default NumPy example given by the link above. +Docstrings should therefore be laid out as follows, including spacing and punctuation: + +```python + +def foo(x: int, y: float) -> float: + """Short description. + + Long description. + + With multiple paragraphs. + + Warning + ------- + Be careful! + + Parameters + ---------- + x: + First input. + y: + Second input. + + Returns + ------- + : + The result. + + Raises + ------ + ValueError + If the input is bad. + IndexError + If some lookup failed. + + See Also + -------- + scitacean.bar: + A bit less foo. + + Examples + -------- + This is how to use it: + + >>> foo(1, 2) + 3 + + And also: + + >>> foo(1, 3) + 6 + """ +``` + +The order of sections is fixed as shown in the example. + +- **Short description** (*required*) A single sentence describing the purpose of the function / class. +- **Long description** (*optional*) One or more paragraphs of detailed explanations. + Can include additional sections like `Warning` or `Hint`. +- **Parameters** (*required for functions*) List of all function arguments including their name but not their type. + Listing arguments like this can seem ridiculous if the explanation is as devoid of content as in the example. + But it is still required in order for sphinx to show the types. +- **Returns** (*required for functions*) Description of the return value. + Required for the same reason as the parameter list. + + For a single return value, neither a name nor type should be given. + But a colon is required as in the example above in order to produce proper formatting. + + For multiple return values, to produce proper formatting, + both name and type must be given even though the latter repeats the type annotation: + + ```python + + """ + Returns + ------- + n: int + The first return value. + z: float + The second return value. + """ + ``` + +- **Raises** (*optional*) We generally do not document what exceptions can be raised from a function. + But if there are some important cases, this section can list those exceptions with an explanation + of when the exception is raised. + The exception type is required. + Note that there are no colons here. +- **See Also** (*optional*) List of related functions and/or classes. + The function/class name should include the module it is in but without reST markup. + For simple cases, the explanation can be left out. + In this case, the colon should be omitted as well and multiple entries must be separated by commas. +- **Examples** (*optional*) Example code given using `>>>` as the Python prompt. + May include text before, after, and between code blocks. + Note the spacing in the example. + +Some functions can be sufficiently described by a single sentence. +In this case, the 'Parameters' and 'Returns' sections may be omitted and the docstring should be laid out on a single line. +If it does not fit on a single line, it is too complicated. +For example + +```python +def bar(self) -> int: + """Returns the number of dimensions.""" +``` + +Note that the argument types are not shown in the rendered documentation. diff --git a/packages/essnmx/docs/developer/dependency-management.md b/packages/essnmx/docs/developer/dependency-management.md new file mode 100644 index 00000000..172722dc --- /dev/null +++ b/packages/essnmx/docs/developer/dependency-management.md @@ -0,0 +1,13 @@ +# Dependency management + +essnmx is a library, so the package dependencies are never pinned. +Lower bounds are fine and individual versions can be excluded. +See, e.g., [Should You Use Upper Bound Version Constraints](https://iscinumpy.dev/post/bound-version-constraints/) for an explanation. + +Development dependencies (as opposed to dependencies of the deployed package that users need to install) are pinned to an exact version in order to ensure reproducibility. +This also includes dependencies used for the various CI builds. +This is done by specifying packages (and potential version constraints) in `requirements/*.in` files and locking those dependencies using [pip-compile-multi](https://pip-compile-multi.readthedocs.io/en/latest/index.html) to produce `requirements/*.txt` files. +Those files are then used by [tox](https://tox.wiki/en/latest/) to create isolated environments and run tests, build docs, etc. + +`tox` can be cumbersome to use for local development. +Therefore `requirements/dev.txt` can be used to create a virtual environment with all dependencies. diff --git a/packages/essnmx/docs/developer/getting-started.md b/packages/essnmx/docs/developer/getting-started.md new file mode 100644 index 00000000..1f0f5950 --- /dev/null +++ b/packages/essnmx/docs/developer/getting-started.md @@ -0,0 +1,91 @@ +# Getting started + +## Setting up + +### Dependencies + +Development dependencies are specified in `requirements/dev.txt` and can be installed using (see [Dependency Management](./dependency-management.md) for more information) + +```sh +pip install -r requirements/dev.txt +``` + +Additionally, building the documentation requires [pandoc](https://pandoc.org/) which is not on PyPI and needs to be installed through other means, e.g. with your OS package manager. + +### Install the package + +Install the package in editable mode using + +```sh +pip install -e . +``` + +### Set up git hooks + +The CI pipeline runs a number of code formatting and static analysis tools. +If they fail, a build is rejected. +To avoid that, you can run the same tools locally. +This can be done conveniently using [pre-commit](https://pre-commit.com/): + +```sh +pre-commit install +``` + +Alternatively, if you want a different workflow, take a look at ``tox.ini`` or ``.pre-commit.yaml`` to see what tools are run and how. + +## Running tests + +`````{tab-set} +````{tab-item} tox +Run the tests using + +```sh +tox -e py38 +``` + +(or just `tox` if you want to run all environments). + +```` +````{tab-item} Manually +Run the tests using + +```sh +python -m pytest +``` +```` +````` + +## Building the docs + +`````{tab-set} +````{tab-item} tox +Build the documentation using + +```sh +tox -e docs +``` + +This builds the docs and also runs `doctest`. +`linkcheck` can be run separately using + +```sh +tox -e linkcheck +``` +```` + +````{tab-item} Manually + +Build the documentation using + +```sh +python -m sphinx -v -b html -d .tox/docs_doctrees docs html +``` + +Additionally, test the documentation using + +```sh +python -m sphinx -v -b doctest -d .tox/docs_doctrees docs html +python -m sphinx -v -b linkcheck -d .tox/docs_doctrees docs html +``` +```` +````` \ No newline at end of file diff --git a/packages/essnmx/docs/developer/index.md b/packages/essnmx/docs/developer/index.md new file mode 100644 index 00000000..23b55441 --- /dev/null +++ b/packages/essnmx/docs/developer/index.md @@ -0,0 +1,16 @@ +# Developer documentation + +```{include} ../../CONTRIBUTING.md +``` + +## Table of contents + +```{toctree} +--- +maxdepth: 2 +--- + +getting-started +coding-conventions +dependency-management +``` diff --git a/packages/essnmx/docs/index.md b/packages/essnmx/docs/index.md new file mode 100644 index 00000000..9d9bd075 --- /dev/null +++ b/packages/essnmx/docs/index.md @@ -0,0 +1,16 @@ +# Essnmx + + + Data reduction for NMX at the European Spallation Source +

+ + +```{toctree} +--- +hidden: +--- + +api-reference/index +developer/index +about/index +``` diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml new file mode 100644 index 00000000..f783626b --- /dev/null +++ b/packages/essnmx/pyproject.toml @@ -0,0 +1,80 @@ +[build-system] +requires = [ + "setuptools>=68", + "setuptools_scm[toml]>=8.0", +] +build-backend = "setuptools.build_meta" + +[project] +name = "essnmx" +description = "Data reduction for NMX at the European Spallation Source" +authors = [{ name = "Scipp contributors" }] +license = { file = "LICENSE" } +readme = "README.md" +classifiers = [ + "Intended Audience :: Science/Research", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Scientific/Engineering", + "Typing :: Typed", +] +requires-python = ">=3.8" + +# IMPORTANT: +# Run 'tox -e deps' after making changes here. This will update requirement files. +# Make sure to list one dependency per line. +dependencies = [ + "dask", + "graphviz", + "plopp", + "sciline>=23.9.1", + "scipp>=23.8.0", + "scippnexus>=23.9.0", +] + +dynamic = ["version"] + +[project.urls] +"Bug Tracker" = "https://github.com/scipp/essnmx/issues" +"Documentation" = "https://scipp.github.io/essnmx" +"Source" = "https://github.com/scipp/essnmx" + +[tool.setuptools_scm] + +[tool.pytest.ini_options] +minversion = "7.0" +addopts = "-ra -v" +testpaths = "tests" +filterwarnings = [ + "error", +] + +[tool.bandit] +# Excluding tests because bandit doesn't like `assert`. +exclude_dirs = ["docs/conf.py", "tests"] + +[tool.black] +skip-string-normalization = true + +[tool.isort] +skip_gitignore = true +profile = "black" +known_first_party = ["essnmx"] + +[tool.mypy] +strict = true +ignore_missing_imports = true +enable_error_code = [ + "ignore-without-code", + "redundant-expr", + "truthy-bool", +] +show_error_codes = true +warn_unreachable = true diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in new file mode 100644 index 00000000..84d908ef --- /dev/null +++ b/packages/essnmx/requirements/base.in @@ -0,0 +1,7 @@ +# Generated by 'tox -e deps', DO NOT EDIT MANUALLY!' +dask +graphviz +plopp +sciline>=23.9.1 +scipp>=23.8.0 +scippnexus>=23.9.0 diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt new file mode 100644 index 00000000..3ff98223 --- /dev/null +++ b/packages/essnmx/requirements/base.txt @@ -0,0 +1,88 @@ +# SHA1:d319e96fc4fdca56b4aa64fcfe16d5daa656be7a +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +click==8.1.7 + # via dask +cloudpickle==3.0.0 + # via dask +confuse==2.0.1 + # via scipp +contourpy==1.1.1 + # via matplotlib +cycler==0.12.1 + # via matplotlib +dask==2023.5.0 + # via -r base.in +fonttools==4.44.1 + # via matplotlib +fsspec==2023.10.0 + # via dask +graphlib-backport==1.0.3 + # via + # sciline + # scipp +graphviz==0.20.1 + # via -r base.in +h5py==3.10.0 + # via scippnexus +importlib-metadata==6.8.0 + # via dask +importlib-resources==6.1.1 + # via matplotlib +kiwisolver==1.4.5 + # via matplotlib +locket==1.0.0 + # via partd +matplotlib==3.7.3 + # via plopp +numpy==1.24.4 + # via + # contourpy + # h5py + # matplotlib + # scipp + # scipy +packaging==23.2 + # via + # dask + # matplotlib +partd==1.4.1 + # via dask +pillow==10.1.0 + # via matplotlib +plopp==23.10.1 + # via -r base.in +pyparsing==3.1.1 + # via matplotlib +python-dateutil==2.8.2 + # via + # matplotlib + # scippnexus +pyyaml==6.0.1 + # via + # confuse + # dask +sciline==23.9.1 + # via -r base.in +scipp==23.8.0 + # via + # -r base.in + # scippnexus +scippnexus==23.11.1 + # via -r base.in +scipy==1.10.1 + # via scippnexus +six==1.16.0 + # via python-dateutil +toolz==0.12.0 + # via + # dask + # partd +zipp==3.17.0 + # via + # importlib-metadata + # importlib-resources diff --git a/packages/essnmx/requirements/ci.in b/packages/essnmx/requirements/ci.in new file mode 100644 index 00000000..e5c3075a --- /dev/null +++ b/packages/essnmx/requirements/ci.in @@ -0,0 +1,4 @@ +gitpython +packaging +requests +tox diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt new file mode 100644 index 00000000..a1a0562f --- /dev/null +++ b/packages/essnmx/requirements/ci.txt @@ -0,0 +1,56 @@ +# SHA1:6344d52635ea11dca331a3bc6eb1833c4c64d585 +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +cachetools==5.3.2 + # via tox +certifi==2023.7.22 + # via requests +chardet==5.2.0 + # via tox +charset-normalizer==3.3.2 + # via requests +colorama==0.4.6 + # via tox +distlib==0.3.7 + # via virtualenv +filelock==3.13.1 + # via + # tox + # virtualenv +gitdb==4.0.11 + # via gitpython +gitpython==3.1.40 + # via -r ci.in +idna==3.4 + # via requests +packaging==23.2 + # via + # -r ci.in + # pyproject-api + # tox +platformdirs==3.11.0 + # via + # tox + # virtualenv +pluggy==1.3.0 + # via tox +pyproject-api==1.6.1 + # via tox +requests==2.31.0 + # via -r ci.in +smmap==5.0.1 + # via gitdb +tomli==2.0.1 + # via + # pyproject-api + # tox +tox==4.11.3 + # via -r ci.in +urllib3==2.1.0 + # via requests +virtualenv==20.24.6 + # via tox diff --git a/packages/essnmx/requirements/dev.in b/packages/essnmx/requirements/dev.in new file mode 100644 index 00000000..1c747e48 --- /dev/null +++ b/packages/essnmx/requirements/dev.in @@ -0,0 +1,10 @@ +-r base.in +-r ci.in +-r docs.in +-r mypy.in +-r static.in +-r test.in +-r wheels.in +jupyterlab +pip-compile-multi +pre-commit diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt new file mode 100644 index 00000000..5ad3d870 --- /dev/null +++ b/packages/essnmx/requirements/dev.txt @@ -0,0 +1,101 @@ +# SHA1:e3e8cd703eb07e0484bb54fe0e131426d458d1bc +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +-r base.txt +-r ci.txt +-r docs.txt +-r mypy.txt +-r static.txt +-r test.txt +-r wheels.txt +anyio==4.0.0 + # via jupyter-server +argon2-cffi==23.1.0 + # via jupyter-server +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +arrow==1.3.0 + # via isoduration +async-lru==2.0.4 + # via jupyterlab +cffi==1.16.0 + # via argon2-cffi-bindings +fqdn==1.5.1 + # via jsonschema +isoduration==20.11.0 + # via jsonschema +json5==0.9.14 + # via jupyterlab-server +jsonpointer==2.4 + # via jsonschema +jsonschema[format-nongpl]==4.19.2 + # via + # jupyter-events + # jupyterlab-server + # nbformat +jupyter-events==0.9.0 + # via jupyter-server +jupyter-lsp==2.2.0 + # via jupyterlab +jupyter-server==2.10.0 + # via + # jupyter-lsp + # jupyterlab + # jupyterlab-server + # notebook-shim +jupyter-server-terminals==0.4.4 + # via jupyter-server +jupyterlab==4.0.8 + # via -r dev.in +jupyterlab-server==2.25.1 + # via jupyterlab +notebook-shim==0.2.3 + # via jupyterlab +overrides==7.4.0 + # via jupyter-server +pip-compile-multi==2.6.3 + # via -r dev.in +pip-tools==7.3.0 + # via pip-compile-multi +prometheus-client==0.18.0 + # via jupyter-server +pycparser==2.21 + # via cffi +python-json-logger==2.0.7 + # via jupyter-events +rfc3339-validator==0.1.4 + # via + # jsonschema + # jupyter-events +rfc3986-validator==0.1.1 + # via + # jsonschema + # jupyter-events +send2trash==1.8.2 + # via jupyter-server +sniffio==1.3.0 + # via anyio +terminado==0.18.0 + # via + # jupyter-server + # jupyter-server-terminals +toposort==1.10 + # via pip-compile-multi +types-python-dateutil==2.8.19.14 + # via arrow +uri-template==1.3.0 + # via jsonschema +webcolors==1.13 + # via jsonschema +websocket-client==1.6.4 + # via jupyter-server +wheel==0.41.3 + # via pip-tools + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/packages/essnmx/requirements/docs.in b/packages/essnmx/requirements/docs.in new file mode 100644 index 00000000..3d138485 --- /dev/null +++ b/packages/essnmx/requirements/docs.in @@ -0,0 +1,12 @@ +-r base.in +autodoc_pydantic +ipykernel +ipython!=8.7.0 # Breaks syntax highlighting in Jupyter code cells. +myst-parser +nbsphinx +pydata-sphinx-theme>=0.13 +sphinx +sphinx-autodoc-typehints==1.23.0 # Higher versions require sphinx-7, which is not supported by sphinx-design yet +sphinx-copybutton +sphinx-design +platformdirs<4 # temporary until virtualenv has release with support for this diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt new file mode 100644 index 00000000..820d951a --- /dev/null +++ b/packages/essnmx/requirements/docs.txt @@ -0,0 +1,248 @@ +# SHA1:894ae0768d7baa8f604a6d257c02840e43ca2f74 +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +-r base.txt +accessible-pygments==0.0.4 + # via pydata-sphinx-theme +alabaster==0.7.13 + # via sphinx +annotated-types==0.6.0 + # via pydantic +asttokens==2.4.1 + # via stack-data +attrs==23.1.0 + # via + # jsonschema + # referencing +autodoc-pydantic==2.0.1 + # via -r docs.in +babel==2.13.1 + # via + # pydata-sphinx-theme + # sphinx +backcall==0.2.0 + # via ipython +beautifulsoup4==4.12.2 + # via + # nbconvert + # pydata-sphinx-theme +bleach==6.1.0 + # via nbconvert +certifi==2023.7.22 + # via requests +charset-normalizer==3.3.2 + # via requests +comm==0.2.0 + # via ipykernel +debugpy==1.8.0 + # via ipykernel +decorator==5.1.1 + # via ipython +defusedxml==0.7.1 + # via nbconvert +docutils==0.20.1 + # via + # myst-parser + # nbsphinx + # pydata-sphinx-theme + # sphinx +executing==2.0.1 + # via stack-data +fastjsonschema==2.18.1 + # via nbformat +idna==3.4 + # via requests +imagesize==1.4.1 + # via sphinx +ipykernel==6.26.0 + # via -r docs.in +ipython==8.12.3 + # via + # -r docs.in + # ipykernel +jedi==0.19.1 + # via ipython +jinja2==3.1.2 + # via + # myst-parser + # nbconvert + # nbsphinx + # sphinx +jsonschema==4.19.2 + # via nbformat +jsonschema-specifications==2023.7.1 + # via jsonschema +jupyter-client==8.6.0 + # via + # ipykernel + # nbclient +jupyter-core==5.5.0 + # via + # ipykernel + # jupyter-client + # nbclient + # nbconvert + # nbformat +jupyterlab-pygments==0.2.2 + # via nbconvert +markdown-it-py==3.0.0 + # via + # mdit-py-plugins + # myst-parser +markupsafe==2.1.3 + # via + # jinja2 + # nbconvert +matplotlib-inline==0.1.6 + # via + # ipykernel + # ipython +mdit-py-plugins==0.4.0 + # via myst-parser +mdurl==0.1.2 + # via markdown-it-py +mistune==3.0.2 + # via nbconvert +myst-parser==2.0.0 + # via -r docs.in +nbclient==0.9.0 + # via nbconvert +nbconvert==7.11.0 + # via nbsphinx +nbformat==5.9.2 + # via + # nbclient + # nbconvert + # nbsphinx +nbsphinx==0.9.3 + # via -r docs.in +nest-asyncio==1.5.8 + # via ipykernel +pandocfilters==1.5.0 + # via nbconvert +parso==0.8.3 + # via jedi +pexpect==4.8.0 + # via ipython +pickleshare==0.7.5 + # via ipython +pkgutil-resolve-name==1.3.10 + # via jsonschema +platformdirs==3.11.0 + # via + # -r docs.in + # jupyter-core +prompt-toolkit==3.0.41 + # via ipython +psutil==5.9.6 + # via ipykernel +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.2 + # via stack-data +pydantic==2.5.0 + # via + # autodoc-pydantic + # pydantic-settings +pydantic-core==2.14.1 + # via pydantic +pydantic-settings==2.1.0 + # via autodoc-pydantic +pydata-sphinx-theme==0.14.3 + # via -r docs.in +pygments==2.16.1 + # via + # accessible-pygments + # ipython + # nbconvert + # pydata-sphinx-theme + # sphinx +python-dotenv==1.0.0 + # via pydantic-settings +pytz==2023.3.post1 + # via babel +pyzmq==25.1.1 + # via + # ipykernel + # jupyter-client +referencing==0.30.2 + # via + # jsonschema + # jsonschema-specifications +requests==2.31.0 + # via sphinx +rpds-py==0.12.0 + # via + # jsonschema + # referencing +snowballstemmer==2.2.0 + # via sphinx +soupsieve==2.5 + # via beautifulsoup4 +sphinx==7.1.2 + # via + # -r docs.in + # autodoc-pydantic + # myst-parser + # nbsphinx + # pydata-sphinx-theme + # sphinx-autodoc-typehints + # sphinx-copybutton + # sphinx-design +sphinx-autodoc-typehints==1.23.0 + # via -r docs.in +sphinx-copybutton==0.5.2 + # via -r docs.in +sphinx-design==0.5.0 + # via -r docs.in +sphinxcontrib-applehelp==1.0.4 + # via sphinx +sphinxcontrib-devhelp==1.0.2 + # via sphinx +sphinxcontrib-htmlhelp==2.0.1 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.3 + # via sphinx +sphinxcontrib-serializinghtml==1.1.5 + # via sphinx +stack-data==0.6.3 + # via ipython +tinycss2==1.2.1 + # via nbconvert +tornado==6.3.3 + # via + # ipykernel + # jupyter-client +traitlets==5.13.0 + # via + # comm + # ipykernel + # ipython + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbclient + # nbconvert + # nbformat + # nbsphinx +typing-extensions==4.8.0 + # via + # annotated-types + # ipython + # pydantic + # pydantic-core + # pydata-sphinx-theme +urllib3==2.1.0 + # via requests +wcwidth==0.2.10 + # via prompt-toolkit +webencodings==0.5.1 + # via + # bleach + # tinycss2 diff --git a/packages/essnmx/requirements/make_base.py b/packages/essnmx/requirements/make_base.py new file mode 100644 index 00000000..3bc6ee51 --- /dev/null +++ b/packages/essnmx/requirements/make_base.py @@ -0,0 +1,52 @@ +import sys +from argparse import ArgumentParser +from typing import List + +import tomli + +parser = ArgumentParser() +parser.add_argument( + "--nightly", + default="", + help="List of dependencies to install from main branch for nightly tests, " + "separated by commas.", +) +args = parser.parse_args() + + +def write_dependencies(dependency_name: str, dependencies: List[str]) -> None: + header = "# Generated by 'tox -e deps', DO NOT EDIT MANUALLY!'\n" + with open(f"{dependency_name}.in", "w") as f: + f.write(header) + f.write("\n".join(dependencies)) + f.write("\n") + + +with open("../pyproject.toml", "rb") as toml_file: + pyproject = tomli.load(toml_file) + dependencies = pyproject["project"].get("dependencies") + if not dependencies: + raise RuntimeError("No dependencies found in pyproject.toml") + dependencies = [dep.strip().strip('"') for dep in dependencies] + +write_dependencies("base", dependencies) + + +def as_nightly(repo: str) -> str: + if "/" in repo: + org, repo = repo.split("/") + else: + org = "scipp" + if repo == "scipp": + version = f"cp{sys.version_info.major}{sys.version_info.minor}" + base = "https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly" + suffix = "manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + return "-".join([base, version, version, suffix]) + return f"{repo} @ git+https://github.com/{org}/{repo}@main" + + +nightly = tuple(args.nightly.split(",") if args.nightly else []) +nightly_dependencies = [dep for dep in dependencies if not dep.startswith(nightly)] +nightly_dependencies += [as_nightly(arg) for arg in nightly] + +write_dependencies("nightly", nightly_dependencies) diff --git a/packages/essnmx/requirements/mypy.in b/packages/essnmx/requirements/mypy.in new file mode 100644 index 00000000..5027d8c3 --- /dev/null +++ b/packages/essnmx/requirements/mypy.in @@ -0,0 +1,2 @@ +-r test.in +mypy diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt new file mode 100644 index 00000000..0286404d --- /dev/null +++ b/packages/essnmx/requirements/mypy.txt @@ -0,0 +1,14 @@ +# SHA1:859ef9c15e5e57c6c91510133c01f5751feee941 +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +-r test.txt +mypy==1.7.0 + # via -r mypy.in +mypy-extensions==1.0.0 + # via mypy +typing-extensions==4.8.0 + # via mypy diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in new file mode 100644 index 00000000..ba26ccdb --- /dev/null +++ b/packages/essnmx/requirements/nightly.in @@ -0,0 +1,7 @@ +# Generated by 'tox -e deps', DO NOT EDIT MANUALLY!' +dask +graphviz +https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +sciline @ git+https://github.com/scipp/sciline@main +scippnexus @ git+https://github.com/scipp/scippnexus@main +plopp @ git+https://github.com/scipp/plopp@main diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt new file mode 100644 index 00000000..b06f3a23 --- /dev/null +++ b/packages/essnmx/requirements/nightly.txt @@ -0,0 +1,84 @@ +# SHA1:9eb1772c4fb67e494b5edf3127d55cfebc6af71e +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +click==8.1.7 + # via dask +cloudpickle==3.0.0 + # via dask +contourpy==1.1.1 + # via matplotlib +cycler==0.12.1 + # via matplotlib +dask==2023.5.0 + # via -r nightly.in +fonttools==4.44.1 + # via matplotlib +fsspec==2023.10.0 + # via dask +graphlib-backport==1.0.3 + # via + # sciline + # scipp +graphviz==0.20.1 + # via -r nightly.in +h5py==3.10.0 + # via scippnexus +importlib-metadata==6.8.0 + # via dask +importlib-resources==6.1.1 + # via matplotlib +kiwisolver==1.4.5 + # via matplotlib +locket==1.0.0 + # via partd +matplotlib==3.7.3 + # via plopp +numpy==1.24.4 + # via + # contourpy + # h5py + # matplotlib + # scipp + # scipy +packaging==23.2 + # via + # dask + # matplotlib +partd==1.4.1 + # via dask +pillow==10.1.0 + # via matplotlib +plopp @ git+https://github.com/scipp/plopp@main + # via -r nightly.in +pyparsing==3.1.1 + # via matplotlib +python-dateutil==2.8.2 + # via + # matplotlib + # scippnexus +pyyaml==6.0.1 + # via dask +sciline @ git+https://github.com/scipp/sciline@main + # via -r nightly.in +scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + # via + # -r nightly.in + # scippnexus +scippnexus @ git+https://github.com/scipp/scippnexus@main + # via -r nightly.in +scipy==1.10.1 + # via scippnexus +six==1.16.0 + # via python-dateutil +toolz==0.12.0 + # via + # dask + # partd +zipp==3.17.0 + # via + # importlib-metadata + # importlib-resources diff --git a/packages/essnmx/requirements/static.in b/packages/essnmx/requirements/static.in new file mode 100644 index 00000000..416634f5 --- /dev/null +++ b/packages/essnmx/requirements/static.in @@ -0,0 +1 @@ +pre-commit diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt new file mode 100644 index 00000000..183c2534 --- /dev/null +++ b/packages/essnmx/requirements/static.txt @@ -0,0 +1,28 @@ +# SHA1:5a0b1bb22ae805d8aebba0f3bf05ab91aceae0d8 +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +cfgv==3.4.0 + # via pre-commit +distlib==0.3.7 + # via virtualenv +filelock==3.13.1 + # via virtualenv +identify==2.5.31 + # via pre-commit +nodeenv==1.8.0 + # via pre-commit +platformdirs==3.11.0 + # via virtualenv +pre-commit==3.5.0 + # via -r static.in +pyyaml==6.0.1 + # via pre-commit +virtualenv==20.24.6 + # via pre-commit + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/packages/essnmx/requirements/test.in b/packages/essnmx/requirements/test.in new file mode 100644 index 00000000..1cf404d7 --- /dev/null +++ b/packages/essnmx/requirements/test.in @@ -0,0 +1,2 @@ +-r base.in +pytest diff --git a/packages/essnmx/requirements/test.txt b/packages/essnmx/requirements/test.txt new file mode 100644 index 00000000..0139655f --- /dev/null +++ b/packages/essnmx/requirements/test.txt @@ -0,0 +1,18 @@ +# SHA1:a035a60fcbac4cd7bf595dbd81ee7994505d4a95 +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +-r base.txt +exceptiongroup==1.1.3 + # via pytest +iniconfig==2.0.0 + # via pytest +pluggy==1.3.0 + # via pytest +pytest==7.4.3 + # via -r test.in +tomli==2.0.1 + # via pytest diff --git a/packages/essnmx/requirements/wheels.in b/packages/essnmx/requirements/wheels.in new file mode 100644 index 00000000..378eac25 --- /dev/null +++ b/packages/essnmx/requirements/wheels.in @@ -0,0 +1 @@ +build diff --git a/packages/essnmx/requirements/wheels.txt b/packages/essnmx/requirements/wheels.txt new file mode 100644 index 00000000..01d277eb --- /dev/null +++ b/packages/essnmx/requirements/wheels.txt @@ -0,0 +1,21 @@ +# SHA1:80754af91bfb6d1073585b046fe0a474ce868509 +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +build==1.0.3 + # via -r wheels.in +importlib-metadata==6.8.0 + # via build +packaging==23.2 + # via build +pyproject-hooks==1.0.0 + # via build +tomli==2.0.1 + # via + # build + # pyproject-hooks +zipp==3.17.0 + # via importlib-metadata diff --git a/packages/essnmx/setup.cfg b/packages/essnmx/setup.cfg new file mode 100644 index 00000000..1ba190c5 --- /dev/null +++ b/packages/essnmx/setup.cfg @@ -0,0 +1,4 @@ +[flake8] +# See https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#line-length +max-line-length = 88 +extend-ignore = E203 diff --git a/packages/essnmx/src/ess/nmx/__init__.py b/packages/essnmx/src/ess/nmx/__init__.py new file mode 100644 index 00000000..6d115cc9 --- /dev/null +++ b/packages/essnmx/src/ess/nmx/__init__.py @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) + +# flake8: noqa +import importlib.metadata + +try: + __version__ = importlib.metadata.version(__package__ or __name__) +except importlib.metadata.PackageNotFoundError: + __version__ = "0.0.0" + +del importlib diff --git a/packages/essnmx/src/ess/nmx/py.typed b/packages/essnmx/src/ess/nmx/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/packages/essnmx/tests/package_test.py b/packages/essnmx/tests/package_test.py new file mode 100644 index 00000000..a0ce3927 --- /dev/null +++ b/packages/essnmx/tests/package_test.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +from ess import nmx as pkg + + +def test_has_version(): + assert hasattr(pkg, '__version__') diff --git a/packages/essnmx/tox.ini b/packages/essnmx/tox.ini new file mode 100644 index 00000000..14ae142d --- /dev/null +++ b/packages/essnmx/tox.ini @@ -0,0 +1,69 @@ +[tox] +envlist = py38 +isolated_build = true + +[testenv] +deps = -r requirements/test.txt +setenv = + JUPYTER_PLATFORM_DIRS = 1 +commands = pytest {posargs} + +[testenv:nightly] +deps = + -r requirements/nightly.txt + pytest +commands = pytest + +[testenv:unpinned] +description = Test with unpinned dependencies, as a user would install now. +deps = + essnmx + pytest +commands = pytest + +[testenv:docs] +description = invoke sphinx-build to build the HTML docs +deps = -r requirements/docs.txt +allowlist_externals=find +commands = python -m sphinx -j2 -v -b html -d {toxworkdir}/docs_doctrees docs html + python -m sphinx -j2 -v -b doctest -d {toxworkdir}/docs_doctrees docs html + find html -type f -name "*.ipynb" -not -path "html/_sources/*" -delete + +[testenv:releasedocs] +description = invoke sphinx-build to build the HTML docs from a released version +skip_install = true +deps = + essnmx=={posargs} + {[testenv:docs]deps} +allowlist_externals={[testenv:docs]allowlist_externals} +commands = {[testenv:docs]commands} + +[testenv:linkcheck] +description = Run Sphinx linkcheck +deps = -r requirements/docs.txt +commands = python -m sphinx -j2 -v -b linkcheck -d {toxworkdir}/docs_doctrees docs html + +[testenv:static] +description = Code formatting and static analysis +skip_install = true +deps = -r requirements/static.txt +allowlist_externals = sh +# The first run of pre-commit may reformat files. If this happens, it returns 1 but this +# should not fail the job. So just run again if it fails. A second failure means that +# either the different formatters can't agree on a format or that static analysis failed. +commands = sh -c 'pre-commit run -a || (echo "" && pre-commit run -a)' + +[testenv:mypy] +description = Type checking (mypy) +deps = -r requirements/mypy.txt +commands = python -m mypy . + +[testenv:deps] +description = Update dependencies by running pip-compile-multi +deps = + pip-compile-multi + tomli +skip_install = true +changedir = requirements +commands = python ./make_base.py --nightly scipp,sciline,scippnexus,plopp + pip-compile-multi -d . From c3dfacba34a6e936a3e0eed7e5ab4633dec7796b Mon Sep 17 00:00:00 2001 From: SimonHeybrock Date: Tue, 14 Nov 2023 13:44:24 +0000 Subject: [PATCH 002/403] Apply automatic formatting --- packages/essnmx/docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/docs/conf.py b/packages/essnmx/docs/conf.py index 09af0b33..dc5436ed 100644 --- a/packages/essnmx/docs/conf.py +++ b/packages/essnmx/docs/conf.py @@ -135,7 +135,7 @@ {"name": "Sciline", "url": "https://scipp.github.io/sciline"}, {"name": "Scipp", "url": "https://scipp.github.io"}, {"name": "ScippNexus", "url": "https://scipp.github.io/scippnexus"}, - ], + ], "icon_links": [ { "name": "GitHub", From 025adb1dc2db582ae385375d61683d5272d7a259 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Nov 2023 17:28:59 +0000 Subject: [PATCH 003/403] Bump scipp from 23.8.0 to 23.11.0 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 23.8.0 to 23.11.0. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/23.08.0...23.11.0) --- updated-dependencies: - dependency-name: scipp dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 3ff98223..c64f1e2f 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -9,8 +9,6 @@ click==8.1.7 # via dask cloudpickle==3.0.0 # via dask -confuse==2.0.1 - # via scipp contourpy==1.1.1 # via matplotlib cycler==0.12.1 @@ -31,8 +29,6 @@ h5py==3.10.0 # via scippnexus importlib-metadata==6.8.0 # via dask -importlib-resources==6.1.1 - # via matplotlib kiwisolver==1.4.5 # via matplotlib locket==1.0.0 @@ -63,12 +59,10 @@ python-dateutil==2.8.2 # matplotlib # scippnexus pyyaml==6.0.1 - # via - # confuse - # dask + # via dask sciline==23.9.1 # via -r base.in -scipp==23.8.0 +scipp==23.11.0 # via # -r base.in # scippnexus @@ -83,6 +77,4 @@ toolz==0.12.0 # dask # partd zipp==3.17.0 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata From 847560ecd00d7c1bd7426a05e4bc0ee74f4421e2 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 28 Nov 2023 10:24:00 +0100 Subject: [PATCH 004/403] Update pre-commit configuration. --- packages/essnmx/.pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/essnmx/.pre-commit-config.yaml b/packages/essnmx/.pre-commit-config.yaml index 6a86994b..19830db7 100644 --- a/packages/essnmx/.pre-commit-config.yaml +++ b/packages/essnmx/.pre-commit-config.yaml @@ -5,6 +5,7 @@ repos: - id: check-added-large-files - id: check-json exclude: asv.conf.json + - id: check-merge-conflict - id: check-toml - id: check-yaml exclude: conda/meta.yaml From a5b7e906f396af3947dd206597ca4df2cba876e5 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 28 Nov 2023 10:25:32 +0100 Subject: [PATCH 005/403] Update dependencies and drop python3.8 support. --- packages/essnmx/.github/workflows/ci.yml | 4 +- packages/essnmx/.github/workflows/docs.yml | 2 +- .../.github/workflows/nightly_at_main.yml | 4 +- .../.github/workflows/nightly_at_release.yml | 4 +- .../.github/workflows/python-version-ci | 2 +- packages/essnmx/.github/workflows/release.yml | 10 +-- packages/essnmx/.github/workflows/test.yml | 4 +- .../essnmx/.github/workflows/unpinned.yml | 4 +- packages/essnmx/conda/meta.yaml | 2 +- packages/essnmx/pyproject.toml | 3 +- packages/essnmx/requirements/base.in | 4 +- packages/essnmx/requirements/base.txt | 20 +++-- packages/essnmx/requirements/ci.txt | 10 +-- packages/essnmx/requirements/dev.in | 1 + packages/essnmx/requirements/dev.txt | 40 +++++++-- packages/essnmx/requirements/docs.in | 9 +- packages/essnmx/requirements/docs.txt | 85 +++++++------------ packages/essnmx/requirements/make_base.py | 24 +++++- packages/essnmx/requirements/mypy.txt | 2 +- packages/essnmx/requirements/nightly.in | 6 +- packages/essnmx/requirements/nightly.txt | 16 ++-- packages/essnmx/requirements/static.txt | 6 +- packages/essnmx/requirements/test.txt | 2 +- packages/essnmx/tox.ini | 2 +- 24 files changed, 148 insertions(+), 118 deletions(-) diff --git a/packages/essnmx/.github/workflows/ci.yml b/packages/essnmx/.github/workflows/ci.yml index 0d294d17..f9eb7efe 100644 --- a/packages/essnmx/.github/workflows/ci.yml +++ b/packages/essnmx/.github/workflows/ci.yml @@ -13,7 +13,7 @@ on: jobs: formatting: name: Formatting and static analysis - runs-on: 'ubuntu-20.04' + runs-on: 'ubuntu-22.04' outputs: min_python: ${{ steps.vars.outputs.min_python }} min_tox_env: ${{ steps.vars.outputs.min_tox_env }} @@ -39,7 +39,7 @@ jobs: needs: formatting strategy: matrix: - os: ['ubuntu-20.04'] + os: ['ubuntu-22.04'] python: - version: '${{needs.formatting.outputs.min_python}}' tox-env: '${{needs.formatting.outputs.min_tox_env}}' diff --git a/packages/essnmx/.github/workflows/docs.yml b/packages/essnmx/.github/workflows/docs.yml index 9dd22f19..6e363afb 100644 --- a/packages/essnmx/.github/workflows/docs.yml +++ b/packages/essnmx/.github/workflows/docs.yml @@ -39,7 +39,7 @@ env: jobs: docs: name: Build documentation - runs-on: 'ubuntu-20.04' + runs-on: 'ubuntu-22.04' steps: - run: sudo apt install --yes graphviz pandoc - uses: actions/checkout@v3 diff --git a/packages/essnmx/.github/workflows/nightly_at_main.yml b/packages/essnmx/.github/workflows/nightly_at_main.yml index f83c0c36..10730688 100644 --- a/packages/essnmx/.github/workflows/nightly_at_main.yml +++ b/packages/essnmx/.github/workflows/nightly_at_main.yml @@ -11,7 +11,7 @@ on: jobs: setup: name: Setup variables - runs-on: 'ubuntu-20.04' + runs-on: 'ubuntu-22.04' outputs: min_python: ${{ steps.vars.outputs.min_python }} steps: @@ -25,7 +25,7 @@ jobs: needs: setup strategy: matrix: - os: ['ubuntu-20.04'] + os: ['ubuntu-22.04'] python: - version: '${{needs.setup.outputs.min_python}}' tox-env: 'nightly' diff --git a/packages/essnmx/.github/workflows/nightly_at_release.yml b/packages/essnmx/.github/workflows/nightly_at_release.yml index f9d811a0..7f1653bb 100644 --- a/packages/essnmx/.github/workflows/nightly_at_release.yml +++ b/packages/essnmx/.github/workflows/nightly_at_release.yml @@ -11,7 +11,7 @@ on: jobs: setup: name: Setup variables - runs-on: 'ubuntu-20.04' + runs-on: 'ubuntu-22.04' outputs: min_python: ${{ steps.vars.outputs.min_python }} release_tag: ${{ steps.release.outputs.release_tag }} @@ -31,7 +31,7 @@ jobs: needs: setup strategy: matrix: - os: ['ubuntu-20.04'] + os: ['ubuntu-22.04'] python: - version: '${{needs.setup.outputs.min_python}}' tox-env: 'nightly' diff --git a/packages/essnmx/.github/workflows/python-version-ci b/packages/essnmx/.github/workflows/python-version-ci index cc1923a4..bd28b9c5 100644 --- a/packages/essnmx/.github/workflows/python-version-ci +++ b/packages/essnmx/.github/workflows/python-version-ci @@ -1 +1 @@ -3.8 +3.9 diff --git a/packages/essnmx/.github/workflows/release.yml b/packages/essnmx/.github/workflows/release.yml index 223dd5af..e492978a 100644 --- a/packages/essnmx/.github/workflows/release.yml +++ b/packages/essnmx/.github/workflows/release.yml @@ -15,7 +15,7 @@ defaults: jobs: build_conda: name: Conda build - runs-on: 'ubuntu-20.04' + runs-on: 'ubuntu-22.04' steps: - uses: actions/checkout@v3 @@ -38,7 +38,7 @@ jobs: build_wheels: name: Wheels - runs-on: 'ubuntu-20.04' + runs-on: 'ubuntu-22.04' steps: - uses: actions/checkout@v3 @@ -64,7 +64,7 @@ jobs: upload_pypi: name: Deploy PyPI needs: [build_wheels, build_conda] - runs-on: 'ubuntu-20.04' + runs-on: 'ubuntu-22.04' environment: release permissions: id-token: write @@ -76,7 +76,7 @@ jobs: upload_conda: name: Deploy Conda needs: [build_wheels, build_conda] - runs-on: 'ubuntu-20.04' + runs-on: 'ubuntu-22.04' if: github.event_name == 'release' && github.event.action == 'published' steps: @@ -100,7 +100,7 @@ jobs: assets: name: Upload docs needs: docs - runs-on: 'ubuntu-20.04' + runs-on: 'ubuntu-22.04' permissions: contents: write # This is needed so that the action can upload the asset steps: diff --git a/packages/essnmx/.github/workflows/test.yml b/packages/essnmx/.github/workflows/test.yml index 8ce71be0..3cfb75de 100644 --- a/packages/essnmx/.github/workflows/test.yml +++ b/packages/essnmx/.github/workflows/test.yml @@ -7,7 +7,7 @@ on: workflow_dispatch: inputs: os-variant: - default: 'ubuntu-20.04' + default: 'ubuntu-22.04' type: string python-version: type: string @@ -26,7 +26,7 @@ on: workflow_call: inputs: os-variant: - default: 'ubuntu-20.04' + default: 'ubuntu-22.04' type: string python-version: type: string diff --git a/packages/essnmx/.github/workflows/unpinned.yml b/packages/essnmx/.github/workflows/unpinned.yml index dbb546f4..853c1ec5 100644 --- a/packages/essnmx/.github/workflows/unpinned.yml +++ b/packages/essnmx/.github/workflows/unpinned.yml @@ -11,7 +11,7 @@ on: jobs: setup: name: Setup variables - runs-on: 'ubuntu-20.04' + runs-on: 'ubuntu-22.04' outputs: min_python: ${{ steps.vars.outputs.min_python }} release_tag: ${{ steps.release.outputs.release_tag }} @@ -31,7 +31,7 @@ jobs: needs: setup strategy: matrix: - os: ['ubuntu-20.04'] + os: ['ubuntu-22.04'] python: - version: '${{needs.setup.outputs.min_python}}' tox-env: 'unpinned' diff --git a/packages/essnmx/conda/meta.yaml b/packages/essnmx/conda/meta.yaml index 0e0aecaf..9cd6379c 100644 --- a/packages/essnmx/conda/meta.yaml +++ b/packages/essnmx/conda/meta.yaml @@ -11,8 +11,8 @@ requirements: - setuptools - setuptools_scm run: + - python>=3.9 - dask - - python>=3.8 - python-graphviz - plopp - sciline>=23.9.1 diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index f783626b..05a483a8 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -18,14 +18,13 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Topic :: Scientific/Engineering", "Typing :: Typed", ] -requires-python = ">=3.8" +requires-python = ">=3.9" # IMPORTANT: # Run 'tox -e deps' after making changes here. This will update requirement files. diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index 84d908ef..7f5f678e 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -1,4 +1,6 @@ -# Generated by 'tox -e deps', DO NOT EDIT MANUALLY!' + +# --- END OF CUSTOM SECTION --- +# The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask graphviz plopp diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index c64f1e2f..1f715543 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -9,13 +9,13 @@ click==8.1.7 # via dask cloudpickle==3.0.0 # via dask -contourpy==1.1.1 +contourpy==1.2.0 # via matplotlib cycler==0.12.1 # via matplotlib -dask==2023.5.0 +dask==2023.11.0 # via -r base.in -fonttools==4.44.1 +fonttools==4.45.1 # via matplotlib fsspec==2023.10.0 # via dask @@ -29,13 +29,15 @@ h5py==3.10.0 # via scippnexus importlib-metadata==6.8.0 # via dask +importlib-resources==6.1.1 + # via matplotlib kiwisolver==1.4.5 # via matplotlib locket==1.0.0 # via partd -matplotlib==3.7.3 +matplotlib==3.8.2 # via plopp -numpy==1.24.4 +numpy==1.26.2 # via # contourpy # h5py @@ -50,7 +52,7 @@ partd==1.4.1 # via dask pillow==10.1.0 # via matplotlib -plopp==23.10.1 +plopp==23.11.0 # via -r base.in pyparsing==3.1.1 # via matplotlib @@ -68,7 +70,7 @@ scipp==23.11.0 # scippnexus scippnexus==23.11.1 # via -r base.in -scipy==1.10.1 +scipy==1.11.4 # via scippnexus six==1.16.0 # via python-dateutil @@ -77,4 +79,6 @@ toolz==0.12.0 # dask # partd zipp==3.17.0 - # via importlib-metadata + # via + # importlib-metadata + # importlib-resources diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index a1a0562f..c446083e 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -7,7 +7,7 @@ # cachetools==5.3.2 # via tox -certifi==2023.7.22 +certifi==2023.11.17 # via requests chardet==5.2.0 # via tox @@ -25,14 +25,14 @@ gitdb==4.0.11 # via gitpython gitpython==3.1.40 # via -r ci.in -idna==3.4 +idna==3.6 # via requests packaging==23.2 # via # -r ci.in # pyproject-api # tox -platformdirs==3.11.0 +platformdirs==4.0.0 # via # tox # virtualenv @@ -48,9 +48,9 @@ tomli==2.0.1 # via # pyproject-api # tox -tox==4.11.3 +tox==4.11.4 # via -r ci.in urllib3==2.1.0 # via requests -virtualenv==20.24.6 +virtualenv==20.24.7 # via tox diff --git a/packages/essnmx/requirements/dev.in b/packages/essnmx/requirements/dev.in index 1c747e48..53ddf47e 100644 --- a/packages/essnmx/requirements/dev.in +++ b/packages/essnmx/requirements/dev.in @@ -5,6 +5,7 @@ -r static.in -r test.in -r wheels.in +copier jupyterlab pip-compile-multi pre-commit diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index 5ad3d870..a72b473c 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -1,4 +1,4 @@ -# SHA1:e3e8cd703eb07e0484bb54fe0e131426d458d1bc +# SHA1:efd19a3a98c69fc3d6d6233ed855de7e4a208f74 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -12,7 +12,9 @@ -r static.txt -r test.txt -r wheels.txt -anyio==4.0.0 +annotated-types==0.6.0 + # via pydantic +anyio==4.1.0 # via jupyter-server argon2-cffi==23.1.0 # via jupyter-server @@ -24,24 +26,32 @@ async-lru==2.0.4 # via jupyterlab cffi==1.16.0 # via argon2-cffi-bindings +copier==9.1.0 + # via -r dev.in +dunamai==1.19.0 + # via copier fqdn==1.5.1 # via jsonschema +funcy==2.0 + # via copier isoduration==20.11.0 # via jsonschema +jinja2-ansible-filters==1.3.2 + # via copier json5==0.9.14 # via jupyterlab-server jsonpointer==2.4 # via jsonschema -jsonschema[format-nongpl]==4.19.2 +jsonschema[format-nongpl]==4.20.0 # via # jupyter-events # jupyterlab-server # nbformat jupyter-events==0.9.0 # via jupyter-server -jupyter-lsp==2.2.0 +jupyter-lsp==2.2.1 # via jupyterlab -jupyter-server==2.10.0 +jupyter-server==2.11.1 # via # jupyter-lsp # jupyterlab @@ -49,24 +59,36 @@ jupyter-server==2.10.0 # notebook-shim jupyter-server-terminals==0.4.4 # via jupyter-server -jupyterlab==4.0.8 +jupyterlab==4.0.9 # via -r dev.in -jupyterlab-server==2.25.1 +jupyterlab-server==2.25.2 # via jupyterlab notebook-shim==0.2.3 # via jupyterlab overrides==7.4.0 # via jupyter-server +pathspec==0.11.2 + # via copier pip-compile-multi==2.6.3 # via -r dev.in pip-tools==7.3.0 # via pip-compile-multi -prometheus-client==0.18.0 +plumbum==1.8.2 + # via copier +prometheus-client==0.19.0 # via jupyter-server pycparser==2.21 # via cffi +pydantic==2.5.2 + # via copier +pydantic-core==2.14.5 + # via pydantic python-json-logger==2.0.7 # via jupyter-events +pyyaml-include==1.3.1 + # via copier +questionary==2.0.1 + # via copier rfc3339-validator==0.1.4 # via # jsonschema @@ -93,7 +115,7 @@ webcolors==1.13 # via jsonschema websocket-client==1.6.4 # via jupyter-server -wheel==0.41.3 +wheel==0.42.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/packages/essnmx/requirements/docs.in b/packages/essnmx/requirements/docs.in index 3d138485..43547152 100644 --- a/packages/essnmx/requirements/docs.in +++ b/packages/essnmx/requirements/docs.in @@ -1,12 +1,13 @@ -r base.in -autodoc_pydantic ipykernel ipython!=8.7.0 # Breaks syntax highlighting in Jupyter code cells. myst-parser nbsphinx -pydata-sphinx-theme>=0.13 +pydata-sphinx-theme>=0.14 sphinx -sphinx-autodoc-typehints==1.23.0 # Higher versions require sphinx-7, which is not supported by sphinx-design yet +sphinx-autodoc-typehints sphinx-copybutton sphinx-design -platformdirs<4 # temporary until virtualenv has release with support for this +# Temporary until questionary (dep of copier) updates +# See https://github.com/tmbo/questionary/blob/2df265534f3eb77aafcf70902e53e80beb1793e0/pyproject.toml#L36C43-L36C110 +prompt-toolkit==3.0.36 diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 820d951a..5808139b 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -1,4 +1,4 @@ -# SHA1:894ae0768d7baa8f604a6d257c02840e43ca2f74 +# SHA1:00de71d158c30ad98785b21ebfd6d3cf65448321 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -10,16 +10,12 @@ accessible-pygments==0.0.4 # via pydata-sphinx-theme alabaster==0.7.13 # via sphinx -annotated-types==0.6.0 - # via pydantic asttokens==2.4.1 # via stack-data attrs==23.1.0 # via # jsonschema # referencing -autodoc-pydantic==2.0.1 - # via -r docs.in babel==2.13.1 # via # pydata-sphinx-theme @@ -32,7 +28,7 @@ beautifulsoup4==4.12.2 # pydata-sphinx-theme bleach==6.1.0 # via nbconvert -certifi==2023.7.22 +certifi==2023.11.17 # via requests charset-normalizer==3.3.2 # via requests @@ -52,15 +48,15 @@ docutils==0.20.1 # sphinx executing==2.0.1 # via stack-data -fastjsonschema==2.18.1 +fastjsonschema==2.19.0 # via nbformat -idna==3.4 +idna==3.6 # via requests imagesize==1.4.1 # via sphinx -ipykernel==6.26.0 +ipykernel==6.27.1 # via -r docs.in -ipython==8.12.3 +ipython==8.9.0 # via # -r docs.in # ipykernel @@ -72,9 +68,9 @@ jinja2==3.1.2 # nbconvert # nbsphinx # sphinx -jsonschema==4.19.2 +jsonschema==4.20.0 # via nbformat -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2023.11.1 # via jsonschema jupyter-client==8.6.0 # via @@ -87,7 +83,7 @@ jupyter-core==5.5.0 # nbclient # nbconvert # nbformat -jupyterlab-pygments==0.2.2 +jupyterlab-pygments==0.3.0 # via nbconvert markdown-it-py==3.0.0 # via @@ -126,56 +122,42 @@ pandocfilters==1.5.0 # via nbconvert parso==0.8.3 # via jedi -pexpect==4.8.0 +pexpect==4.9.0 # via ipython pickleshare==0.7.5 # via ipython -pkgutil-resolve-name==1.3.10 - # via jsonschema -platformdirs==3.11.0 +platformdirs==4.0.0 + # via jupyter-core +prompt-toolkit==3.0.36 # via # -r docs.in - # jupyter-core -prompt-toolkit==3.0.41 - # via ipython + # ipython psutil==5.9.6 # via ipykernel ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 # via stack-data -pydantic==2.5.0 - # via - # autodoc-pydantic - # pydantic-settings -pydantic-core==2.14.1 - # via pydantic -pydantic-settings==2.1.0 - # via autodoc-pydantic -pydata-sphinx-theme==0.14.3 +pydata-sphinx-theme==0.14.4 # via -r docs.in -pygments==2.16.1 +pygments==2.17.2 # via # accessible-pygments # ipython # nbconvert # pydata-sphinx-theme # sphinx -python-dotenv==1.0.0 - # via pydantic-settings -pytz==2023.3.post1 - # via babel pyzmq==25.1.1 # via # ipykernel # jupyter-client -referencing==0.30.2 +referencing==0.31.0 # via # jsonschema # jsonschema-specifications requests==2.31.0 # via sphinx -rpds-py==0.12.0 +rpds-py==0.13.1 # via # jsonschema # referencing @@ -183,33 +165,37 @@ snowballstemmer==2.2.0 # via sphinx soupsieve==2.5 # via beautifulsoup4 -sphinx==7.1.2 +sphinx==7.2.6 # via # -r docs.in - # autodoc-pydantic # myst-parser # nbsphinx # pydata-sphinx-theme # sphinx-autodoc-typehints # sphinx-copybutton # sphinx-design -sphinx-autodoc-typehints==1.23.0 + # sphinxcontrib-applehelp + # sphinxcontrib-devhelp + # sphinxcontrib-htmlhelp + # sphinxcontrib-qthelp + # sphinxcontrib-serializinghtml +sphinx-autodoc-typehints==1.25.2 # via -r docs.in sphinx-copybutton==0.5.2 # via -r docs.in sphinx-design==0.5.0 # via -r docs.in -sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-applehelp==1.0.7 # via sphinx -sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-devhelp==1.0.5 # via sphinx -sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-htmlhelp==2.0.4 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-qthelp==1.0.6 # via sphinx -sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-serializinghtml==1.1.9 # via sphinx stack-data==0.6.3 # via ipython @@ -219,7 +205,7 @@ tornado==6.3.3 # via # ipykernel # jupyter-client -traitlets==5.13.0 +traitlets==5.14.0 # via # comm # ipykernel @@ -232,15 +218,10 @@ traitlets==5.13.0 # nbformat # nbsphinx typing-extensions==4.8.0 - # via - # annotated-types - # ipython - # pydantic - # pydantic-core - # pydata-sphinx-theme + # via pydata-sphinx-theme urllib3==2.1.0 # via requests -wcwidth==0.2.10 +wcwidth==0.2.12 # via prompt-toolkit webencodings==0.5.1 # via diff --git a/packages/essnmx/requirements/make_base.py b/packages/essnmx/requirements/make_base.py index 3bc6ee51..b26a1c2e 100644 --- a/packages/essnmx/requirements/make_base.py +++ b/packages/essnmx/requirements/make_base.py @@ -1,5 +1,9 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) + import sys from argparse import ArgumentParser +from pathlib import Path from typing import List import tomli @@ -13,11 +17,25 @@ ) args = parser.parse_args() +CUSTOM_AUTO_SEPARATOR = """ +# --- END OF CUSTOM SECTION --- +# The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! +""" + def write_dependencies(dependency_name: str, dependencies: List[str]) -> None: - header = "# Generated by 'tox -e deps', DO NOT EDIT MANUALLY!'\n" - with open(f"{dependency_name}.in", "w") as f: - f.write(header) + path = Path(f"{dependency_name}.in") + if path.exists(): + sections = path.read_text().split(CUSTOM_AUTO_SEPARATOR) + if len(sections) > 1: + custom = sections[0] + else: + custom = "" + else: + custom = "" + with path.open("w") as f: + f.write(custom) + f.write(CUSTOM_AUTO_SEPARATOR) f.write("\n".join(dependencies)) f.write("\n") diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index 0286404d..f98e1a53 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -6,7 +6,7 @@ # pip-compile-multi # -r test.txt -mypy==1.7.0 +mypy==1.7.1 # via -r mypy.in mypy-extensions==1.0.0 # via mypy diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index ba26ccdb..074c3768 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -1,7 +1,9 @@ -# Generated by 'tox -e deps', DO NOT EDIT MANUALLY!' + +# --- END OF CUSTOM SECTION --- +# The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask graphviz -https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl sciline @ git+https://github.com/scipp/sciline@main scippnexus @ git+https://github.com/scipp/scippnexus@main plopp @ git+https://github.com/scipp/plopp@main diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index b06f3a23..2f5ec066 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:9eb1772c4fb67e494b5edf3127d55cfebc6af71e +# SHA1:bb62654b0c2633ab5970957538299410683a0be9 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -9,13 +9,13 @@ click==8.1.7 # via dask cloudpickle==3.0.0 # via dask -contourpy==1.1.1 +contourpy==1.2.0 # via matplotlib cycler==0.12.1 # via matplotlib -dask==2023.5.0 +dask==2023.11.0 # via -r nightly.in -fonttools==4.44.1 +fonttools==4.45.1 # via matplotlib fsspec==2023.10.0 # via dask @@ -35,9 +35,9 @@ kiwisolver==1.4.5 # via matplotlib locket==1.0.0 # via partd -matplotlib==3.7.3 +matplotlib==3.8.2 # via plopp -numpy==1.24.4 +numpy==1.26.2 # via # contourpy # h5py @@ -64,13 +64,13 @@ pyyaml==6.0.1 # via dask sciline @ git+https://github.com/scipp/sciline@main # via -r nightly.in -scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl # via # -r nightly.in # scippnexus scippnexus @ git+https://github.com/scipp/scippnexus@main # via -r nightly.in -scipy==1.10.1 +scipy==1.11.4 # via scippnexus six==1.16.0 # via python-dateutil diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index 183c2534..cb2d12b3 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -11,17 +11,17 @@ distlib==0.3.7 # via virtualenv filelock==3.13.1 # via virtualenv -identify==2.5.31 +identify==2.5.32 # via pre-commit nodeenv==1.8.0 # via pre-commit -platformdirs==3.11.0 +platformdirs==4.0.0 # via virtualenv pre-commit==3.5.0 # via -r static.in pyyaml==6.0.1 # via pre-commit -virtualenv==20.24.6 +virtualenv==20.24.7 # via pre-commit # The following packages are considered to be unsafe in a requirements file: diff --git a/packages/essnmx/requirements/test.txt b/packages/essnmx/requirements/test.txt index 0139655f..a392d3d9 100644 --- a/packages/essnmx/requirements/test.txt +++ b/packages/essnmx/requirements/test.txt @@ -6,7 +6,7 @@ # pip-compile-multi # -r base.txt -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via pytest iniconfig==2.0.0 # via pytest diff --git a/packages/essnmx/tox.ini b/packages/essnmx/tox.ini index 14ae142d..b70da3d0 100644 --- a/packages/essnmx/tox.ini +++ b/packages/essnmx/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38 +envlist = py39 isolated_build = true [testenv] From c38feedf9de2c5de2b59b2394ea7047bde1282cf Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 28 Nov 2023 10:25:48 +0100 Subject: [PATCH 006/403] Update documentation materials. --- packages/essnmx/docs/_static/anaconda-icon.js | 13 +++ .../essnmx/docs/_static/anaconda-logo.svg | 103 ------------------ packages/essnmx/docs/_static/css/custom.css | 21 ---- .../essnmx/docs/_templates/doc_version.html | 2 +- packages/essnmx/docs/about/index.md | 2 +- packages/essnmx/docs/api-reference/index.md | 4 +- packages/essnmx/docs/conf.py | 10 +- .../essnmx/docs/developer/getting-started.md | 2 +- 8 files changed, 24 insertions(+), 133 deletions(-) create mode 100644 packages/essnmx/docs/_static/anaconda-icon.js delete mode 100644 packages/essnmx/docs/_static/anaconda-logo.svg delete mode 100644 packages/essnmx/docs/_static/css/custom.css diff --git a/packages/essnmx/docs/_static/anaconda-icon.js b/packages/essnmx/docs/_static/anaconda-icon.js new file mode 100644 index 00000000..024350ec --- /dev/null +++ b/packages/essnmx/docs/_static/anaconda-icon.js @@ -0,0 +1,13 @@ +FontAwesome.library.add( + (faListOldStyle = { + prefix: "fa-custom", + iconName: "anaconda", + icon: [ + 67.65, // viewBox width + 67.500267, // viewBox height + [], // ligature + "e001", // unicode codepoint - private use area + "M 33.900391 0 C 32.600392 0 31.299608 0.09921885 30.099609 0.19921875 A 39.81 39.81 0 0 1 35.199219 4.3007812 L 36.5 5.5 L 35.199219 6.8007812 A 34.65 34.65 0 0 0 32 10.199219 L 32 10.300781 A 6.12 6.12 0 0 0 31.5 10.900391 A 19.27 19.27 0 0 1 33.900391 10.800781 A 23 23 0 0 1 33.900391 56.800781 A 22.39 22.39 0 0 1 21.900391 53.400391 A 45.33 45.33 0 0 1 16.699219 53.699219 A 19.27 19.27 0 0 1 14.300781 53.599609 A 78.24 78.24 0 0 0 15 61.699219 A 33.26 33.26 0 0 0 33.900391 67.5 A 33.75 33.75 0 0 0 33.900391 0 z M 23 1.8007812 A 33.78 33.78 0 0 0 15.599609 5.4003906 A 47 47 0 0 1 20.699219 6.5996094 A 52.38 52.38 0 0 1 23 1.8007812 z M 26.5 2 A 41.8 41.8 0 0 0 23.699219 7.5996094 C 25.199217 8.1996088 26.69922 8.8000007 28.199219 9.5 C 28.799218 8.7000008 29.300391 8.0999999 29.400391 8 C 30.10039 7.2000008 30.800001 6.399218 31.5 5.6992188 A 58.59 58.59 0 0 0 26.5 2 z M 13.199219 8.1992188 A 48.47 48.47 0 0 0 13.099609 14.800781 A 44.05 44.05 0 0 1 18.300781 14.5 A 39.43 39.43 0 0 1 19.699219 9.5996094 A 46.94 46.94 0 0 0 13.199219 8.1992188 z M 10.099609 9.8007812 A 33.47 33.47 0 0 0 4.9003906 16.5 C 6.6003889 16 8.3992205 15.599218 10.199219 15.199219 C 10.099219 13.399221 10.099609 11.600779 10.099609 9.8007812 z M 22.599609 10.599609 C 22.19961 11.799608 21.8 13.100392 21.5 14.400391 A 29.18 29.18 0 0 1 26.199219 12.099609 A 27.49 27.49 0 0 0 22.599609 10.599609 z M 17.699219 17.5 C 16.19922 17.5 14.80078 17.599219 13.300781 17.699219 A 33.92 33.92 0 0 0 14.099609 22.099609 A 20.36 20.36 0 0 1 17.699219 17.5 z M 10.599609 17.900391 A 43.62 43.62 0 0 0 3.3007812 19.900391 L 3.0996094 20 L 3.1992188 20.199219 A 30.3 30.3 0 0 0 6.5 27.300781 L 6.5996094 27.5 L 6.8007812 27.400391 A 50.41 50.41 0 0 1 11.699219 24.300781 L 11.900391 24.199219 L 11.900391 24 A 38.39 38.39 0 0 1 10.800781 18.099609 L 10.800781 17.900391 L 10.599609 17.900391 z M 1.8007812 22.800781 L 1.5996094 23.400391 A 33.77 33.77 0 0 0 0 32.900391 L 0 33.5 L 0.40039062 33.099609 A 24.93 24.93 0 0 1 4.8007812 28.900391 L 5 28.800781 L 4.9003906 28.599609 A 54.49 54.49 0 0 1 2 23.300781 L 1.8007812 22.800781 z M 12.300781 26.300781 L 11.800781 26.599609 C 10.500783 27.399609 9.2003893 28.19961 7.9003906 29.099609 L 7.6992188 29.199219 L 8 29.400391 C 8.8999991 30.600389 9.8007822 31.900001 10.800781 33 L 11.099609 33.5 L 11.099609 32.900391 A 23.54 23.54 0 0 1 12.099609 26.900391 L 12.300781 26.300781 z M 6.0996094 30.5 L 5.9003906 30.699219 A 47 47 0 0 0 0.80078125 35.599609 L 0.59960938 35.800781 L 0.80078125 36 A 58.38 58.38 0 0 0 6.4003906 40.199219 L 6.5996094 40.300781 L 6.6992188 40.099609 A 45.3 45.3 0 0 1 9.6992188 35.5 L 9.8007812 35.300781 L 9.6992188 35.199219 A 52 52 0 0 1 6.1992188 30.800781 L 6.0996094 30.5 z M 11.300781 36.400391 L 11 36.900391 C 10.100001 38.200389 9.2003898 39.600001 8.4003906 41 L 8.3007812 41.199219 L 8.5 41.300781 C 9.8999986 42.10078 11.400392 42.800001 12.900391 43.5 L 13.400391 43.699219 L 13.199219 43.199219 A 23.11 23.11 0 0 1 11.400391 37 L 11.300781 36.400391 z M 0.099609375 37.699219 L 0.19921875 38.300781 A 31.56 31.56 0 0 0 2.9003906 47.699219 L 3.0996094 48.199219 L 3.3007812 47.699219 A 55.47 55.47 0 0 1 5.6992188 42.099609 L 5.8007812 41.800781 L 5.5996094 41.699219 A 57.36 57.36 0 0 1 0.59960938 38.099609 L 0.099609375 37.699219 z M 7.4003906 42.800781 L 7.3007812 43 A 53.76 53.76 0 0 0 4.5 50 L 4.4003906 50.199219 L 4.5996094 50.300781 A 39.14 39.14 0 0 0 12.199219 51.699219 L 12.5 51.699219 L 12.5 51.5 A 36.79 36.79 0 0 1 13 45.699219 L 13 45.5 L 12.800781 45.400391 A 49.67 49.67 0 0 1 7.5996094 42.900391 L 7.4003906 42.800781 z M 14.5 45.900391 L 14.5 46.199219 A 45.53 45.53 0 0 0 14.099609 51.5 L 14.099609 51.699219 L 14.300781 51.699219 C 15.10078 51.699219 15.89922 51.800781 16.699219 51.800781 A 12.19 12.19 0 0 0 19.400391 51.800781 L 20 51.800781 L 19.5 51.400391 A 20.73 20.73 0 0 1 14.900391 46.199219 L 14.900391 46.099609 L 14.5 45.900391 z M 5.1992188 52.099609 L 5.5 52.599609 A 34.87 34.87 0 0 0 12.599609 60.400391 L 13 60.800781 L 13 60.099609 A 51.43 51.43 0 0 1 12.5 53.5 L 12.5 53.300781 L 12.300781 53.300781 A 51.94 51.94 0 0 1 5.8007812 52.199219 L 5.1992188 52.099609 z" + ], + }) +); diff --git a/packages/essnmx/docs/_static/anaconda-logo.svg b/packages/essnmx/docs/_static/anaconda-logo.svg deleted file mode 100644 index 67bd68d9..00000000 --- a/packages/essnmx/docs/_static/anaconda-logo.svg +++ /dev/null @@ -1,103 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - diff --git a/packages/essnmx/docs/_static/css/custom.css b/packages/essnmx/docs/_static/css/custom.css deleted file mode 100644 index d7fabbbb..00000000 --- a/packages/essnmx/docs/_static/css/custom.css +++ /dev/null @@ -1,21 +0,0 @@ -html[data-theme="light"] { - /* match pst-color-text-muted (used in header buttons) */ - --scipp-filter-header-icon: saturate(0) brightness(0.696); -} - -html[data-theme="dark"] { - /* match pst-color-text-muted (used in header buttons) */ - --scipp-filter-header-icon: saturate(0) brightness(1.161); -} - -/* This selects custom icon links in the header but not the builtin ones. - * Currently, this is only the anaconda logo and the filters are adjusted to it. - */ -.bd-header .navbar-nav li a.nav-link .icon-link-image { - filter: var(--scipp-filter-header-icon); -} - -.bd-header .navbar-nav li a.nav-link .icon-link-image:hover { - /* match primary color */ - filter: hue-rotate(85deg) saturate(0.829) brightness(0.945); -} diff --git a/packages/essnmx/docs/_templates/doc_version.html b/packages/essnmx/docs/_templates/doc_version.html index 7fd881ac..a348e28c 100644 --- a/packages/essnmx/docs/_templates/doc_version.html +++ b/packages/essnmx/docs/_templates/doc_version.html @@ -1,2 +1,2 @@ -Current {{ project }} version: {{ version }} (older versions). +Current {{ project }} version: {{ version }} (older versions). diff --git a/packages/essnmx/docs/about/index.md b/packages/essnmx/docs/about/index.md index 98182ed2..b697ffaa 100644 --- a/packages/essnmx/docs/about/index.md +++ b/packages/essnmx/docs/about/index.md @@ -1,4 +1,4 @@ -# About Essnmx +# About ## Development diff --git a/packages/essnmx/docs/api-reference/index.md b/packages/essnmx/docs/api-reference/index.md index 2cb58f2e..7c36e9b8 100644 --- a/packages/essnmx/docs/api-reference/index.md +++ b/packages/essnmx/docs/api-reference/index.md @@ -3,7 +3,7 @@ ## Classes ```{eval-rst} -.. currentmodule:: essnmx +.. currentmodule:: ess.nmx .. autosummary:: :toctree: ../generated/classes @@ -26,4 +26,4 @@ :toctree: ../generated/modules :template: module-template.rst :recursive: -``` \ No newline at end of file +``` diff --git a/packages/essnmx/docs/conf.py b/packages/essnmx/docs/conf.py index dc5436ed..7db4e0da 100644 --- a/packages/essnmx/docs/conf.py +++ b/packages/essnmx/docs/conf.py @@ -121,6 +121,7 @@ html_theme_options = { "primary_sidebar_end": ["edit-this-page", "sourcelink"], "secondary_sidebar_items": [], + "navbar_persistent": ["search-button"], "show_nav_level": 1, # Adjust this to ensure external links are moved to "Move" menu "header_links_before_dropdown": 4, @@ -152,8 +153,8 @@ { "name": "Conda", "url": "https://anaconda.org/conda-forge/essnmx", - "icon": "_static/anaconda-logo.svg", - "type": "local", + "icon": "fa-custom fa-anaconda", + "type": "fontawesome", }, ], "footer_start": ["copyright", "sphinx-version"], @@ -174,7 +175,8 @@ # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] -html_css_files = ["css/custom.css"] +html_css_files = [] +html_js_files = ["anaconda-icon.js"] # -- Options for HTMLHelp output ------------------------------------------ @@ -184,7 +186,7 @@ # -- Options for Matplotlib in notebooks ---------------------------------- nbsphinx_execute_arguments = [ - "--Session.metadata=scipp_docs_build=True", + "--Session.metadata=scipp_sphinx_build=True", ] # -- Options for doctest -------------------------------------------------- diff --git a/packages/essnmx/docs/developer/getting-started.md b/packages/essnmx/docs/developer/getting-started.md index 1f0f5950..046d5978 100644 --- a/packages/essnmx/docs/developer/getting-started.md +++ b/packages/essnmx/docs/developer/getting-started.md @@ -40,7 +40,7 @@ Alternatively, if you want a different workflow, take a look at ``tox.ini`` or ` Run the tests using ```sh -tox -e py38 +tox -e py39 ``` (or just `tox` if you want to run all environments). From 3c7bc1ab9a8376722e9a8e79c66ac0ce13d3556e Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 28 Nov 2023 10:26:17 +0100 Subject: [PATCH 007/403] Update git ignore. --- packages/essnmx/.gitignore | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/packages/essnmx/.gitignore b/packages/essnmx/.gitignore index 9529e0f3..2af065bb 100644 --- a/packages/essnmx/.gitignore +++ b/packages/essnmx/.gitignore @@ -1,13 +1,14 @@ +# Build artifacts +build dist html .tox -src/essnmx.egg-info +*.egg-info *.sw? +# Caches .clangd/ -.idea/ -.vscode/ *.ipynb_checkpoints __pycache__/ .vs/ @@ -16,3 +17,20 @@ __pycache__/ .pytest_cache .mypy_cache docs/generated/ + +# Editor settings +.idea/ +.vscode/ + +# Data files +*.data +*.dat +*.csv +*.xye +*.h5 +*.hdf5 +*.hdf +*.nxs +*.raw +*.cif +*.rcif From 9755392a0e27ca593d6532fb7175b95069c47bd3 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 28 Nov 2023 10:27:01 +0100 Subject: [PATCH 008/403] Update pretty name. --- packages/essnmx/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/README.md b/packages/essnmx/README.md index f33d2aef..8756129e 100644 --- a/packages/essnmx/README.md +++ b/packages/essnmx/README.md @@ -3,7 +3,7 @@ [![Anaconda-Server Badge](https://anaconda.org/scipp/essnmx/badges/version.svg)](https://anaconda.org/scipp/essnmx) [![License: BSD 3-Clause](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](LICENSE) -# essnmx +# Essnmx ## About From 289c9a122a78c979ada50ebc97c03c6c855f4e28 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 28 Nov 2023 10:27:14 +0100 Subject: [PATCH 009/403] Copier update. --- packages/essnmx/.copier-answers.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/.copier-answers.yml b/packages/essnmx/.copier-answers.yml index 04f10eed..624ac6d8 100644 --- a/packages/essnmx/.copier-answers.yml +++ b/packages/essnmx/.copier-answers.yml @@ -1,9 +1,9 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY -_commit: 3ad29de +_commit: ff7f76b _src_path: gh:scipp/copier_template description: Data reduction for NMX at the European Spallation Source max_python: '3.11' -min_python: '3.8' +min_python: '3.9' namespace_package: ess nightly_deps: scipp,sciline,scippnexus,plopp orgname: scipp From c08cf66c735337319f564ddb8173bdfc077fa6b7 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 12 Dec 2023 09:58:01 +0100 Subject: [PATCH 010/403] Update dependencies. --- packages/essnmx/.github/workflows/ci.yml | 1 + packages/essnmx/.github/workflows/docs.yml | 8 +- packages/essnmx/README.md | 2 +- packages/essnmx/conda/meta.yaml | 7 +- packages/essnmx/pyproject.toml | 10 ++- packages/essnmx/requirements/base.in | 7 +- packages/essnmx/requirements/base.txt | 71 +++++++++++++++-- packages/essnmx/requirements/basetest.in | 4 + packages/essnmx/requirements/basetest.txt | 19 +++++ packages/essnmx/requirements/ci.txt | 6 +- packages/essnmx/requirements/dev.txt | 8 +- packages/essnmx/requirements/docs.in | 4 +- packages/essnmx/requirements/docs.txt | 93 ++++------------------ packages/essnmx/requirements/mypy.txt | 2 +- packages/essnmx/requirements/nightly.in | 3 +- packages/essnmx/requirements/nightly.txt | 29 ++++--- packages/essnmx/requirements/static.txt | 10 +-- packages/essnmx/requirements/test.in | 4 +- packages/essnmx/requirements/test.txt | 13 +-- packages/essnmx/requirements/wheels.txt | 2 +- 20 files changed, 172 insertions(+), 131 deletions(-) create mode 100644 packages/essnmx/requirements/basetest.in create mode 100644 packages/essnmx/requirements/basetest.txt diff --git a/packages/essnmx/.github/workflows/ci.yml b/packages/essnmx/.github/workflows/ci.yml index f9eb7efe..20c647dc 100644 --- a/packages/essnmx/.github/workflows/ci.yml +++ b/packages/essnmx/.github/workflows/ci.yml @@ -54,4 +54,5 @@ jobs: uses: ./.github/workflows/docs.yml with: publish: false + linkcheck: ${{ contains(matrix.variant.os, 'ubuntu') && github.ref == 'refs/heads/main' }} branch: ${{ github.head_ref == '' && github.ref_name || github.head_ref }} diff --git a/packages/essnmx/.github/workflows/docs.yml b/packages/essnmx/.github/workflows/docs.yml index 6e363afb..5e189fb0 100644 --- a/packages/essnmx/.github/workflows/docs.yml +++ b/packages/essnmx/.github/workflows/docs.yml @@ -32,6 +32,11 @@ on: default: '' required: false type: string + linkcheck: + description: 'Run the link checker. If not set the link checker will not be run.' + default: false + required: false + type: boolean env: VERSION: ${{ inputs.version }} @@ -55,6 +60,8 @@ jobs: if: ${{ inputs.version != '' }} - run: tox -e docs if: ${{ inputs.version == '' }} + - run: tox -e linkcheck + if: ${{ inputs.linkcheck }} - uses: actions/upload-artifact@v3 with: name: docs_html @@ -66,4 +73,3 @@ jobs: branch: gh-pages folder: html single-commit: true - ssh-key: ${{ secrets.GH_PAGES_DEPLOY_KEY }} diff --git a/packages/essnmx/README.md b/packages/essnmx/README.md index 8756129e..485cb08f 100644 --- a/packages/essnmx/README.md +++ b/packages/essnmx/README.md @@ -7,7 +7,7 @@ ## About -Data reduction for NMX at the European Spallation Source +Data reduction for NMX at the European Spallation Source. ## Installation diff --git a/packages/essnmx/conda/meta.yaml b/packages/essnmx/conda/meta.yaml index 9cd6379c..b945859d 100644 --- a/packages/essnmx/conda/meta.yaml +++ b/packages/essnmx/conda/meta.yaml @@ -18,6 +18,7 @@ requirements: - sciline>=23.9.1 - scipp>=23.8.0 - scippnexus>=23.9.0 + - pooch test: imports: @@ -33,12 +34,12 @@ test: build: noarch: python script: - - pip install . + - python -m pip install . about: home: https://github.com/scipp/essnmx license: BSD-3-Clause - summary: Data reduction for NMX at the European Spallation Source - description: Data reduction for NMX at the European Spallation Source + summary: Data reduction for NMX at the European Spallation Source. + description: Data reduction for NMX at the European Spallation Source. dev_url: https://github.com/scipp/essnmx doc_url: https://scipp.github.io/essnmx diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 05a483a8..dcca7582 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" [project] name = "essnmx" -description = "Data reduction for NMX at the European Spallation Source" +description = "Data reduction for NMX at the European Spallation Source." authors = [{ name = "Scipp contributors" }] license = { file = "LICENSE" } readme = "README.md" @@ -36,6 +36,7 @@ dependencies = [ "sciline>=23.9.1", "scipp>=23.8.0", "scippnexus>=23.9.0", + "pooch", ] dynamic = ["version"] @@ -49,7 +50,12 @@ dynamic = ["version"] [tool.pytest.ini_options] minversion = "7.0" -addopts = "-ra -v" +addopts = """ +--strict-config +--strict-markers +-ra +-v +""" testpaths = "tests" filterwarnings = [ "error", diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index 7f5f678e..a0270738 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -1,4 +1,8 @@ - +# Temporary until questionary (dep of copier) updates +# See https://github.com/tmbo/questionary/blob/2df265534f3eb77aafcf70902e53e80beb1793e0/pyproject.toml#L36C43-L36C110 +prompt-toolkit==3.0.36 +# Temporary pinned until prompt-tookit conflict is resolved. +ipython==8.9.0 # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask @@ -7,3 +11,4 @@ plopp sciline>=23.9.1 scipp>=23.8.0 scippnexus>=23.9.0 +pooch diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 1f715543..c460d075 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,10 +1,18 @@ -# SHA1:d319e96fc4fdca56b4aa64fcfe16d5daa656be7a +# SHA1:dc8a3b216315f3203ec4048ef1930f26643a833e # # This file is autogenerated by pip-compile-multi # To update, run: # # pip-compile-multi # +asttokens==2.4.1 + # via stack-data +backcall==0.2.0 + # via ipython +certifi==2023.11.17 + # via requests +charset-normalizer==3.3.2 + # via requests click==8.1.7 # via dask cloudpickle==3.0.0 @@ -13,11 +21,15 @@ contourpy==1.2.0 # via matplotlib cycler==0.12.1 # via matplotlib -dask==2023.11.0 +dask==2023.12.0 # via -r base.in -fonttools==4.45.1 +decorator==5.1.1 + # via ipython +executing==2.0.1 + # via stack-data +fonttools==4.46.0 # via matplotlib -fsspec==2023.10.0 +fsspec==2023.12.2 # via dask graphlib-backport==1.0.3 # via @@ -27,16 +39,24 @@ graphviz==0.20.1 # via -r base.in h5py==3.10.0 # via scippnexus -importlib-metadata==6.8.0 +idna==3.6 + # via requests +importlib-metadata==7.0.0 # via dask importlib-resources==6.1.1 # via matplotlib +ipython==8.9.0 + # via -r base.in +jedi==0.19.1 + # via ipython kiwisolver==1.4.5 # via matplotlib locket==1.0.0 # via partd matplotlib==3.8.2 # via plopp +matplotlib-inline==0.1.6 + # via ipython numpy==1.26.2 # via # contourpy @@ -48,12 +68,33 @@ packaging==23.2 # via # dask # matplotlib + # pooch +parso==0.8.3 + # via jedi partd==1.4.1 # via dask +pexpect==4.9.0 + # via ipython +pickleshare==0.7.5 + # via ipython pillow==10.1.0 # via matplotlib +platformdirs==4.1.0 + # via pooch plopp==23.11.0 # via -r base.in +pooch==1.8.0 + # via -r base.in +prompt-toolkit==3.0.36 + # via + # -r base.in + # ipython +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.2 + # via stack-data +pygments==2.17.2 + # via ipython pyparsing==3.1.1 # via matplotlib python-dateutil==2.8.2 @@ -62,22 +103,36 @@ python-dateutil==2.8.2 # scippnexus pyyaml==6.0.1 # via dask -sciline==23.9.1 +requests==2.31.0 + # via pooch +sciline==23.12.0 # via -r base.in scipp==23.11.0 # via # -r base.in # scippnexus -scippnexus==23.11.1 +scippnexus==23.12.0 # via -r base.in scipy==1.11.4 # via scippnexus six==1.16.0 - # via python-dateutil + # via + # asttokens + # python-dateutil +stack-data==0.6.3 + # via ipython toolz==0.12.0 # via # dask # partd +traitlets==5.14.0 + # via + # ipython + # matplotlib-inline +urllib3==2.1.0 + # via requests +wcwidth==0.2.12 + # via prompt-toolkit zipp==3.17.0 # via # importlib-metadata diff --git a/packages/essnmx/requirements/basetest.in b/packages/essnmx/requirements/basetest.in new file mode 100644 index 00000000..e4a48b29 --- /dev/null +++ b/packages/essnmx/requirements/basetest.in @@ -0,0 +1,4 @@ +# Dependencies that are only used by tests. +# Do not make an environment from this file, use test.txt instead! + +pytest diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt new file mode 100644 index 00000000..6bf43fa2 --- /dev/null +++ b/packages/essnmx/requirements/basetest.txt @@ -0,0 +1,19 @@ +# SHA1:0eaa389e1fdb3a1917c0f987514bd561be5718ee +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +exceptiongroup==1.2.0 + # via pytest +iniconfig==2.0.0 + # via pytest +packaging==23.2 + # via pytest +pluggy==1.3.0 + # via pytest +pytest==7.4.3 + # via -r basetest.in +tomli==2.0.1 + # via pytest diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index c446083e..e33fbbcf 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -15,7 +15,7 @@ charset-normalizer==3.3.2 # via requests colorama==0.4.6 # via tox -distlib==0.3.7 +distlib==0.3.8 # via virtualenv filelock==3.13.1 # via @@ -32,7 +32,7 @@ packaging==23.2 # -r ci.in # pyproject-api # tox -platformdirs==4.0.0 +platformdirs==4.1.0 # via # tox # virtualenv @@ -52,5 +52,5 @@ tox==4.11.4 # via -r ci.in urllib3==2.1.0 # via requests -virtualenv==20.24.7 +virtualenv==20.25.0 # via tox diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index a72b473c..5ab837c3 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -51,13 +51,13 @@ jupyter-events==0.9.0 # via jupyter-server jupyter-lsp==2.2.1 # via jupyterlab -jupyter-server==2.11.1 +jupyter-server==2.12.1 # via # jupyter-lsp # jupyterlab # jupyterlab-server # notebook-shim -jupyter-server-terminals==0.4.4 +jupyter-server-terminals==0.5.0 # via jupyter-server jupyterlab==4.0.9 # via -r dev.in @@ -67,7 +67,7 @@ notebook-shim==0.2.3 # via jupyterlab overrides==7.4.0 # via jupyter-server -pathspec==0.11.2 +pathspec==0.12.1 # via copier pip-compile-multi==2.6.3 # via -r dev.in @@ -113,7 +113,7 @@ uri-template==1.3.0 # via jsonschema webcolors==1.13 # via jsonschema -websocket-client==1.6.4 +websocket-client==1.7.0 # via jupyter-server wheel==0.42.0 # via pip-tools diff --git a/packages/essnmx/requirements/docs.in b/packages/essnmx/requirements/docs.in index 43547152..1dc7fb02 100644 --- a/packages/essnmx/requirements/docs.in +++ b/packages/essnmx/requirements/docs.in @@ -8,6 +8,4 @@ sphinx sphinx-autodoc-typehints sphinx-copybutton sphinx-design -# Temporary until questionary (dep of copier) updates -# See https://github.com/tmbo/questionary/blob/2df265534f3eb77aafcf70902e53e80beb1793e0/pyproject.toml#L36C43-L36C110 -prompt-toolkit==3.0.36 +ipywidgets diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 5808139b..45b41948 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -1,4 +1,4 @@ -# SHA1:00de71d158c30ad98785b21ebfd6d3cf65448321 +# SHA1:732848612841b246bdcb75a85f7305e3489d51ad # # This file is autogenerated by pip-compile-multi # To update, run: @@ -10,8 +10,6 @@ accessible-pygments==0.0.4 # via pydata-sphinx-theme alabaster==0.7.13 # via sphinx -asttokens==2.4.1 - # via stack-data attrs==23.1.0 # via # jsonschema @@ -20,24 +18,18 @@ babel==2.13.1 # via # pydata-sphinx-theme # sphinx -backcall==0.2.0 - # via ipython beautifulsoup4==4.12.2 # via # nbconvert # pydata-sphinx-theme bleach==6.1.0 # via nbconvert -certifi==2023.11.17 - # via requests -charset-normalizer==3.3.2 - # via requests comm==0.2.0 - # via ipykernel + # via + # ipykernel + # ipywidgets debugpy==1.8.0 # via ipykernel -decorator==5.1.1 - # via ipython defusedxml==0.7.1 # via nbconvert docutils==0.20.1 @@ -46,22 +38,14 @@ docutils==0.20.1 # nbsphinx # pydata-sphinx-theme # sphinx -executing==2.0.1 - # via stack-data fastjsonschema==2.19.0 # via nbformat -idna==3.6 - # via requests imagesize==1.4.1 # via sphinx ipykernel==6.27.1 # via -r docs.in -ipython==8.9.0 - # via - # -r docs.in - # ipykernel -jedi==0.19.1 - # via ipython +ipywidgets==8.1.1 + # via -r docs.in jinja2==3.1.2 # via # myst-parser @@ -70,7 +54,7 @@ jinja2==3.1.2 # sphinx jsonschema==4.20.0 # via nbformat -jsonschema-specifications==2023.11.1 +jsonschema-specifications==2023.11.2 # via jsonschema jupyter-client==8.6.0 # via @@ -85,6 +69,8 @@ jupyter-core==5.5.0 # nbformat jupyterlab-pygments==0.3.0 # via nbconvert +jupyterlab-widgets==3.0.9 + # via ipywidgets markdown-it-py==3.0.0 # via # mdit-py-plugins @@ -93,10 +79,6 @@ markupsafe==2.1.3 # via # jinja2 # nbconvert -matplotlib-inline==0.1.6 - # via - # ipykernel - # ipython mdit-py-plugins==0.4.0 # via myst-parser mdurl==0.1.2 @@ -107,7 +89,7 @@ myst-parser==2.0.0 # via -r docs.in nbclient==0.9.0 # via nbconvert -nbconvert==7.11.0 +nbconvert==7.12.0 # via nbsphinx nbformat==5.9.2 # via @@ -120,44 +102,19 @@ nest-asyncio==1.5.8 # via ipykernel pandocfilters==1.5.0 # via nbconvert -parso==0.8.3 - # via jedi -pexpect==4.9.0 - # via ipython -pickleshare==0.7.5 - # via ipython -platformdirs==4.0.0 - # via jupyter-core -prompt-toolkit==3.0.36 - # via - # -r docs.in - # ipython psutil==5.9.6 # via ipykernel -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data pydata-sphinx-theme==0.14.4 # via -r docs.in -pygments==2.17.2 - # via - # accessible-pygments - # ipython - # nbconvert - # pydata-sphinx-theme - # sphinx -pyzmq==25.1.1 +pyzmq==25.1.2 # via # ipykernel # jupyter-client -referencing==0.31.0 +referencing==0.32.0 # via # jsonschema # jsonschema-specifications -requests==2.31.0 - # via sphinx -rpds-py==0.13.1 +rpds-py==0.13.2 # via # jsonschema # referencing @@ -197,33 +154,17 @@ sphinxcontrib-qthelp==1.0.6 # via sphinx sphinxcontrib-serializinghtml==1.1.9 # via sphinx -stack-data==0.6.3 - # via ipython tinycss2==1.2.1 # via nbconvert -tornado==6.3.3 +tornado==6.4 # via # ipykernel # jupyter-client -traitlets==5.14.0 - # via - # comm - # ipykernel - # ipython - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbclient - # nbconvert - # nbformat - # nbsphinx -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via pydata-sphinx-theme -urllib3==2.1.0 - # via requests -wcwidth==0.2.12 - # via prompt-toolkit webencodings==0.5.1 # via # bleach # tinycss2 +widgetsnbextension==4.0.9 + # via ipywidgets diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index f98e1a53..99e44980 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -10,5 +10,5 @@ mypy==1.7.1 # via -r mypy.in mypy-extensions==1.0.0 # via mypy -typing-extensions==4.8.0 +typing-extensions==4.9.0 # via mypy diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index 074c3768..3b7a8998 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -1,8 +1,9 @@ - +-r basetest.in # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask graphviz +pooch https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl sciline @ git+https://github.com/scipp/sciline@main scippnexus @ git+https://github.com/scipp/scippnexus@main diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 2f5ec066..000d7ffc 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,10 +1,15 @@ -# SHA1:bb62654b0c2633ab5970957538299410683a0be9 +# SHA1:a82ae6ed279eacc614c956cfe2f650068f6d21ca # # This file is autogenerated by pip-compile-multi # To update, run: # # pip-compile-multi # +-r basetest.txt +certifi==2023.11.17 + # via requests +charset-normalizer==3.3.2 + # via requests click==8.1.7 # via dask cloudpickle==3.0.0 @@ -13,11 +18,11 @@ contourpy==1.2.0 # via matplotlib cycler==0.12.1 # via matplotlib -dask==2023.11.0 +dask==2023.12.0 # via -r nightly.in -fonttools==4.45.1 +fonttools==4.46.0 # via matplotlib -fsspec==2023.10.0 +fsspec==2023.12.2 # via dask graphlib-backport==1.0.3 # via @@ -27,7 +32,9 @@ graphviz==0.20.1 # via -r nightly.in h5py==3.10.0 # via scippnexus -importlib-metadata==6.8.0 +idna==3.6 + # via requests +importlib-metadata==7.0.0 # via dask importlib-resources==6.1.1 # via matplotlib @@ -44,16 +51,16 @@ numpy==1.26.2 # matplotlib # scipp # scipy -packaging==23.2 - # via - # dask - # matplotlib partd==1.4.1 # via dask pillow==10.1.0 # via matplotlib +platformdirs==4.1.0 + # via pooch plopp @ git+https://github.com/scipp/plopp@main # via -r nightly.in +pooch==1.8.0 + # via -r nightly.in pyparsing==3.1.1 # via matplotlib python-dateutil==2.8.2 @@ -62,6 +69,8 @@ python-dateutil==2.8.2 # scippnexus pyyaml==6.0.1 # via dask +requests==2.31.0 + # via pooch sciline @ git+https://github.com/scipp/sciline@main # via -r nightly.in scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl @@ -78,6 +87,8 @@ toolz==0.12.0 # via # dask # partd +urllib3==2.1.0 + # via requests zipp==3.17.0 # via # importlib-metadata diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index cb2d12b3..fa660fba 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -7,21 +7,21 @@ # cfgv==3.4.0 # via pre-commit -distlib==0.3.7 +distlib==0.3.8 # via virtualenv filelock==3.13.1 # via virtualenv -identify==2.5.32 +identify==2.5.33 # via pre-commit nodeenv==1.8.0 # via pre-commit -platformdirs==4.0.0 +platformdirs==4.1.0 # via virtualenv -pre-commit==3.5.0 +pre-commit==3.6.0 # via -r static.in pyyaml==6.0.1 # via pre-commit -virtualenv==20.24.7 +virtualenv==20.25.0 # via pre-commit # The following packages are considered to be unsafe in a requirements file: diff --git a/packages/essnmx/requirements/test.in b/packages/essnmx/requirements/test.in index 1cf404d7..7b409792 100644 --- a/packages/essnmx/requirements/test.in +++ b/packages/essnmx/requirements/test.in @@ -1,2 +1,4 @@ +# Add test dependencies in basetest.in + -r base.in -pytest +-r basetest.in diff --git a/packages/essnmx/requirements/test.txt b/packages/essnmx/requirements/test.txt index a392d3d9..3c7454d8 100644 --- a/packages/essnmx/requirements/test.txt +++ b/packages/essnmx/requirements/test.txt @@ -1,4 +1,4 @@ -# SHA1:a035a60fcbac4cd7bf595dbd81ee7994505d4a95 +# SHA1:ef2ee9576d8a9e65b44e2865a26887eed3fc49d1 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -6,13 +6,4 @@ # pip-compile-multi # -r base.txt -exceptiongroup==1.2.0 - # via pytest -iniconfig==2.0.0 - # via pytest -pluggy==1.3.0 - # via pytest -pytest==7.4.3 - # via -r test.in -tomli==2.0.1 - # via pytest +-r basetest.txt diff --git a/packages/essnmx/requirements/wheels.txt b/packages/essnmx/requirements/wheels.txt index 01d277eb..d5378cf7 100644 --- a/packages/essnmx/requirements/wheels.txt +++ b/packages/essnmx/requirements/wheels.txt @@ -7,7 +7,7 @@ # build==1.0.3 # via -r wheels.in -importlib-metadata==6.8.0 +importlib-metadata==7.0.0 # via build packaging==23.2 # via build From 7aa9d22a1961629aee040351d75af31aa17fce78 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 12 Dec 2023 09:58:25 +0100 Subject: [PATCH 011/403] Update from copier template. --- packages/essnmx/.copier-answers.yml | 4 ++-- packages/essnmx/.gitignore | 3 +++ packages/essnmx/docs/conf.py | 25 +++++++++++++++++++++++-- packages/essnmx/docs/index.md | 2 +- packages/essnmx/tox.ini | 6 ++---- 5 files changed, 31 insertions(+), 9 deletions(-) diff --git a/packages/essnmx/.copier-answers.yml b/packages/essnmx/.copier-answers.yml index 624ac6d8..b8eabc0f 100644 --- a/packages/essnmx/.copier-answers.yml +++ b/packages/essnmx/.copier-answers.yml @@ -1,7 +1,7 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY -_commit: ff7f76b +_commit: ec09f73 _src_path: gh:scipp/copier_template -description: Data reduction for NMX at the European Spallation Source +description: Data reduction for NMX at the European Spallation Source. max_python: '3.11' min_python: '3.9' namespace_package: ess diff --git a/packages/essnmx/.gitignore b/packages/essnmx/.gitignore index 2af065bb..e8b1ec78 100644 --- a/packages/essnmx/.gitignore +++ b/packages/essnmx/.gitignore @@ -7,6 +7,9 @@ html *.sw? +# Environments +venv + # Caches .clangd/ *.ipynb_checkpoints diff --git a/packages/essnmx/docs/conf.py b/packages/essnmx/docs/conf.py index 7db4e0da..01842ec5 100644 --- a/packages/essnmx/docs/conf.py +++ b/packages/essnmx/docs/conf.py @@ -25,7 +25,7 @@ 'sphinx.ext.napoleon', 'sphinx_autodoc_typehints', 'sphinx_copybutton', - "sphinx_design", + 'sphinx_design', 'nbsphinx', 'myst_parser', ] @@ -152,7 +152,7 @@ }, { "name": "Conda", - "url": "https://anaconda.org/conda-forge/essnmx", + "url": "https://anaconda.org/scipp/essnmx", "icon": "fa-custom fa-anaconda", "type": "fontawesome", }, @@ -191,8 +191,29 @@ # -- Options for doctest -------------------------------------------------- +# sc.plot returns a Figure object and doctest compares that against the +# output written in the docstring. But we only want to show an image of the +# figure, not its `repr`. +# In addition, there is no need to make plots in doctest as the documentation +# build already tests if those plots can be made. +# So we simply disable plots in doctests. doctest_global_setup = ''' import numpy as np + +try: + import scipp as sc + + def do_not_plot(*args, **kwargs): + pass + + sc.plot = do_not_plot + sc.Variable.plot = do_not_plot + sc.DataArray.plot = do_not_plot + sc.DataGroup.plot = do_not_plot + sc.Dataset.plot = do_not_plot +except ImportError: + # Scipp is not needed by docs if it is not installed. + pass ''' # Using normalize whitespace because many __str__ functions in scipp produce diff --git a/packages/essnmx/docs/index.md b/packages/essnmx/docs/index.md index 9d9bd075..2127a70b 100644 --- a/packages/essnmx/docs/index.md +++ b/packages/essnmx/docs/index.md @@ -1,7 +1,7 @@ # Essnmx - Data reduction for NMX at the European Spallation Source + Data reduction for NMX at the European Spallation Source.

diff --git a/packages/essnmx/tox.ini b/packages/essnmx/tox.ini index b70da3d0..4eaad04e 100644 --- a/packages/essnmx/tox.ini +++ b/packages/essnmx/tox.ini @@ -9,9 +9,7 @@ setenv = commands = pytest {posargs} [testenv:nightly] -deps = - -r requirements/nightly.txt - pytest +deps = -r requirements/nightly.txt commands = pytest [testenv:unpinned] @@ -66,4 +64,4 @@ deps = skip_install = true changedir = requirements commands = python ./make_base.py --nightly scipp,sciline,scippnexus,plopp - pip-compile-multi -d . + pip-compile-multi -d . --backtracking From 3af771f3f403217180502b9f9f6db3bd11785f01 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 12 Dec 2023 14:29:34 +0100 Subject: [PATCH 012/403] Remove unnecessary dependency. --- packages/essnmx/requirements/docs.in | 1 - packages/essnmx/requirements/docs.txt | 12 ++---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/packages/essnmx/requirements/docs.in b/packages/essnmx/requirements/docs.in index 1dc7fb02..e542e53b 100644 --- a/packages/essnmx/requirements/docs.in +++ b/packages/essnmx/requirements/docs.in @@ -8,4 +8,3 @@ sphinx sphinx-autodoc-typehints sphinx-copybutton sphinx-design -ipywidgets diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 45b41948..b1613253 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -1,4 +1,4 @@ -# SHA1:732848612841b246bdcb75a85f7305e3489d51ad +# SHA1:2175813590b5d31dc1cdf3e3c820f699647e9043 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -25,9 +25,7 @@ beautifulsoup4==4.12.2 bleach==6.1.0 # via nbconvert comm==0.2.0 - # via - # ipykernel - # ipywidgets + # via ipykernel debugpy==1.8.0 # via ipykernel defusedxml==0.7.1 @@ -44,8 +42,6 @@ imagesize==1.4.1 # via sphinx ipykernel==6.27.1 # via -r docs.in -ipywidgets==8.1.1 - # via -r docs.in jinja2==3.1.2 # via # myst-parser @@ -69,8 +65,6 @@ jupyter-core==5.5.0 # nbformat jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-widgets==3.0.9 - # via ipywidgets markdown-it-py==3.0.0 # via # mdit-py-plugins @@ -166,5 +160,3 @@ webencodings==0.5.1 # via # bleach # tinycss2 -widgetsnbextension==4.0.9 - # via ipywidgets From a9bc77f529fce25f7694ccd5d826a09a05f75474 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:28:50 +0000 Subject: [PATCH 013/403] Bump scipp from 23.11.0 to 23.11.1 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 23.11.0 to 23.11.1. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/23.11.0...23.11.1) --- updated-dependencies: - dependency-name: scipp dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index c460d075..82c87812 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -43,8 +43,6 @@ idna==3.6 # via requests importlib-metadata==7.0.0 # via dask -importlib-resources==6.1.1 - # via matplotlib ipython==8.9.0 # via -r base.in jedi==0.19.1 @@ -107,7 +105,7 @@ requests==2.31.0 # via pooch sciline==23.12.0 # via -r base.in -scipp==23.11.0 +scipp==23.11.1 # via # -r base.in # scippnexus @@ -134,6 +132,4 @@ urllib3==2.1.0 wcwidth==0.2.12 # via prompt-toolkit zipp==3.17.0 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata From 0f78cdd9b2fe903d23360aa603ba52b2f93af9e2 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 12 Dec 2023 14:26:59 +0100 Subject: [PATCH 014/403] Minimal workflow for McStas data. --- packages/essnmx/docs/examples/workflow.ipynb | 117 +++++++++++++++++++ packages/essnmx/src/ess/nmx/data/__init__.py | 36 ++++++ packages/essnmx/src/ess/nmx/detector.py | 16 +++ packages/essnmx/src/ess/nmx/loader.py | 92 +++++++++++++++ packages/essnmx/src/ess/nmx/reduction.py | 21 ++++ packages/essnmx/src/ess/nmx/workflow.py | 11 ++ packages/essnmx/tests/loader_test.py | 29 +++++ packages/essnmx/tests/workflow_test.py | 47 ++++++++ 8 files changed, 369 insertions(+) create mode 100644 packages/essnmx/docs/examples/workflow.ipynb create mode 100644 packages/essnmx/src/ess/nmx/data/__init__.py create mode 100644 packages/essnmx/src/ess/nmx/detector.py create mode 100644 packages/essnmx/src/ess/nmx/loader.py create mode 100644 packages/essnmx/src/ess/nmx/reduction.py create mode 100644 packages/essnmx/src/ess/nmx/workflow.py create mode 100644 packages/essnmx/tests/loader_test.py create mode 100644 packages/essnmx/tests/workflow_test.py diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb new file mode 100644 index 00000000..b4452d8a --- /dev/null +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -0,0 +1,117 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Workflow\n", + "\n", + "## Collect Parameters and Providers\n", + "### Simulation(McStas) Data\n", + "There is an extra parameter that can be configured for **McStas** simulation data workflow.
\n", + "It is because ``weights`` are given as probability, not integer number of events in a McStas file.
\n", + "Therefore users can set a scale factor called ``MaximumProabability`` to derive more realistic number of events.
" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Collect parameters and providers\n", + "import scipp as sc\n", + "from ess.nmx.workflow import collect_default_parameters, providers\n", + "from ess.nmx.loader import (\n", + " InputFileName,\n", + " MaximumProbability,\n", + " DefaultMaximumProbability\n", + ")\n", + "from ess.nmx.data import small_mcstas_sample\n", + "\n", + "file_path = small_mcstas_sample() # Replace it with your data file path\n", + "\n", + "params = {\n", + " **collect_default_parameters(),\n", + " InputFileName: InputFileName(file_path),\n", + " # Additional parameters for McStas data handling.\n", + " MaximumProbability: DefaultMaximumProbability,\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from typing import get_type_hints\n", + "param_reprs = {key.__name__: value for key, value in params.items()}\n", + "prov_reprs = {get_type_hints(prov)['return'].__name__: prov.__name__ for prov in providers}\n", + "\n", + "# Providers and parameters to be used for pipeline\n", + "sc.DataGroup(**prov_reprs, **param_reprs)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Build Workflow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sciline as sl\n", + "from ess.nmx.reduction import GroupedByPixelID\n", + "\n", + "nmx_pl = sl.Pipeline(list(providers), params=params)\n", + "nmx_workflow = nmx_pl.get(GroupedByPixelID)\n", + "nmx_workflow.visualize()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute Desired Types" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Event data grouped by detector panel and pixel id.\n", + "da = nmx_workflow.compute(GroupedByPixelID)\n", + "da" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nmx-dev-39", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/packages/essnmx/src/ess/nmx/data/__init__.py b/packages/essnmx/src/ess/nmx/data/__init__.py new file mode 100644 index 00000000..360c4eba --- /dev/null +++ b/packages/essnmx/src/ess/nmx/data/__init__.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) + +_version = '0' + +__all__ = ['small_mcstas_sample', 'get_path'] + + +def _make_pooch(): + import pooch + + return pooch.create( + path=pooch.os_cache('essnmx'), + env='ESSNMX_DATA_DIR', + retry_if_failed=3, + base_url='https://public.esss.dk/groups/scipp/ess/nmx/', + version=_version, + registry={'small_mcstas_sample.h5': 'md5:c3affe636397f8c9eea1d9c10a2bf487'}, + ) + + +_pooch = _make_pooch() + + +def small_mcstas_sample(): + return get_path('small_mcstas_sample.h5') + + +def get_path(name: str) -> str: + """ + Return the path to a data file bundled with ess nmx. + + This function only works with example data and cannot handle + paths to custom files. + """ + return _pooch.fetch(name) diff --git a/packages/essnmx/src/ess/nmx/detector.py b/packages/essnmx/src/ess/nmx/detector.py new file mode 100644 index 00000000..f2e79164 --- /dev/null +++ b/packages/essnmx/src/ess/nmx/detector.py @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +from typing import NewType + +MaxNumberOfPixelsPerAxis = NewType("MaxNumberOfPixelsPerAxis", int) +PixelStep = NewType("PixelStep", int) +NumberOfDetectors = NewType("NumberOfDetectors", int) +NumberOfAxes = NewType("NumberOfAxes", int) + + +default_params = { + MaxNumberOfPixelsPerAxis: MaxNumberOfPixelsPerAxis(1280), + NumberOfAxes: NumberOfAxes(2), + PixelStep: PixelStep(1), + NumberOfDetectors: NumberOfDetectors(3), +} diff --git a/packages/essnmx/src/ess/nmx/loader.py b/packages/essnmx/src/ess/nmx/loader.py new file mode 100644 index 00000000..99060600 --- /dev/null +++ b/packages/essnmx/src/ess/nmx/loader.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +from dataclasses import dataclass +from typing import Iterable, NewType, Optional + +import scipp as sc +import scippnexus as snx + +PixelIDs = NewType("PixelIDs", sc.Variable) +InputFileName = NewType("InputFileName", str) +Events = NewType("Events", sc.DataArray) + +# McStas Configurations +MaximumProbability = NewType("MaximumProbability", int) +DefaultMaximumProbability = MaximumProbability(100_000) + + +def _retrieve_event_list_name(keys: Iterable[str]) -> str: + for key in keys: + if key.startswith("bank01_events_dat_list"): + return key + raise ValueError("Can not find event list name.") + + +def _copy_partial_var( + var: sc.Variable, dim: str, idx: int, unit: str, dtype: Optional[str] = None +) -> sc.Variable: + """Retrieve property from variable.""" + original_var = var[dim, idx] + var = original_var.copy().astype(dtype) if dtype else original_var.copy() + var.unit = sc.Unit(unit) + return var + + +def _get_mcstas_pixel_ids() -> PixelIDs: + """pixel IDs for each detector""" + intervals = [(1, 1638401), (2000001, 3638401), (4000001, 5638401)] + ids = [sc.arange('id', start, stop) for start, stop in intervals] + return PixelIDs(sc.concat(ids, 'id')) + + +@dataclass +class NMXData: + """Data class for NMX data""" + + events: Events + all_pixel_ids: PixelIDs + + +def _load_mcstas_nmx_file(file: snx.File, max_prop: MaximumProbability) -> NMXData: + """Load McStas NMX data from h5 file. + + ``max_pro`` is used to scale ``weights`` to derive more realistic number of events. + """ + from functools import partial + + bank_name = _retrieve_event_list_name(file["entry1/data"].keys()) + var: sc.Variable + var = file["entry1/data/" + bank_name]["events"][()].rename_dims({'dim_0': 'event'}) + + copier = partial(_copy_partial_var, var, dim='dim_1') + weights = copier(idx=0, unit='counts') + id_list = copier(idx=4, unit='dimensionless', dtype="int64") + t_list = copier(idx=5, unit='s') + + weights = (max_prop / weights.max()) * weights + + loaded = sc.DataArray(data=weights, coords={'t': t_list, 'id': id_list}) + pixel_ids = _get_mcstas_pixel_ids() + return NMXData(Events(loaded), pixel_ids) + + +def load_nmx_file( + file_name: InputFileName, + max_prop: Optional[MaximumProbability] = None, +) -> NMXData: + """Load NMX data from a file and generate pixel ID coordinate. + + Check an entry path in the file and handle the data accordingly. + Nexus file should have an entry path called ``entry`` and + McStas simulation file should have an entry path called ``entry1``. + + Pixel id coordinate are wrapped together with the data + since they are different for simulation and real data. + """ + with snx.File(file_name) as file: + if "entry1" in file: # McStas file + return _load_mcstas_nmx_file(file, max_prop or DefaultMaximumProbability) + elif 'entry' in file: # Nexus file + raise NotImplementedError("Measurement data loader is not implemented yet.") + else: + raise ValueError(f"Can not load {file_name} with NMX file loader.") diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py new file mode 100644 index 00000000..09a9763a --- /dev/null +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +from typing import NewType + +import scipp as sc + +from .detector import NumberOfDetectors +from .loader import NMXData + +GroupedByPixelID = NewType("GroupedByPixelID", sc.DataArray) + + +def get_grouped_by_pixel_id( + loaded: NMXData, num_panels: NumberOfDetectors +) -> GroupedByPixelID: + """group events by pixel ID""" + grouped = loaded.events.group(loaded.all_pixel_ids) + + return GroupedByPixelID( + grouped.fold(dim='id', sizes={'panel': num_panels, 'id': -1}) + ) diff --git a/packages/essnmx/src/ess/nmx/workflow.py b/packages/essnmx/src/ess/nmx/workflow.py new file mode 100644 index 00000000..bdd1634d --- /dev/null +++ b/packages/essnmx/src/ess/nmx/workflow.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +from .detector import default_params as detector_default_params +from .loader import load_nmx_file +from .reduction import get_grouped_by_pixel_id + +providers = (load_nmx_file, get_grouped_by_pixel_id) + + +def collect_default_parameters() -> dict: + return dict(detector_default_params) diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py new file mode 100644 index 00000000..c0e7e545 --- /dev/null +++ b/packages/essnmx/tests/loader_test.py @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +import scipp as sc + + +def test_file_reader_mcstas() -> None: + import scippnexus as snx + + from ess.nmx.data import small_mcstas_sample + from ess.nmx.loader import DefaultMaximumProbability, InputFileName, load_nmx_file + + file_path = InputFileName(small_mcstas_sample()) + entry_path = "entry1/data/bank01_events_dat_list_p_x_y_n_id_t" + da = load_nmx_file(file_path).events + + with snx.File(file_path) as file: + raw_data: sc.Variable = file[entry_path]["events"][()] + + weights = raw_data['dim_1', 0].copy() + weights.unit = '1' + expected_data = (DefaultMaximumProbability / weights.max()) * weights + + assert isinstance(da, sc.DataArray) + assert list(da.values) == list(expected_data.values) + assert list(da.coords['id'].values) == list(raw_data['dim_1', 4].values) + assert list(da.coords['t'].values) == list(raw_data['dim_1', 5].values) + assert da.unit == '1' + assert da.coords['id'].unit == 'dimensionless' + assert da.coords['t'].unit == 's' diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py new file mode 100644 index 00000000..c6c31d29 --- /dev/null +++ b/packages/essnmx/tests/workflow_test.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +import pytest +import sciline as sl +import scipp as sc + + +@pytest.fixture +def mcstas_workflow() -> sl.Pipeline: + from ess.nmx.data import small_mcstas_sample + from ess.nmx.loader import ( + DefaultMaximumProbability, + InputFileName, + MaximumProbability, + ) + from ess.nmx.workflow import collect_default_parameters, providers + + return sl.Pipeline( + list(providers), + params={ + **collect_default_parameters(), + InputFileName: small_mcstas_sample(), + MaximumProbability: DefaultMaximumProbability, + }, + ) + + +def test_pipeline_builder(mcstas_workflow: sl.Pipeline) -> None: + from ess.nmx.data import small_mcstas_sample + from ess.nmx.loader import InputFileName + + assert mcstas_workflow.get(InputFileName).compute() == small_mcstas_sample() + + +def test_pipeline_mcstas_loader(mcstas_workflow: sl.Pipeline) -> None: + """Test if the loader graph is complete.""" + from ess.nmx.loader import NMXData + + nmx_data = mcstas_workflow.compute(NMXData) + assert isinstance(nmx_data.events, sc.DataArray) + + +def test_pipeline_mcstas_grouping(mcstas_workflow: sl.Pipeline) -> None: + """Test if the data reduction graph is complete.""" + from ess.nmx.reduction import GroupedByPixelID + + assert isinstance(mcstas_workflow.compute(GroupedByPixelID), sc.DataArray) From c5437e3ae6d73ed0d1a35d3e6cfabc2acfc72be3 Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Wed, 13 Dec 2023 14:25:40 +0100 Subject: [PATCH 015/403] Apply suggestions from code review Co-authored-by: Simon Heybrock <12912489+SimonHeybrock@users.noreply.github.com> --- packages/essnmx/src/ess/nmx/loader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/loader.py b/packages/essnmx/src/ess/nmx/loader.py index 99060600..66603b42 100644 --- a/packages/essnmx/src/ess/nmx/loader.py +++ b/packages/essnmx/src/ess/nmx/loader.py @@ -7,7 +7,7 @@ import scippnexus as snx PixelIDs = NewType("PixelIDs", sc.Variable) -InputFileName = NewType("InputFileName", str) +InputFilename = NewType("InputFilename", str) Events = NewType("Events", sc.DataArray) # McStas Configurations @@ -26,8 +26,8 @@ def _copy_partial_var( var: sc.Variable, dim: str, idx: int, unit: str, dtype: Optional[str] = None ) -> sc.Variable: """Retrieve property from variable.""" - original_var = var[dim, idx] - var = original_var.copy().astype(dtype) if dtype else original_var.copy() + original_var = var[dim, idx].copy() + var = original_var.astype(dtype) if dtype else original_var var.unit = sc.Unit(unit) return var From 1621de4648fa19391b369b2ec340fcc967781f45 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 13 Dec 2023 16:58:15 +0100 Subject: [PATCH 016/403] Separate mcstas loader. --- packages/essnmx/docs/examples/workflow.ipynb | 26 +++--- packages/essnmx/src/ess/nmx/loader.py | 92 ------------------- packages/essnmx/src/ess/nmx/mcstas_loader.py | 96 ++++++++++++++++++++ packages/essnmx/src/ess/nmx/reduction.py | 21 ----- packages/essnmx/src/ess/nmx/workflow.py | 4 - packages/essnmx/tests/loader_test.py | 28 +++--- packages/essnmx/tests/workflow_test.py | 26 ++---- 7 files changed, 134 insertions(+), 159 deletions(-) delete mode 100644 packages/essnmx/src/ess/nmx/loader.py create mode 100644 packages/essnmx/src/ess/nmx/mcstas_loader.py delete mode 100644 packages/essnmx/src/ess/nmx/reduction.py diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index b4452d8a..5712f828 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -8,9 +8,9 @@ "\n", "## Collect Parameters and Providers\n", "### Simulation(McStas) Data\n", - "There is an extra parameter that can be configured for **McStas** simulation data workflow.
\n", - "It is because ``weights`` are given as probability, not integer number of events in a McStas file.
\n", - "Therefore users can set a scale factor called ``MaximumProabability`` to derive more realistic number of events.
" + "There is a dedicated loader, ``load_mcstas_nmx_file`` for ``McStas`` simulation data workflow.
\n", + "``MaximumProabability`` can be manually provided to the loader derive more realistic number of events.
\n", + "It is because ``weights`` are given as probability, not integer number of events in a McStas file.
" ] }, { @@ -21,19 +21,21 @@ "source": [ "# Collect parameters and providers\n", "import scipp as sc\n", - "from ess.nmx.workflow import collect_default_parameters, providers\n", - "from ess.nmx.loader import (\n", - " InputFileName,\n", + "from ess.nmx.workflow import collect_default_parameters\n", + "from ess.nmx.mcstas_loader import load_mcstas_nmx_file\n", + "from ess.nmx.mcstas_loader import (\n", + " InputFilename,\n", " MaximumProbability,\n", " DefaultMaximumProbability\n", ")\n", "from ess.nmx.data import small_mcstas_sample\n", "\n", - "file_path = small_mcstas_sample() # Replace it with your data file path\n", + "providers = (load_mcstas_nmx_file, )\n", "\n", + "file_path = small_mcstas_sample() # Replace it with your data file path\n", "params = {\n", " **collect_default_parameters(),\n", - " InputFileName: InputFileName(file_path),\n", + " InputFilename: InputFilename(file_path),\n", " # Additional parameters for McStas data handling.\n", " MaximumProbability: DefaultMaximumProbability,\n", "}" @@ -67,10 +69,10 @@ "outputs": [], "source": [ "import sciline as sl\n", - "from ess.nmx.reduction import GroupedByPixelID\n", + "from ess.nmx.mcstas_loader import NMXData\n", "\n", "nmx_pl = sl.Pipeline(list(providers), params=params)\n", - "nmx_workflow = nmx_pl.get(GroupedByPixelID)\n", + "nmx_workflow = nmx_pl.get(NMXData)\n", "nmx_workflow.visualize()" ] }, @@ -88,7 +90,7 @@ "outputs": [], "source": [ "# Event data grouped by detector panel and pixel id.\n", - "da = nmx_workflow.compute(GroupedByPixelID)\n", + "da = nmx_workflow.compute(NMXData)\n", "da" ] } @@ -109,7 +111,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.9.18" } }, "nbformat": 4, diff --git a/packages/essnmx/src/ess/nmx/loader.py b/packages/essnmx/src/ess/nmx/loader.py deleted file mode 100644 index 66603b42..00000000 --- a/packages/essnmx/src/ess/nmx/loader.py +++ /dev/null @@ -1,92 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) -from dataclasses import dataclass -from typing import Iterable, NewType, Optional - -import scipp as sc -import scippnexus as snx - -PixelIDs = NewType("PixelIDs", sc.Variable) -InputFilename = NewType("InputFilename", str) -Events = NewType("Events", sc.DataArray) - -# McStas Configurations -MaximumProbability = NewType("MaximumProbability", int) -DefaultMaximumProbability = MaximumProbability(100_000) - - -def _retrieve_event_list_name(keys: Iterable[str]) -> str: - for key in keys: - if key.startswith("bank01_events_dat_list"): - return key - raise ValueError("Can not find event list name.") - - -def _copy_partial_var( - var: sc.Variable, dim: str, idx: int, unit: str, dtype: Optional[str] = None -) -> sc.Variable: - """Retrieve property from variable.""" - original_var = var[dim, idx].copy() - var = original_var.astype(dtype) if dtype else original_var - var.unit = sc.Unit(unit) - return var - - -def _get_mcstas_pixel_ids() -> PixelIDs: - """pixel IDs for each detector""" - intervals = [(1, 1638401), (2000001, 3638401), (4000001, 5638401)] - ids = [sc.arange('id', start, stop) for start, stop in intervals] - return PixelIDs(sc.concat(ids, 'id')) - - -@dataclass -class NMXData: - """Data class for NMX data""" - - events: Events - all_pixel_ids: PixelIDs - - -def _load_mcstas_nmx_file(file: snx.File, max_prop: MaximumProbability) -> NMXData: - """Load McStas NMX data from h5 file. - - ``max_pro`` is used to scale ``weights`` to derive more realistic number of events. - """ - from functools import partial - - bank_name = _retrieve_event_list_name(file["entry1/data"].keys()) - var: sc.Variable - var = file["entry1/data/" + bank_name]["events"][()].rename_dims({'dim_0': 'event'}) - - copier = partial(_copy_partial_var, var, dim='dim_1') - weights = copier(idx=0, unit='counts') - id_list = copier(idx=4, unit='dimensionless', dtype="int64") - t_list = copier(idx=5, unit='s') - - weights = (max_prop / weights.max()) * weights - - loaded = sc.DataArray(data=weights, coords={'t': t_list, 'id': id_list}) - pixel_ids = _get_mcstas_pixel_ids() - return NMXData(Events(loaded), pixel_ids) - - -def load_nmx_file( - file_name: InputFileName, - max_prop: Optional[MaximumProbability] = None, -) -> NMXData: - """Load NMX data from a file and generate pixel ID coordinate. - - Check an entry path in the file and handle the data accordingly. - Nexus file should have an entry path called ``entry`` and - McStas simulation file should have an entry path called ``entry1``. - - Pixel id coordinate are wrapped together with the data - since they are different for simulation and real data. - """ - with snx.File(file_name) as file: - if "entry1" in file: # McStas file - return _load_mcstas_nmx_file(file, max_prop or DefaultMaximumProbability) - elif 'entry' in file: # Nexus file - raise NotImplementedError("Measurement data loader is not implemented yet.") - else: - raise ValueError(f"Can not load {file_name} with NMX file loader.") diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py new file mode 100644 index 00000000..42b1e2b9 --- /dev/null +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +from typing import Iterable, NewType, Optional + +import scipp as sc +import scippnexus as snx + +from .detector import NumberOfDetectors + +PixelIDs = NewType("PixelIDs", sc.Variable) +InputFilename = NewType("InputFilename", str) +NMXData = NewType("NMXData", sc.DataArray) + +# McStas Configurations +MaximumProbability = NewType("MaximumProbability", int) +DefaultMaximumProbability = MaximumProbability(100_000) + + +def _find_all(name: str, *properties: str) -> bool: + if not properties: + return True + return name.find(properties[0]) != -1 and _find_all(name, *properties[1:]) + + +def _retrieve_event_list_name(keys: Iterable[str]) -> str: + prefix = "bank01_events_dat_list" + mandatory_keys = ('_p', '_x', '_y', '_n', '_id', '_t') + + for key in keys: + if key.startswith(prefix) and _find_all( + key.removeprefix(prefix), *mandatory_keys + ): + return key + + raise ValueError("Can not find event list name.") + + +def _copy_partial_var( + var: sc.Variable, idx: int, unit: Optional[str] = None, dtype: Optional[str] = None +) -> sc.Variable: + """Retrieve property from variable.""" + original_var = var['dim_1', idx].copy() + var = original_var.astype(dtype) if dtype else original_var + if unit: + var.unit = sc.Unit(unit) + return var + + +def _get_mcstas_pixel_ids() -> PixelIDs: + """pixel IDs for each detector""" + intervals = [(1, 1638401), (2000001, 3638401), (4000001, 5638401)] + ids = [sc.arange('id', start, stop, unit=None) for start, stop in intervals] + return PixelIDs(sc.concat(ids, 'id')) + + +def load_mcstas_nmx_file( + file_name: InputFilename, + max_prop: Optional[MaximumProbability] = None, + num_panels: Optional[NumberOfDetectors] = None, +) -> NMXData: + """Load McStas NMX data from h5 file. + + Parameters + ---------- + file: + The file to load. + + max_prop: + The maximum probability to scale the weights. + + num_panels: + The number of detector panels. + The data will be folded into number of panels. + + """ + + prop = max_prop or DefaultMaximumProbability + panels = num_panels or NumberOfDetectors(3) + + with snx.File(file_name) as file: + bank_name = _retrieve_event_list_name(file["entry1/data"].keys()) + var: sc.Variable + var = file["entry1/data/" + bank_name]["events"][()].rename_dims( + {'dim_0': 'event'} + ) + + weights = _copy_partial_var(var, idx=0, unit='counts') + id_list = _copy_partial_var(var, idx=4, dtype='int64') + t_list = _copy_partial_var(var, idx=5, unit='s') + + weights = (prop / weights.max()) * weights + + loaded = sc.DataArray(data=weights, coords={'t': t_list, 'id': id_list}) + grouped = loaded.group(_get_mcstas_pixel_ids()) + + return NMXData(grouped.fold(dim='id', sizes={'panel': panels, 'id': -1})) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py deleted file mode 100644 index 09a9763a..00000000 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ /dev/null @@ -1,21 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) -from typing import NewType - -import scipp as sc - -from .detector import NumberOfDetectors -from .loader import NMXData - -GroupedByPixelID = NewType("GroupedByPixelID", sc.DataArray) - - -def get_grouped_by_pixel_id( - loaded: NMXData, num_panels: NumberOfDetectors -) -> GroupedByPixelID: - """group events by pixel ID""" - grouped = loaded.events.group(loaded.all_pixel_ids) - - return GroupedByPixelID( - grouped.fold(dim='id', sizes={'panel': num_panels, 'id': -1}) - ) diff --git a/packages/essnmx/src/ess/nmx/workflow.py b/packages/essnmx/src/ess/nmx/workflow.py index bdd1634d..f56e732c 100644 --- a/packages/essnmx/src/ess/nmx/workflow.py +++ b/packages/essnmx/src/ess/nmx/workflow.py @@ -1,10 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) from .detector import default_params as detector_default_params -from .loader import load_nmx_file -from .reduction import get_grouped_by_pixel_id - -providers = (load_nmx_file, get_grouped_by_pixel_id) def collect_default_parameters() -> dict: diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index c0e7e545..b919675a 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -7,23 +7,23 @@ def test_file_reader_mcstas() -> None: import scippnexus as snx from ess.nmx.data import small_mcstas_sample - from ess.nmx.loader import DefaultMaximumProbability, InputFileName, load_nmx_file + from ess.nmx.mcstas_loader import ( + DefaultMaximumProbability, + InputFilename, + load_mcstas_nmx_file, + ) - file_path = InputFileName(small_mcstas_sample()) - entry_path = "entry1/data/bank01_events_dat_list_p_x_y_n_id_t" - da = load_nmx_file(file_path).events + file_path = InputFilename(small_mcstas_sample()) + da = load_mcstas_nmx_file(file_path) + entry_path = "entry1/data/bank01_events_dat_list_p_x_y_n_id_t" with snx.File(file_path) as file: - raw_data: sc.Variable = file[entry_path]["events"][()] + raw_data = file[entry_path]["events"][()] + data_length = raw_data.sizes['dim_0'] - weights = raw_data['dim_1', 0].copy() - weights.unit = '1' - expected_data = (DefaultMaximumProbability / weights.max()) * weights + expected_weight_max = sc.scalar(DefaultMaximumProbability, unit='1', dtype=float) assert isinstance(da, sc.DataArray) - assert list(da.values) == list(expected_data.values) - assert list(da.coords['id'].values) == list(raw_data['dim_1', 4].values) - assert list(da.coords['t'].values) == list(raw_data['dim_1', 5].values) - assert da.unit == '1' - assert da.coords['id'].unit == 'dimensionless' - assert da.coords['t'].unit == 's' + assert da.shape == (3, 1280 * 1280) + assert da.bins.size().sum().value == data_length + assert sc.identical(da.data.max(), expected_weight_max) diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index c6c31d29..aec37223 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -8,18 +8,19 @@ @pytest.fixture def mcstas_workflow() -> sl.Pipeline: from ess.nmx.data import small_mcstas_sample - from ess.nmx.loader import ( + from ess.nmx.mcstas_loader import ( DefaultMaximumProbability, - InputFileName, + InputFilename, MaximumProbability, + load_mcstas_nmx_file, ) - from ess.nmx.workflow import collect_default_parameters, providers + from ess.nmx.workflow import collect_default_parameters return sl.Pipeline( - list(providers), + [load_mcstas_nmx_file], params={ **collect_default_parameters(), - InputFileName: small_mcstas_sample(), + InputFilename: small_mcstas_sample(), MaximumProbability: DefaultMaximumProbability, }, ) @@ -27,21 +28,14 @@ def mcstas_workflow() -> sl.Pipeline: def test_pipeline_builder(mcstas_workflow: sl.Pipeline) -> None: from ess.nmx.data import small_mcstas_sample - from ess.nmx.loader import InputFileName + from ess.nmx.mcstas_loader import InputFilename - assert mcstas_workflow.get(InputFileName).compute() == small_mcstas_sample() + assert mcstas_workflow.get(InputFilename).compute() == small_mcstas_sample() def test_pipeline_mcstas_loader(mcstas_workflow: sl.Pipeline) -> None: """Test if the loader graph is complete.""" - from ess.nmx.loader import NMXData + from ess.nmx.mcstas_loader import NMXData nmx_data = mcstas_workflow.compute(NMXData) - assert isinstance(nmx_data.events, sc.DataArray) - - -def test_pipeline_mcstas_grouping(mcstas_workflow: sl.Pipeline) -> None: - """Test if the data reduction graph is complete.""" - from ess.nmx.reduction import GroupedByPixelID - - assert isinstance(mcstas_workflow.compute(GroupedByPixelID), sc.DataArray) + assert isinstance(nmx_data, sc.DataArray) From 056a0caaf79655d4c646f098ee2a48d380fc1da1 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 14 Dec 2023 14:28:07 +0100 Subject: [PATCH 017/403] Update variable/function names and add docs. --- packages/essnmx/docs/api-reference/index.md | 16 +++++++++++++ packages/essnmx/docs/examples/index.md | 9 ++++++++ packages/essnmx/docs/examples/workflow.ipynb | 14 ++++++------ packages/essnmx/docs/index.md | 1 + packages/essnmx/src/ess/nmx/__init__.py | 24 ++++++++++++++++++++ packages/essnmx/src/ess/nmx/mcstas_loader.py | 16 ++++++------- packages/essnmx/tests/loader_test.py | 8 +++---- packages/essnmx/tests/workflow_test.py | 12 +++++----- 8 files changed, 75 insertions(+), 25 deletions(-) create mode 100644 packages/essnmx/docs/examples/index.md diff --git a/packages/essnmx/docs/api-reference/index.md b/packages/essnmx/docs/api-reference/index.md index 7c36e9b8..85d19317 100644 --- a/packages/essnmx/docs/api-reference/index.md +++ b/packages/essnmx/docs/api-reference/index.md @@ -9,6 +9,14 @@ :toctree: ../generated/classes :template: class-template.rst :recursive: + + NumberOfDetectors + NMXData + InputFilepath + PixelIDs + MaximumProbability + DefaultMaximumProbability + ``` ## Top-level functions @@ -17,6 +25,10 @@ .. autosummary:: :toctree: ../generated/functions :recursive: + + small_mcstas_sample, + load_mcstas_nexus, + collect_default_parameters, ``` ## Submodules @@ -26,4 +38,8 @@ :toctree: ../generated/modules :template: module-template.rst :recursive: + + workflow + detector + mcstas_loader ``` diff --git a/packages/essnmx/docs/examples/index.md b/packages/essnmx/docs/examples/index.md new file mode 100644 index 00000000..76ac4ef0 --- /dev/null +++ b/packages/essnmx/docs/examples/index.md @@ -0,0 +1,9 @@ +# Examples + +```{toctree} +--- +maxdepth: 2 +--- + +workflow +``` diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index 5712f828..9f58dcfe 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -8,9 +8,9 @@ "\n", "## Collect Parameters and Providers\n", "### Simulation(McStas) Data\n", - "There is a dedicated loader, ``load_mcstas_nmx_file`` for ``McStas`` simulation data workflow.
\n", - "``MaximumProabability`` can be manually provided to the loader derive more realistic number of events.
\n", - "It is because ``weights`` are given as probability, not integer number of events in a McStas file.
" + "There is a dedicated loader, ``load_mcstas_nexus`` for ``McStas`` simulation data workflow.
\n", + "``MaximumProbability`` can be manually provided to the loader to derive more realistic number of events.
\n", + "It is because ``weights`` are given as probability, not number of events in a McStas file.
" ] }, { @@ -22,20 +22,20 @@ "# Collect parameters and providers\n", "import scipp as sc\n", "from ess.nmx.workflow import collect_default_parameters\n", - "from ess.nmx.mcstas_loader import load_mcstas_nmx_file\n", + "from ess.nmx.mcstas_loader import load_mcstas_nexus\n", "from ess.nmx.mcstas_loader import (\n", - " InputFilename,\n", + " InputFilepath,\n", " MaximumProbability,\n", " DefaultMaximumProbability\n", ")\n", "from ess.nmx.data import small_mcstas_sample\n", "\n", - "providers = (load_mcstas_nmx_file, )\n", + "providers = (load_mcstas_nexus, )\n", "\n", "file_path = small_mcstas_sample() # Replace it with your data file path\n", "params = {\n", " **collect_default_parameters(),\n", - " InputFilename: InputFilename(file_path),\n", + " InputFilepath: InputFilepath(file_path),\n", " # Additional parameters for McStas data handling.\n", " MaximumProbability: DefaultMaximumProbability,\n", "}" diff --git a/packages/essnmx/docs/index.md b/packages/essnmx/docs/index.md index 2127a70b..bc62b358 100644 --- a/packages/essnmx/docs/index.md +++ b/packages/essnmx/docs/index.md @@ -10,6 +10,7 @@ hidden: --- +examples/index api-reference/index developer/index about/index diff --git a/packages/essnmx/src/ess/nmx/__init__.py b/packages/essnmx/src/ess/nmx/__init__.py index 6d115cc9..34e5fdf9 100644 --- a/packages/essnmx/src/ess/nmx/__init__.py +++ b/packages/essnmx/src/ess/nmx/__init__.py @@ -10,3 +10,27 @@ __version__ = "0.0.0" del importlib + +from .data import small_mcstas_sample +from .detector import NumberOfDetectors +from .mcstas_loader import ( + DefaultMaximumProbability, + InputFilepath, + MaximumProbability, + NMXData, + PixelIDs, + load_mcstas_nexus, +) +from .workflow import collect_default_parameters + +__all__ = [ + "small_mcstas_sample", + "NumberOfDetectors", + "load_mcstas_nexus", + "NMXData", + "InputFilepath", + "PixelIDs", + "MaximumProbability", + "DefaultMaximumProbability", + "collect_default_parameters", +] diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 42b1e2b9..bf121175 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -8,7 +8,7 @@ from .detector import NumberOfDetectors PixelIDs = NewType("PixelIDs", sc.Variable) -InputFilename = NewType("InputFilename", str) +InputFilepath = NewType("InputFilepath", str) NMXData = NewType("NMXData", sc.DataArray) # McStas Configurations @@ -38,7 +38,7 @@ def _retrieve_event_list_name(keys: Iterable[str]) -> str: def _copy_partial_var( var: sc.Variable, idx: int, unit: Optional[str] = None, dtype: Optional[str] = None ) -> sc.Variable: - """Retrieve property from variable.""" + """Retrieve a property from a variable.""" original_var = var['dim_1', idx].copy() var = original_var.astype(dtype) if dtype else original_var if unit: @@ -53,17 +53,17 @@ def _get_mcstas_pixel_ids() -> PixelIDs: return PixelIDs(sc.concat(ids, 'id')) -def load_mcstas_nmx_file( - file_name: InputFilename, +def load_mcstas_nexus( + file_path: InputFilepath, max_prop: Optional[MaximumProbability] = None, num_panels: Optional[NumberOfDetectors] = None, ) -> NMXData: - """Load McStas NMX data from h5 file. + """Load McStas simulation result from h5(nexus) file. Parameters ---------- - file: - The file to load. + file_path: + File name to load. max_prop: The maximum probability to scale the weights. @@ -77,7 +77,7 @@ def load_mcstas_nmx_file( prop = max_prop or DefaultMaximumProbability panels = num_panels or NumberOfDetectors(3) - with snx.File(file_name) as file: + with snx.File(file_path) as file: bank_name = _retrieve_event_list_name(file["entry1/data"].keys()) var: sc.Variable var = file["entry1/data/" + bank_name]["events"][()].rename_dims( diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index b919675a..96f09d89 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -9,12 +9,12 @@ def test_file_reader_mcstas() -> None: from ess.nmx.data import small_mcstas_sample from ess.nmx.mcstas_loader import ( DefaultMaximumProbability, - InputFilename, - load_mcstas_nmx_file, + InputFilepath, + load_mcstas_nexus, ) - file_path = InputFilename(small_mcstas_sample()) - da = load_mcstas_nmx_file(file_path) + file_path = InputFilepath(small_mcstas_sample()) + da = load_mcstas_nexus(file_path) entry_path = "entry1/data/bank01_events_dat_list_p_x_y_n_id_t" with snx.File(file_path) as file: diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index aec37223..55d24335 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -10,17 +10,17 @@ def mcstas_workflow() -> sl.Pipeline: from ess.nmx.data import small_mcstas_sample from ess.nmx.mcstas_loader import ( DefaultMaximumProbability, - InputFilename, + InputFilepath, MaximumProbability, - load_mcstas_nmx_file, + load_mcstas_nexus, ) from ess.nmx.workflow import collect_default_parameters return sl.Pipeline( - [load_mcstas_nmx_file], + [load_mcstas_nexus], params={ **collect_default_parameters(), - InputFilename: small_mcstas_sample(), + InputFilepath: small_mcstas_sample(), MaximumProbability: DefaultMaximumProbability, }, ) @@ -28,9 +28,9 @@ def mcstas_workflow() -> sl.Pipeline: def test_pipeline_builder(mcstas_workflow: sl.Pipeline) -> None: from ess.nmx.data import small_mcstas_sample - from ess.nmx.mcstas_loader import InputFilename + from ess.nmx.mcstas_loader import InputFilepath - assert mcstas_workflow.get(InputFilename).compute() == small_mcstas_sample() + assert mcstas_workflow.get(InputFilepath).compute() == small_mcstas_sample() def test_pipeline_mcstas_loader(mcstas_workflow: sl.Pipeline) -> None: From 39f3f2e89474c104b319561691e1dc862d860c54 Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Tue, 19 Dec 2023 11:07:02 +0100 Subject: [PATCH 018/403] Remove unnecessary copies. Co-authored-by: Simon Heybrock <12912489+SimonHeybrock@users.noreply.github.com> --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index bf121175..609d7747 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -40,7 +40,7 @@ def _copy_partial_var( ) -> sc.Variable: """Retrieve a property from a variable.""" original_var = var['dim_1', idx].copy() - var = original_var.astype(dtype) if dtype else original_var + var = original_var.astype(dtype, copy=False) if dtype else original_var if unit: var.unit = sc.Unit(unit) return var From 2c4e3143786477ee041e4ab0060f7ab8199960ab Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 19 Dec 2023 13:01:27 +0100 Subject: [PATCH 019/403] Remove number of detector option for loading and check fixed-ordered event data name fields. --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 30 +++++----------- packages/essnmx/tests/loader_test.py | 38 +++++++++++++++++++- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 609d7747..720e426e 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -5,8 +5,6 @@ import scipp as sc import scippnexus as snx -from .detector import NumberOfDetectors - PixelIDs = NewType("PixelIDs", sc.Variable) InputFilepath = NewType("InputFilepath", str) NMXData = NewType("NMXData", sc.DataArray) @@ -16,20 +14,14 @@ DefaultMaximumProbability = MaximumProbability(100_000) -def _find_all(name: str, *properties: str) -> bool: - if not properties: - return True - return name.find(properties[0]) != -1 and _find_all(name, *properties[1:]) - - def _retrieve_event_list_name(keys: Iterable[str]) -> str: prefix = "bank01_events_dat_list" - mandatory_keys = ('_p', '_x', '_y', '_n', '_id', '_t') + + # (weight, x, y, n, pixel id, time of arrival) + mandatory_fields = 'p_x_y_n_id_t' for key in keys: - if key.startswith(prefix) and _find_all( - key.removeprefix(prefix), *mandatory_keys - ): + if key.startswith(prefix) and mandatory_fields in key: return key raise ValueError("Can not find event list name.") @@ -56,7 +48,6 @@ def _get_mcstas_pixel_ids() -> PixelIDs: def load_mcstas_nexus( file_path: InputFilepath, max_prop: Optional[MaximumProbability] = None, - num_panels: Optional[NumberOfDetectors] = None, ) -> NMXData: """Load McStas simulation result from h5(nexus) file. @@ -68,14 +59,9 @@ def load_mcstas_nexus( max_prop: The maximum probability to scale the weights. - num_panels: - The number of detector panels. - The data will be folded into number of panels. - """ prop = max_prop or DefaultMaximumProbability - panels = num_panels or NumberOfDetectors(3) with snx.File(file_path) as file: bank_name = _retrieve_event_list_name(file["entry1/data"].keys()) @@ -84,13 +70,13 @@ def load_mcstas_nexus( {'dim_0': 'event'} ) - weights = _copy_partial_var(var, idx=0, unit='counts') - id_list = _copy_partial_var(var, idx=4, dtype='int64') - t_list = _copy_partial_var(var, idx=5, unit='s') + weights = _copy_partial_var(var, idx=0, unit='counts') # p + id_list = _copy_partial_var(var, idx=4, dtype='int64') # id + t_list = _copy_partial_var(var, idx=5, unit='s') # t weights = (prop / weights.max()) * weights loaded = sc.DataArray(data=weights, coords={'t': t_list, 'id': id_list}) grouped = loaded.group(_get_mcstas_pixel_ids()) - return NMXData(grouped.fold(dim='id', sizes={'panel': panels, 'id': -1})) + return NMXData(grouped.fold(dim='id', sizes={'panel': 3, 'id': -1})) diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 96f09d89..cef49f20 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -1,12 +1,17 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +import pathlib +from typing import Generator + +import pytest import scipp as sc +from ess.nmx.data import small_mcstas_sample + def test_file_reader_mcstas() -> None: import scippnexus as snx - from ess.nmx.data import small_mcstas_sample from ess.nmx.mcstas_loader import ( DefaultMaximumProbability, InputFilepath, @@ -27,3 +32,34 @@ def test_file_reader_mcstas() -> None: assert da.shape == (3, 1280 * 1280) assert da.bins.size().sum().value == data_length assert sc.identical(da.data.max(), expected_weight_max) + + +@pytest.fixture +def tmp_mcstas_file(tmp_path: pathlib.Path) -> Generator[pathlib.Path, None, None]: + import os + import shutil + + tmp_file = tmp_path / pathlib.Path('file.h5') + shutil.copy(small_mcstas_sample(), tmp_file) + yield tmp_file + os.remove(tmp_file) + + +def test_file_reader_mcstas_additional_fields(tmp_mcstas_file: pathlib.Path) -> None: + """Check if additional fields names do not break the loader.""" + import h5py + + from ess.nmx.mcstas_loader import InputFilepath, load_mcstas_nexus + + entry_path = "entry1/data/bank01_events_dat_list_p_x_y_n_id_t" + new_entry_path = entry_path + '_L' + + with h5py.File(tmp_mcstas_file, 'r+') as file: + dataset = file[entry_path] + del file[entry_path] + file[new_entry_path] = dataset + + da = load_mcstas_nexus(InputFilepath(str(tmp_mcstas_file))) + + assert isinstance(da, sc.DataArray) + assert da.shape == (3, 1280 * 1280) From b51e71dad1de0f2d49a922ae8e7308857a284390 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 19 Dec 2023 13:01:53 +0100 Subject: [PATCH 020/403] Remove unnecessary classes from root. --- packages/essnmx/docs/api-reference/index.md | 4 ---- packages/essnmx/src/ess/nmx/__init__.py | 16 ++-------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/packages/essnmx/docs/api-reference/index.md b/packages/essnmx/docs/api-reference/index.md index 85d19317..ec943c25 100644 --- a/packages/essnmx/docs/api-reference/index.md +++ b/packages/essnmx/docs/api-reference/index.md @@ -10,12 +10,8 @@ :template: class-template.rst :recursive: - NumberOfDetectors NMXData InputFilepath - PixelIDs - MaximumProbability - DefaultMaximumProbability ``` diff --git a/packages/essnmx/src/ess/nmx/__init__.py b/packages/essnmx/src/ess/nmx/__init__.py index 34e5fdf9..a357ac27 100644 --- a/packages/essnmx/src/ess/nmx/__init__.py +++ b/packages/essnmx/src/ess/nmx/__init__.py @@ -12,25 +12,13 @@ del importlib from .data import small_mcstas_sample -from .detector import NumberOfDetectors -from .mcstas_loader import ( - DefaultMaximumProbability, - InputFilepath, - MaximumProbability, - NMXData, - PixelIDs, - load_mcstas_nexus, -) +from .mcstas_loader import InputFilepath, NMXData, load_mcstas_nexus from .workflow import collect_default_parameters __all__ = [ "small_mcstas_sample", - "NumberOfDetectors", - "load_mcstas_nexus", "NMXData", "InputFilepath", - "PixelIDs", - "MaximumProbability", - "DefaultMaximumProbability", + "load_mcstas_nexus", "collect_default_parameters", ] From 6ccb847da890a7c185e9c59c3d6e72e4f2fe81d4 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 20 Dec 2023 13:18:50 +0100 Subject: [PATCH 021/403] Remove unused modules. --- packages/essnmx/docs/api-reference/index.md | 10 +++++----- packages/essnmx/docs/examples/workflow.ipynb | 2 -- packages/essnmx/src/ess/nmx/__init__.py | 2 -- packages/essnmx/src/ess/nmx/detector.py | 16 ---------------- packages/essnmx/src/ess/nmx/workflow.py | 7 ------- 5 files changed, 5 insertions(+), 32 deletions(-) delete mode 100644 packages/essnmx/src/ess/nmx/detector.py delete mode 100644 packages/essnmx/src/ess/nmx/workflow.py diff --git a/packages/essnmx/docs/api-reference/index.md b/packages/essnmx/docs/api-reference/index.md index ec943c25..cd74eadf 100644 --- a/packages/essnmx/docs/api-reference/index.md +++ b/packages/essnmx/docs/api-reference/index.md @@ -22,9 +22,9 @@ :toctree: ../generated/functions :recursive: - small_mcstas_sample, - load_mcstas_nexus, - collect_default_parameters, + small_mcstas_sample + load_mcstas_nexus + ``` ## Submodules @@ -35,7 +35,7 @@ :template: module-template.rst :recursive: - workflow - detector + data mcstas_loader + ``` diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index 9f58dcfe..5a0551c0 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -21,7 +21,6 @@ "source": [ "# Collect parameters and providers\n", "import scipp as sc\n", - "from ess.nmx.workflow import collect_default_parameters\n", "from ess.nmx.mcstas_loader import load_mcstas_nexus\n", "from ess.nmx.mcstas_loader import (\n", " InputFilepath,\n", @@ -34,7 +33,6 @@ "\n", "file_path = small_mcstas_sample() # Replace it with your data file path\n", "params = {\n", - " **collect_default_parameters(),\n", " InputFilepath: InputFilepath(file_path),\n", " # Additional parameters for McStas data handling.\n", " MaximumProbability: DefaultMaximumProbability,\n", diff --git a/packages/essnmx/src/ess/nmx/__init__.py b/packages/essnmx/src/ess/nmx/__init__.py index a357ac27..af30da5d 100644 --- a/packages/essnmx/src/ess/nmx/__init__.py +++ b/packages/essnmx/src/ess/nmx/__init__.py @@ -13,12 +13,10 @@ from .data import small_mcstas_sample from .mcstas_loader import InputFilepath, NMXData, load_mcstas_nexus -from .workflow import collect_default_parameters __all__ = [ "small_mcstas_sample", "NMXData", "InputFilepath", "load_mcstas_nexus", - "collect_default_parameters", ] diff --git a/packages/essnmx/src/ess/nmx/detector.py b/packages/essnmx/src/ess/nmx/detector.py deleted file mode 100644 index f2e79164..00000000 --- a/packages/essnmx/src/ess/nmx/detector.py +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) -from typing import NewType - -MaxNumberOfPixelsPerAxis = NewType("MaxNumberOfPixelsPerAxis", int) -PixelStep = NewType("PixelStep", int) -NumberOfDetectors = NewType("NumberOfDetectors", int) -NumberOfAxes = NewType("NumberOfAxes", int) - - -default_params = { - MaxNumberOfPixelsPerAxis: MaxNumberOfPixelsPerAxis(1280), - NumberOfAxes: NumberOfAxes(2), - PixelStep: PixelStep(1), - NumberOfDetectors: NumberOfDetectors(3), -} diff --git a/packages/essnmx/src/ess/nmx/workflow.py b/packages/essnmx/src/ess/nmx/workflow.py deleted file mode 100644 index f56e732c..00000000 --- a/packages/essnmx/src/ess/nmx/workflow.py +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) -from .detector import default_params as detector_default_params - - -def collect_default_parameters() -> dict: - return dict(detector_default_params) From 95a8c3920bf6df04d5b2edde1638b7ebaba34fbc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Dec 2023 17:54:32 +0000 Subject: [PATCH 022/403] Bump scipp from 23.11.1 to 23.12.0 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 23.11.1 to 23.12.0. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/23.11.1...23.12.0) --- updated-dependencies: - dependency-name: scipp dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 82c87812..e98e8c1b 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -32,9 +32,7 @@ fonttools==4.46.0 fsspec==2023.12.2 # via dask graphlib-backport==1.0.3 - # via - # sciline - # scipp + # via sciline graphviz==0.20.1 # via -r base.in h5py==3.10.0 @@ -105,7 +103,7 @@ requests==2.31.0 # via pooch sciline==23.12.0 # via -r base.in -scipp==23.11.1 +scipp==23.12.0 # via # -r base.in # scippnexus From 66523cba8b4a0af4bfbad00e578e326d8f6a9d4e Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Thu, 21 Dec 2023 09:16:02 +0100 Subject: [PATCH 023/403] More compact variable copy and type change. Co-authored-by: Simon Heybrock <12912489+SimonHeybrock@users.noreply.github.com> --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 720e426e..f1fdf0a8 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -31,8 +31,7 @@ def _copy_partial_var( var: sc.Variable, idx: int, unit: Optional[str] = None, dtype: Optional[str] = None ) -> sc.Variable: """Retrieve a property from a variable.""" - original_var = var['dim_1', idx].copy() - var = original_var.astype(dtype, copy=False) if dtype else original_var + var = var['dim_1', idx].astype(dtype or var.dtype , copy=True) if unit: var.unit = sc.Unit(unit) return var From 9e05103bcc9f4589c8f2674fe9d4a7fa4793804e Mon Sep 17 00:00:00 2001 From: YooSunYoung Date: Thu, 21 Dec 2023 08:17:02 +0000 Subject: [PATCH 024/403] Apply automatic formatting --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index f1fdf0a8..c0e96309 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -31,7 +31,7 @@ def _copy_partial_var( var: sc.Variable, idx: int, unit: Optional[str] = None, dtype: Optional[str] = None ) -> sc.Variable: """Retrieve a property from a variable.""" - var = var['dim_1', idx].astype(dtype or var.dtype , copy=True) + var = var['dim_1', idx].astype(dtype or var.dtype, copy=True) if unit: var.unit = sc.Unit(unit) return var From 70f77d7a95ad7e6f3438abff70dc129a5608b9a1 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 21 Dec 2023 09:18:11 +0100 Subject: [PATCH 025/403] Remove unused package. --- packages/essnmx/tests/workflow_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index 55d24335..7f00b28e 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -14,12 +14,10 @@ def mcstas_workflow() -> sl.Pipeline: MaximumProbability, load_mcstas_nexus, ) - from ess.nmx.workflow import collect_default_parameters return sl.Pipeline( [load_mcstas_nexus], params={ - **collect_default_parameters(), InputFilepath: small_mcstas_sample(), MaximumProbability: DefaultMaximumProbability, }, From 4e5286f3b5e12c596db8b1473e990ea26b441b1e Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 21 Dec 2023 09:25:37 +0100 Subject: [PATCH 026/403] Use more specific names for arguments. --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index c0e96309..90855426 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -46,7 +46,7 @@ def _get_mcstas_pixel_ids() -> PixelIDs: def load_mcstas_nexus( file_path: InputFilepath, - max_prop: Optional[MaximumProbability] = None, + max_probability: Optional[MaximumProbability] = None, ) -> NMXData: """Load McStas simulation result from h5(nexus) file. @@ -55,12 +55,12 @@ def load_mcstas_nexus( file_path: File name to load. - max_prop: + max_probability: The maximum probability to scale the weights. """ - prop = max_prop or DefaultMaximumProbability + probability = max_probability or DefaultMaximumProbability with snx.File(file_path) as file: bank_name = _retrieve_event_list_name(file["entry1/data"].keys()) @@ -73,7 +73,7 @@ def load_mcstas_nexus( id_list = _copy_partial_var(var, idx=4, dtype='int64') # id t_list = _copy_partial_var(var, idx=5, unit='s') # t - weights = (prop / weights.max()) * weights + weights = (probability / weights.max()) * weights loaded = sc.DataArray(data=weights, coords={'t': t_list, 'id': id_list}) grouped = loaded.group(_get_mcstas_pixel_ids()) From 17c6c292ea50be5295fb6dbf9d0af304fd94d2ea Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 21 Dec 2023 18:13:21 +0100 Subject: [PATCH 027/403] Update dependencies. --- packages/essnmx/pyproject.toml | 1 + packages/essnmx/requirements/base.in | 1 + packages/essnmx/requirements/base.txt | 16 ++++++++++---- packages/essnmx/requirements/docs.in | 1 + packages/essnmx/requirements/docs.txt | 32 +++++++++++++++++++-------- 5 files changed, 38 insertions(+), 13 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index dcca7582..79092497 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "scipp>=23.8.0", "scippnexus>=23.9.0", "pooch", + "defusedxml", ] dynamic = ["version"] diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index a0270738..ded1646c 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -12,3 +12,4 @@ sciline>=23.9.1 scipp>=23.8.0 scippnexus>=23.9.0 pooch +defusedxml diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index e98e8c1b..b3843f08 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,10 +1,12 @@ -# SHA1:dc8a3b216315f3203ec4048ef1930f26643a833e +# SHA1:f2f02404509e42e072ec3a85641b6b2fe68d380a # # This file is autogenerated by pip-compile-multi # To update, run: # # pip-compile-multi # +appnope==0.1.3 + # via ipython asttokens==2.4.1 # via stack-data backcall==0.2.0 @@ -21,13 +23,15 @@ contourpy==1.2.0 # via matplotlib cycler==0.12.1 # via matplotlib -dask==2023.12.0 +dask==2023.12.1 # via -r base.in decorator==5.1.1 # via ipython +defusedxml==0.7.1 + # via -r base.in executing==2.0.1 # via stack-data -fonttools==4.46.0 +fonttools==4.47.0 # via matplotlib fsspec==2023.12.2 # via dask @@ -41,6 +45,8 @@ idna==3.6 # via requests importlib-metadata==7.0.0 # via dask +importlib-resources==6.1.1 + # via matplotlib ipython==8.9.0 # via -r base.in jedi==0.19.1 @@ -130,4 +136,6 @@ urllib3==2.1.0 wcwidth==0.2.12 # via prompt-toolkit zipp==3.17.0 - # via importlib-metadata + # via + # importlib-metadata + # importlib-resources diff --git a/packages/essnmx/requirements/docs.in b/packages/essnmx/requirements/docs.in index e542e53b..fb5e3a0b 100644 --- a/packages/essnmx/requirements/docs.in +++ b/packages/essnmx/requirements/docs.in @@ -8,3 +8,4 @@ sphinx sphinx-autodoc-typehints sphinx-copybutton sphinx-design +pythreejs # For instrument view. diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index b1613253..1d0751c8 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -1,4 +1,4 @@ -# SHA1:2175813590b5d31dc1cdf3e3c820f699647e9043 +# SHA1:6d4b9a85dde4357f502d5cfae2ada4870d83bd3d # # This file is autogenerated by pip-compile-multi # To update, run: @@ -14,7 +14,7 @@ attrs==23.1.0 # via # jsonschema # referencing -babel==2.13.1 +babel==2.14.0 # via # pydata-sphinx-theme # sphinx @@ -25,11 +25,11 @@ beautifulsoup4==4.12.2 bleach==6.1.0 # via nbconvert comm==0.2.0 - # via ipykernel + # via + # ipykernel + # ipywidgets debugpy==1.8.0 # via ipykernel -defusedxml==0.7.1 - # via nbconvert docutils==0.20.1 # via # myst-parser @@ -40,8 +40,14 @@ fastjsonschema==2.19.0 # via nbformat imagesize==1.4.1 # via sphinx +ipydatawidgets==4.3.5 + # via pythreejs ipykernel==6.27.1 # via -r docs.in +ipywidgets==8.1.1 + # via + # ipydatawidgets + # pythreejs jinja2==3.1.2 # via # myst-parser @@ -56,7 +62,7 @@ jupyter-client==8.6.0 # via # ipykernel # nbclient -jupyter-core==5.5.0 +jupyter-core==5.5.1 # via # ipykernel # jupyter-client @@ -65,6 +71,8 @@ jupyter-core==5.5.0 # nbformat jupyterlab-pygments==0.3.0 # via nbconvert +jupyterlab-widgets==3.0.9 + # via ipywidgets markdown-it-py==3.0.0 # via # mdit-py-plugins @@ -83,7 +91,7 @@ myst-parser==2.0.0 # via -r docs.in nbclient==0.9.0 # via nbconvert -nbconvert==7.12.0 +nbconvert==7.13.0 # via nbsphinx nbformat==5.9.2 # via @@ -96,10 +104,12 @@ nest-asyncio==1.5.8 # via ipykernel pandocfilters==1.5.0 # via nbconvert -psutil==5.9.6 +psutil==5.9.7 # via ipykernel pydata-sphinx-theme==0.14.4 # via -r docs.in +pythreejs==2.4.2 + # via -r docs.in pyzmq==25.1.2 # via # ipykernel @@ -108,7 +118,7 @@ referencing==0.32.0 # via # jsonschema # jsonschema-specifications -rpds-py==0.13.2 +rpds-py==0.15.2 # via # jsonschema # referencing @@ -154,9 +164,13 @@ tornado==6.4 # via # ipykernel # jupyter-client +traittypes==0.2.1 + # via ipydatawidgets typing-extensions==4.9.0 # via pydata-sphinx-theme webencodings==0.5.1 # via # bleach # tinycss2 +widgetsnbextension==4.0.9 + # via ipywidgets From 3a73f4c709b4e6ef92ac40a7d6ebc288ebf0c5a5 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Fri, 22 Dec 2023 09:30:23 +0100 Subject: [PATCH 028/403] Add scippneutron for docs build. --- packages/essnmx/requirements/docs.in | 1 + packages/essnmx/requirements/docs.txt | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/requirements/docs.in b/packages/essnmx/requirements/docs.in index fb5e3a0b..cc0c975f 100644 --- a/packages/essnmx/requirements/docs.in +++ b/packages/essnmx/requirements/docs.in @@ -9,3 +9,4 @@ sphinx-autodoc-typehints sphinx-copybutton sphinx-design pythreejs # For instrument view. +scippneutron diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 1d0751c8..406b920d 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -1,4 +1,4 @@ -# SHA1:6d4b9a85dde4357f502d5cfae2ada4870d83bd3d +# SHA1:a4b21b1181ffe0d85627ecf73a8b997699993f2a # # This file is autogenerated by pip-compile-multi # To update, run: @@ -91,7 +91,7 @@ myst-parser==2.0.0 # via -r docs.in nbclient==0.9.0 # via nbconvert -nbconvert==7.13.0 +nbconvert==7.13.1 # via nbsphinx nbformat==5.9.2 # via @@ -122,6 +122,8 @@ rpds-py==0.15.2 # via # jsonschema # referencing +scippneutron==23.11.0 + # via -r docs.in snowballstemmer==2.2.0 # via sphinx soupsieve==2.5 From 0c0a402b353aa6ee1d99af6af6199dc2ff065c31 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 4 Jan 2024 09:28:37 +0100 Subject: [PATCH 029/403] Update dependency recipes. --- packages/essnmx/requirements/base.txt | 14 +++++--------- packages/essnmx/requirements/basetest.txt | 2 +- packages/essnmx/requirements/dev.txt | 14 +++++++------- packages/essnmx/requirements/docs.txt | 16 ++++++++-------- packages/essnmx/requirements/mypy.txt | 2 +- packages/essnmx/requirements/nightly.in | 1 + packages/essnmx/requirements/nightly.txt | 18 ++++++++---------- packages/essnmx/requirements/wheels.txt | 2 +- 8 files changed, 32 insertions(+), 37 deletions(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index b3843f08..fb39acdd 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -5,8 +5,6 @@ # # pip-compile-multi # -appnope==0.1.3 - # via ipython asttokens==2.4.1 # via stack-data backcall==0.2.0 @@ -35,15 +33,13 @@ fonttools==4.47.0 # via matplotlib fsspec==2023.12.2 # via dask -graphlib-backport==1.0.3 - # via sciline graphviz==0.20.1 # via -r base.in h5py==3.10.0 # via scippnexus idna==3.6 # via requests -importlib-metadata==7.0.0 +importlib-metadata==7.0.1 # via dask importlib-resources==6.1.1 # via matplotlib @@ -59,7 +55,7 @@ matplotlib==3.8.2 # via plopp matplotlib-inline==0.1.6 # via ipython -numpy==1.26.2 +numpy==1.26.3 # via # contourpy # h5py @@ -79,7 +75,7 @@ pexpect==4.9.0 # via ipython pickleshare==0.7.5 # via ipython -pillow==10.1.0 +pillow==10.2.0 # via matplotlib platformdirs==4.1.0 # via pooch @@ -107,7 +103,7 @@ pyyaml==6.0.1 # via dask requests==2.31.0 # via pooch -sciline==23.12.0 +sciline==24.1.0 # via -r base.in scipp==23.12.0 # via @@ -127,7 +123,7 @@ toolz==0.12.0 # via # dask # partd -traitlets==5.14.0 +traitlets==5.14.1 # via # ipython # matplotlib-inline diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt index 6bf43fa2..c2562f8a 100644 --- a/packages/essnmx/requirements/basetest.txt +++ b/packages/essnmx/requirements/basetest.txt @@ -13,7 +13,7 @@ packaging==23.2 # via pytest pluggy==1.3.0 # via pytest -pytest==7.4.3 +pytest==7.4.4 # via -r basetest.in tomli==2.0.1 # via pytest diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index 5ab837c3..b82a08bc 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -14,7 +14,7 @@ -r wheels.txt annotated-types==0.6.0 # via pydantic -anyio==4.1.0 +anyio==4.2.0 # via jupyter-server argon2-cffi==23.1.0 # via jupyter-server @@ -51,15 +51,15 @@ jupyter-events==0.9.0 # via jupyter-server jupyter-lsp==2.2.1 # via jupyterlab -jupyter-server==2.12.1 +jupyter-server==2.12.2 # via # jupyter-lsp # jupyterlab # jupyterlab-server # notebook-shim -jupyter-server-terminals==0.5.0 +jupyter-server-terminals==0.5.1 # via jupyter-server -jupyterlab==4.0.9 +jupyterlab==4.0.10 # via -r dev.in jupyterlab-server==2.25.2 # via jupyterlab @@ -79,13 +79,13 @@ prometheus-client==0.19.0 # via jupyter-server pycparser==2.21 # via cffi -pydantic==2.5.2 +pydantic==2.5.3 # via copier -pydantic-core==2.14.5 +pydantic-core==2.14.6 # via pydantic python-json-logger==2.0.7 # via jupyter-events -pyyaml-include==1.3.1 +pyyaml-include==1.3.2 # via copier questionary==2.0.1 # via copier diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 406b920d..d9dc6ed7 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -10,7 +10,7 @@ accessible-pygments==0.0.4 # via pydata-sphinx-theme alabaster==0.7.13 # via sphinx -attrs==23.1.0 +attrs==23.2.0 # via # jsonschema # referencing @@ -24,7 +24,7 @@ beautifulsoup4==4.12.2 # pydata-sphinx-theme bleach==6.1.0 # via nbconvert -comm==0.2.0 +comm==0.2.1 # via # ipykernel # ipywidgets @@ -36,13 +36,13 @@ docutils==0.20.1 # nbsphinx # pydata-sphinx-theme # sphinx -fastjsonschema==2.19.0 +fastjsonschema==2.19.1 # via nbformat imagesize==1.4.1 # via sphinx ipydatawidgets==4.3.5 # via pythreejs -ipykernel==6.27.1 +ipykernel==6.28.0 # via -r docs.in ipywidgets==8.1.1 # via @@ -56,13 +56,13 @@ jinja2==3.1.2 # sphinx jsonschema==4.20.0 # via nbformat -jsonschema-specifications==2023.11.2 +jsonschema-specifications==2023.12.1 # via jsonschema jupyter-client==8.6.0 # via # ipykernel # nbclient -jupyter-core==5.5.1 +jupyter-core==5.7.0 # via # ipykernel # jupyter-client @@ -91,7 +91,7 @@ myst-parser==2.0.0 # via -r docs.in nbclient==0.9.0 # via nbconvert -nbconvert==7.13.1 +nbconvert==7.14.0 # via nbsphinx nbformat==5.9.2 # via @@ -118,7 +118,7 @@ referencing==0.32.0 # via # jsonschema # jsonschema-specifications -rpds-py==0.15.2 +rpds-py==0.16.2 # via # jsonschema # referencing diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index 99e44980..ac285686 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -6,7 +6,7 @@ # pip-compile-multi # -r test.txt -mypy==1.7.1 +mypy==1.8.0 # via -r mypy.in mypy-extensions==1.0.0 # via mypy diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index 3b7a8998..01325973 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -4,6 +4,7 @@ dask graphviz pooch +defusedxml https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl sciline @ git+https://github.com/scipp/sciline@main scippnexus @ git+https://github.com/scipp/scippnexus@main diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 000d7ffc..3b124cfd 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:a82ae6ed279eacc614c956cfe2f650068f6d21ca +# SHA1:15630cfbae70da78bb40bd86bc6b9d0737fd1dd1 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -18,23 +18,21 @@ contourpy==1.2.0 # via matplotlib cycler==0.12.1 # via matplotlib -dask==2023.12.0 +dask==2023.12.1 # via -r nightly.in -fonttools==4.46.0 +defusedxml==0.7.1 + # via -r nightly.in +fonttools==4.47.0 # via matplotlib fsspec==2023.12.2 # via dask -graphlib-backport==1.0.3 - # via - # sciline - # scipp graphviz==0.20.1 # via -r nightly.in h5py==3.10.0 # via scippnexus idna==3.6 # via requests -importlib-metadata==7.0.0 +importlib-metadata==7.0.1 # via dask importlib-resources==6.1.1 # via matplotlib @@ -44,7 +42,7 @@ locket==1.0.0 # via partd matplotlib==3.8.2 # via plopp -numpy==1.26.2 +numpy==1.26.3 # via # contourpy # h5py @@ -53,7 +51,7 @@ numpy==1.26.2 # scipy partd==1.4.1 # via dask -pillow==10.1.0 +pillow==10.2.0 # via matplotlib platformdirs==4.1.0 # via pooch diff --git a/packages/essnmx/requirements/wheels.txt b/packages/essnmx/requirements/wheels.txt index d5378cf7..c26530af 100644 --- a/packages/essnmx/requirements/wheels.txt +++ b/packages/essnmx/requirements/wheels.txt @@ -7,7 +7,7 @@ # build==1.0.3 # via -r wheels.in -importlib-metadata==7.0.0 +importlib-metadata==7.0.1 # via build packaging==23.2 # via build From 2751082046ce10f1c06fc060c81c47c1f8df7c6b Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Thu, 4 Jan 2024 09:36:48 +0100 Subject: [PATCH 030/403] Use common prettyname structure --- packages/essnmx/.copier-answers.yml | 4 ++-- packages/essnmx/CONTRIBUTING.md | 8 ++++---- packages/essnmx/README.md | 2 +- packages/essnmx/conda/meta.yaml | 5 +++-- packages/essnmx/docs/about/index.md | 10 +++++----- packages/essnmx/docs/conf.py | 4 ++-- packages/essnmx/docs/index.md | 2 +- 7 files changed, 18 insertions(+), 17 deletions(-) diff --git a/packages/essnmx/.copier-answers.yml b/packages/essnmx/.copier-answers.yml index b8eabc0f..4a0801f7 100644 --- a/packages/essnmx/.copier-answers.yml +++ b/packages/essnmx/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY -_commit: ec09f73 +_commit: 8cfdb1b _src_path: gh:scipp/copier_template description: Data reduction for NMX at the European Spallation Source. max_python: '3.11' @@ -7,7 +7,7 @@ min_python: '3.9' namespace_package: ess nightly_deps: scipp,sciline,scippnexus,plopp orgname: scipp -prettyname: Essnmx +prettyname: ESSnmx projectname: essnmx related_projects: Scipp,Sciline,Plopp,ScippNexus year: 2023 diff --git a/packages/essnmx/CONTRIBUTING.md b/packages/essnmx/CONTRIBUTING.md index bcd9e616..753fb793 100644 --- a/packages/essnmx/CONTRIBUTING.md +++ b/packages/essnmx/CONTRIBUTING.md @@ -1,18 +1,18 @@ -## Contributing to Essnmx +## Contributing to ESSnmx -Welcome to the developer side of Essnmx! +Welcome to the developer side of ESSnmx! Contributions are always welcome. This includes reporting bugs or other issues, submitting pull requests, requesting new features, etc. -If you need help with using Essnmx or contributing to it, have a look at the GitHub [discussions](https://github.com/scipp/essnmx/discussions) and start a new [Q&A discussion](https://github.com/scipp/essnmx/discussions/categories/q-a) if you can't find what you are looking for. +If you need help with using ESSnmx or contributing to it, have a look at the GitHub [discussions](https://github.com/scipp/essnmx/discussions) and start a new [Q&A discussion](https://github.com/scipp/essnmx/discussions/categories/q-a) if you can't find what you are looking for. For bug reports and other problems, please open an [issue](https://github.com/scipp/essnmx/issues/new) in GitHub. You are welcome to submit pull requests at any time. But to avoid having to make large modifications during review or even have your PR rejected, please first open an issue first to discuss your idea! -Check out the subsections of the [Developer documentation](https://scipp.github.io/essnmx/developer/index.html) for details on how Essnmx is developed. +Check out the subsections of the [Developer documentation](https://scipp.github.io/essnmx/developer/index.html) for details on how ESSnmx is developed. ## Code of conduct diff --git a/packages/essnmx/README.md b/packages/essnmx/README.md index 485cb08f..85269345 100644 --- a/packages/essnmx/README.md +++ b/packages/essnmx/README.md @@ -3,7 +3,7 @@ [![Anaconda-Server Badge](https://anaconda.org/scipp/essnmx/badges/version.svg)](https://anaconda.org/scipp/essnmx) [![License: BSD 3-Clause](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](LICENSE) -# Essnmx +# ESSnmx ## About diff --git a/packages/essnmx/conda/meta.yaml b/packages/essnmx/conda/meta.yaml index b945859d..ab9ae253 100644 --- a/packages/essnmx/conda/meta.yaml +++ b/packages/essnmx/conda/meta.yaml @@ -22,14 +22,15 @@ requirements: test: imports: - - essnmx + - ess.nmx requires: - pytest source_files: - pyproject.toml - tests/ commands: - - python -m pytest tests + # We ignore warnings during release package builds + - python -m pytest -Wignore tests build: noarch: python diff --git a/packages/essnmx/docs/about/index.md b/packages/essnmx/docs/about/index.md index b697ffaa..22ab8121 100644 --- a/packages/essnmx/docs/about/index.md +++ b/packages/essnmx/docs/about/index.md @@ -2,19 +2,19 @@ ## Development -Essnmx is an open source project by the [European Spallation Source ERIC](https://europeanspallationsource.se/) (ESS). +ESSnmx is an open source project by the [European Spallation Source ERIC](https://europeanspallationsource.se/) (ESS). ## License -Essnmx is available as open source under the [BSD-3 license](https://opensource.org/licenses/BSD-3-Clause). +ESSnmx is available as open source under the [BSD-3 license](https://opensource.org/licenses/BSD-3-Clause). -## Citing Essnmx +## Citing ESSnmx Please cite the following: [![DOI](https://zenodo.org/badge/FIXME.svg)](https://zenodo.org/doi/10.5281/zenodo.FIXME) -To cite a specific version of Essnmx, select the desired version on Zenodo to get the corresponding DOI. +To cite a specific version of ESSnmx, select the desired version on Zenodo to get the corresponding DOI. ## Older versions of the documentation @@ -23,4 +23,4 @@ Simply download the archive, unzip and view locally in a web browser. ## Source code and development -Essnmx is hosted and developed [on GitHub](https://github.com/scipp/essnmx). +ESSnmx is hosted and developed [on GitHub](https://github.com/scipp/essnmx). diff --git a/packages/essnmx/docs/conf.py b/packages/essnmx/docs/conf.py index 01842ec5..276bf231 100644 --- a/packages/essnmx/docs/conf.py +++ b/packages/essnmx/docs/conf.py @@ -9,7 +9,7 @@ sys.path.insert(0, os.path.abspath('.')) # General information about the project. -project = u'Essnmx' +project = u'ESSnmx' copyright = u'2023 Scipp contributors' author = u'Scipp contributors' @@ -167,7 +167,7 @@ "**": ["sidebar-nav-bs", "page-toc"], } -html_title = "Essnmx" +html_title = "ESSnmx" html_logo = "_static/logo.svg" html_favicon = "_static/favicon.ico" diff --git a/packages/essnmx/docs/index.md b/packages/essnmx/docs/index.md index bc62b358..f6214638 100644 --- a/packages/essnmx/docs/index.md +++ b/packages/essnmx/docs/index.md @@ -1,4 +1,4 @@ -# Essnmx +# ESSnmx Data reduction for NMX at the European Spallation Source. From 6dac24eca3fa08473c91391604d06afdda7bbf4e Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 9 Jan 2024 15:58:05 +0100 Subject: [PATCH 031/403] Add static images for published documentation page. --- packages/essnmx/docs/_static/favicon.ico | Bin 0 -> 137750 bytes packages/essnmx/docs/_static/logo-dark.svg | 162 +++++ packages/essnmx/docs/_static/logo-text.svg | 665 +++++++++++++++++++++ packages/essnmx/docs/_static/logo.svg | 166 +++++ 4 files changed, 993 insertions(+) create mode 100644 packages/essnmx/docs/_static/favicon.ico create mode 100644 packages/essnmx/docs/_static/logo-dark.svg create mode 100644 packages/essnmx/docs/_static/logo-text.svg create mode 100644 packages/essnmx/docs/_static/logo.svg diff --git a/packages/essnmx/docs/_static/favicon.ico b/packages/essnmx/docs/_static/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..af3f7a3142ff9087f812789e8ae0c687b469892e GIT binary patch literal 137750 zcmeI5349$@`M;-4X$lll%2uF|LIGJz5s*b7l|}f2pduifP$(!UV%gk&NK!yHl~ooI zEj!)Iz6lbjARr)uY$`Nm5fRX^M~uz?`JS0~dNP?icW!cTdXr}Qd3w&7Im>(A_srR5 z?j#5X1w(_TrT}+Lu)(lF!PkQzm^d*EPy2ZgoX5Ed6Qb)o1;J;2H7FQAKDxeM5F9$6 zlw+dsh#(lc)ZkzQuJSv2ND$0fd2p}+c{Twj-FbQC8#^v=gyH`C|Ewc}VAfHNVYpv} zX80KX0Y8J)W=Zh_N59g*%+Tfm_8*Lmw}GqRU(n&vj{j;H-mfaZ1XpH8WcTYrcN1)T z0q9=$?e6 zcCV*hAFzKPu|M`zA0J@l7{@B;qowZyI{;gpuB-e6z?VXG_ zVE-z$|EzCB_j@?Ex3%0G*gk2)LSfAX9)oG1bzWM$0sB{({k0$G#`w_gTFW)T`~vn% zv#HK$4KNwBCRiG7g|xT>_OBBA-|?+QYdP#MyL|`#T)=K=w$!=#ptb+nHcw`z%%ALmxN`WKsco}RRp`!f6nN?FVOoA_73HlX`FNIuV5O!gmxEZURU z$mTJUZQuwsFq<)6`k;@rwcH%SrL5&XB>r631(Zj@`)o1U|NoF_A-n{qg4#nt8JDDX zfjxqojt&N)k1gQWP|8|PeZzgQFR0Hbs?)_}|Fe+IA8#~|ISxij<|X-<5zgy%-@XjK zj8 z{WZ6{9LDi{SMvnD$J6a&!2aDGaQ}Gi}d|IuG+!pprZXj;@#Rh z$~zRaZcjd!D``|B{eb;Tv;T7F>_M>mP3}8zC2UFi(AZG6{y_6N%MW)M_#)g5ZQ#pb z{sH?Z?1`Sz_J2)ZCGG!Uk2L#RUVo>opTdUInTO#9GkfrRHl=rYgni2d{4+p(j+K|@ z4%mP3+28E(H*9nY)W`R43G;G$9?Kk0eaV)fzD0eESqRtKhXbYYN+!? z^rb#n&pZp+z|PASe}ZqpDsel=ad+NuxV!H^MLP8*uY=`T%zc%(n{}2rJ4E~e`)8}D z)76>ZaC_i-4pPh(cKyG^eFzQ)%^iy##L*c499SNCT*W_-eW1Zb3-BV4X2AZQK8krY zo})9p4|xT=3wdm#IGV3bg%yjHL7e9yZ@vqNdj^cPKBidy{&ms~*xy%34nI26JJEV3 zvOC-gA3+{_wG)rtJ^LVz#*Sk7X>aD|FmBdy;XE<a z2xj-t?vi0?Zs-Cr`n&z3Hc)l;_iU%4QJ)i$f7s{_V3;M9~)_YKNkwwvM1;DJF1Nq zS+}@v!2Z2%|DCXr-r>(<=eLNb`JmRBdG;Wc=Sjt_z9`0k{d?X1`(UGD`;z}8&LyxF ztOOgt58*FRtlgZs$hyUS1NN^f`|BRpyF}abTr1p5j^_3=!hH-qgVuaezmqG}4hHO> zX!hu;dgg!OH~9+P|1Th`+fPjH@!31>T-SQ&NZ(o1iegEc~3>#`6btDwrL(NEA zZu{RtSxY6yeyg8;b1*yn?qgGiOiT3rKPP(6JG$>zK~MS}*83Uy(p>CvSQ}g$NR9^? zY0GVY?E~*UBYM6kd2~c`fA?Mq=QaiH2@K5tLiNP|OJ5OnxQBq=^VBobLb~$L%a-n& zm^Rn8-wI@;FSq?A&+Bk1)PvPWuKK7W;A!yXPWqR)`yYLnp7adl3vegs{m!IriiQ72 znn&Pp&_1o@TPR(3?5}6Mh1P$5IkNNjLG{si|69mf_gmT@$2UX{Pha2vnW6FUNcRr7 z-oKj;?}6!}yRP*78`ukmX5`l$`+u7Jw6B|2Hno+!d%x=I_~m*2qp#b4RyZHtoqB7J zJg-jk#?yN^CuQW@9sBEf{|`ZV=anN*JoR^)=Z?&8{l5SFm-Nise?OQ*hc6Q62QVHshHt=Q zApPahb6T8|?JwKJ^ULt}NBdF!{g7u)kd#5sd)=JX-hVH}_KMr*`~U3U|4Lt4`~3{o zGri-tdEILGzYAIScv9znxLfv@-e>SDXHSk5@gD_GK%TL_g?NpycC^mz*0=wb0Q*aq z=dzAdf9v$EI_ogK-5&n7Q++{q)(hRXe<($#HX!-dgli$I4g8+#TU>2_51`22rIl02 z?dSKu)&kq9Z!B~~eZpGAdkOq|JCDD|_BXlgH|EH&0lZQ`7QOHH73kEKEsyTFMeMJ2 z&>kwUoCo=x35E1q&hI~HomKSvOT^JWp=?=5rb6e-W`F7TrtohwXb$&C0a-L}9S3DQ zir8QM&=0yhwi@?OXFjCqJnLLp?|;dN4Ao|T)n9uRQ(*V86YgK7+ZYHI&A>EL*2t>+|-f&Z_@aIqR-<()qCQ4bd~I?(_$w?=Abw zW*@;7u-Wu*pV+nC=)Tu8zcb(+NV9LTbA8_azYgCgIh6XBvi^CQ_>B;+Ka0IX;dd6K zE2sU3liveS%vPFHp9LBZwuX~I`zOWHWnJ&{_K)6^SOz_4pL>3Z`jGy<1Eav*SJ&kj z7l>Ov`-gj}Hb|g z0GFSCzbE~~DVzOo4&U>cK%RetEIGS#P4n$zGV<*g_O~`bJvB$t{B9e#3QBoKrTLSd znd>N}>wXLxT>Ym9jPg}s^*$vRoM-`@Ye0%EY8pqVH1$(calQ z=;bt+pP?V$uIAUctbI8PC*88x-(;p7*-y`RljT_WeC-8g9T{ntDEpg^qP?@=y70FM z-B~l5?$g||+FvqbKaJ~YW$D~c{C#1oqwi*z-C?|-G)3OTc$|7b6JPjF*fTKdIt&U<&E7TG=zH^YbE+cmF$ z!2X4w|4N6NJFN?@4V3!5r%JXDlZiFMaQyp%=2oSw8CAal`yUnj9zA5;FVArOSkQYb z)!Poj-z;hDEoIHPq}yNj&OFGnzpiQD{UJCQG-kMVaKrJHyuYEnGI2O);1z_^~uHM(% zx~m0yc7{CrjQyiI^%!KH39_xPpDe$gYfgh5X$M1atEwI7Z?j6K=sDv!{67c(Ipci7 zM{B>PXV&hTntc0={Utr}isj${(7O%)g)H0q*WM=XFW}R><31R-s@j3QcaL1^=eC0H zz&ByzIs7f3Vr%iP)-8U9y!({>?cFrwmhD!Aufr3NwQuIP0r_8ppTGuuQ^?v>ab@*V z4CIPsGQZZ5l{m=1PubseVEb&+5BJ3yxZi~rAZhEY@L!4lLs$!(&h-tECF!8vebWAB zBh8`IFU}0#by<)4==s?zkY#uOn);2WK>h5>;M#%Z-``#8+o$bs_OX37-q{;OoyNnZ zpgm#V)>(edBYp=5L%e3R{QIj*UHgUo%|7Z2sFN51+r#znE@au>zovD?9qdWzTUs*v;2$JjQ01^%f1V@_*zhZw!dk0R`@L5#)Xxrv)*}l5=vb&oEK#?4{=6?JcXm+BZE4^sGEtCqJC$@4x05L$waS1jgx|4BV2} zR4$EopINw6J*36AyrO4AI}pAR^vuZDMV4RdirYc!46Qd@9n;EM9Oq0PT+M;M3eAw` z-IF|j|M_|H%F>O0P0u#ZfcWpft3S#*;{GapCZc=vaM%}Cx4C_}_QuIGY@6Y_e^0{N z2as+3I%N5^&!GNpd+xWg^mkdhDR#|thpRUE|KL_opPW?(yQcBG=wDGK&X?f1oOxQ_ zYBMLn>fm&q{Ef7}FRW`b4|T4_sf*wZP#)@cp8(C{!yT>f2n}u2OdIjOB-`=K`AR{Soqv zcP6WzL!1TcS$o!+tI`MM4F=sa7lYcO$y3agJ|Bid!0EG?tY)|AVVPT!?n-zEij`&8 zH4gn2_Jb8*D69!5K;C{uee`XhKCZ0euRR}*>M6N?2!92WKaZ>DlHY|jOkembc98cL z@Dk*aO>xww{{-KHRlwax8b9(LalMXozcc+DdCY|OA+HP{5%&?09c6d*4|&sCJoRzf z@82HW_*<^IINt&b`=!uTf2X~m*TBjy=1Rv;K<_aM=&?lzh#gF}#>ueSI3*mxK0*^U7~y)&Zn( zeJrl~+a={{2+@0m8neC$dbXW5W@)Vc8Ekl`?Pr$h9TRU|EnSU(L*e(3r#};YPtV)^3!84G@c4=|cel)izMb8)0FAeHhO$w(*7Sr|8`1Q>A z4$yogP5!iV8nZM%-yVIrF$;fL4&@_m4qdf_?O`KWO6Nsy95y$Io;Qr4+*6^H`HAeV zxxv?AGf+LW-pI3OJD2inDp}9#_+0_AgnZrY10GEHcc8vL&E{$6G*7<-He>I=tr_u` z1chYui4A9hYQJ|0FvEI-fSq$7>LS3}nPVlLM!YJXG1 zTz$hK+P9!gjlY^tYhJw&tUkqDtr>p|>)Ja_rOd-kmTGowEleG2;e05hAFUq_xG81N zIMz-0cLJ^@ULn2qn*B}Z(frGvuc_bF{Q9pMGN#F{HRH*!2Do;g+#|E{`l=h%>$A9p z#&fNgMykCOI-)$Z2U(~O=ym&>tkHNEObg#J-vAlUh1Vd>-f8Ew2DUx274f@vpm$yS z%R!ws1FhH6>i+gY^fC94oUIufG)J_2`?5>jv+SSQ z|6d!v!yx;A5eoJH_i}AmwalTWg>zrmk0Y1XjM^i+3)&#f=4t2dq%AGom+fw{Q|B!E z>)zOZPBj0U>+MH#-gLMu{%asjPdcZ*WmmBHR`g0z-p;4&3XT{eS4Gp#b2ut(WYC)G9Y+F0#CZKV~8*}KqntrO;g`cOSV%Qta5%xNlJ21+dBN=XA_)2BVt!2U~-M(8_@h;Hs1^Cm>=o6!g|mMFGHR@ERNn8m<;iJOZRs2;JRxc ze7i?u$JX#`(BAI9K+h_Vf)&BV(;9Z*D6Rp$dnBEC-^4nda0{4yd{^^9-R}oO9k{gi z{N>t%TzF&7pAYKi{j^zr^*xWm;SjDFqr9!3zcmbJ<*jk78TN-}x5d#IavyvJ+&mF?;OIjd zu=U3E;rOr#_R~DC*qTx6&#vplBJ!`4bq+5K zI)~d5;g&?WsdJd*ozk}_!mYv3PS=}*JcTzzb6gEEA_+7&LZZ882Ml z(cCFq+tJi1Jg5WB#_<&hYP>L!gIX`#Lb%=ww-Iji!l(t27Uf4s;YIpz;ij5c9}25+ zm_8MzaWweh)*z_&!!(xIK8jy7T+`e+eqBo<+?qFxf~BrFetXbBxHitOHK-$8(`5NI z^E(w^KH6z%!Oml4v>%TDAp6eZfwYP=AsR}oneu<@UtpBWVLx-zH9KX(m zU4AZojSG*CuMcuz$*24&r%~zSa8ppO#;RPCn|=&?(%atlbe_r^D$KrY@br)TJRj5ynSvQ3g>sk|-)) zoj}xIbG=b1F^?QIfO>f_j~v~vVO`A8{r-S1xD1-O?y{gtjY=Mnaz~|gRZ^^GOiUK8 zN^SUPtxDUbDzy=lx~Mj;O6|%-VdpeRT+~V!udC8L!peYftdsU&w8C+f)CLG!mGCq+ zDcmfvbVOlVT&HkUrTF^7SVv)g9ZjaQ(V=!*EPW7~r>#E94*}&{*tjU%5PIGJOO>3S+74AMJZQNTZ5EbR*9wADT%E!QR3I2-wc*qQE{po`qWfQ2_+bna8rb4AS>>V= zu`;4?Yp3MWZlfg)-AET59UGAKL}-RV!5IGMxF86|jtdD-RQLbP<=k8^`ui(e5!Uk> z?fJhBCqiv?*Pw4o=dHD*|FLiew1B_XY9+jD->hMEN^Q@u(W~^XcP&na7r@uM@BdD9 z)}(Jqg??Q>6S*UC^(<4r?^_7IzLWmReXFL`B`vzO-{MOD!(l(TAM(CyIH3P-niL(o zHcwx#eU#ncX3#qk$u^!99?*Y@`j7kg@cH9*==)lDw}{>g-46r$a(>)QKm;dc@3 zy(R4#X0`9Pxqc~p6`oItJD~p}`nSDUbSM4m{jAer0c7d@J+9vfJHQa00S>{v3zBJi zq<^~?i&UXcUe-4$(3hdPq}EGmZP?f8$Hch<_Jm>J(xWH&%c}pm z-xmG=MuXG4{@!ldQ3?IuakTrL@>K~LUn0D1nhbsA9P+FGcqCCTU-#NOI|tVH_j$7F z*Ngg>oTp{UQtj7}Uw!_Q;Oks|t*w3q`WwfCa9x|vQ=eYef6>1!l1Jvn6$ibp0BQRF z1@Dx_bDC1*>t+2H{o4+u$X<@Pvr70|%mwtXxz9lm_}>c2D_gbc-=ACOtwVR>Nnak? zr>QBH4{_4kzvh=;Eta;>^=i|9G;ZlVUacQ0@||4e8=?0`_k~;Fz(P8*^Tg@*`lpWS zN6&{}!j`PtH0R4bo|&$auKEVc2RH3EL1!n^B*UcZ|N1}DeQQSF;L^O}@9;C&(0*Tu zzvM${9O2*6+y?(upx;aQ^O&=WWla1fMgNZ_bfdMykDwkxUFP^5l*OG(T>SQa^FQ$W zdN==o{%L^xc4q|o`9lF6>HXht!z$qHko6vDbkErD3t9@~Y5Fg^-$?rY-~Pkz#6g;?g2>cmL$7yaG*Js0;1v82+5U1bkKl%;+=c(({kftv?C%tLjvRG%3)J{oO;I|^O^D_gGa zC%x0-zA>4VPujI=*1yTCx<%h;(erA*?(&zS{||}3qw>ppC~kz-%HIc#J4qRuIrk}h z9<1N>{#6(1C(cZ%d#UnyPPnp<}LNBYzL)7CH*l51rvJLzgKRBM+>ps}LpF`)mhDnf7C4^>~2HGfU& z-Vf`!u-(uva9Qg|r+RL!GXWcTfYySS30sZ^h z2>s|){e4r2xuV=c}j^*n6XBvOBKjp?(xzXPj z(K|Nt!H+NhReBepYYB_a4CsH6F3gACc7~+>U*p`m5b&F!_)vIm_zd%M{QXw{swecy z{F8Fh^#3=`uMsQW^F;fs`a4VgQvcM;?)M8*>NuPIAI%lJ94T=%ZWf*Y6DLjoEu3Eq zIvwttO5yW`3%J&=^zZ+@4(g^k$+IbSRKKwE^yu4C{$5;4+@9!PZwJmzZt)2GH?>R@eZB>p2o?JBKqR!85}9)N7MTJ)Vx|C;=H^}j7~--l#+ z3%|@Y?MLYOLE3(h#ryfK{C&=>quS;_(O9+>^|}u}0jo=1S9<_zkD8+=)8y5^p2u7T z-6`M7)cv~SZAG3dRGa>hS!;hi543uux$kgYeL>naJNHV;^}PDmJy-{mK>InespVVD z)pLrAVdH6sWVQV8TGi=4`}w=QU%UnN8wWoGz3W{}Cw4uT{-fV7SU;%!Ozmr`Z?k;T z+;_QtEvWBu^RYaCv$IO|pM3vB_0@Byt&ul=zZk!Cv=RIU(sYtMmrMWFYNNi;{=V{9 z%9sjz-j&SvMb3QzbniO5<$V`4OaIfob)LzT$1U*wzho}@{%!RBNmBnBf4^;YGZ~_H z6;H@odVfIcpwmF| zChL$4SE~JYTmQ(s6wJzyUG3^-*bU|~sjvn5*Ze}`u-2jHK`SKV zr-iFd|1%GF?RqcdZz)m-)$tmL=U4bsj~?ki{9D_<%aG;oT)Q4*i!-t5d+)`@hMwp@>f6=NX-;`Bq_yKTd2~)=fab0fL3;?vcvY!? z>qL;(>@W=fo^S_f-!)keKWuwmtD7Crzw)^%C0)_^f1*8TzIQliKd0FE`W$g|-;4sQ zhcBo6Rjq&3J+b$zHgpiEKh``qSsx2aHySf1!3xp3vD^BW9R3`KJf+hgLE1g3zUBg0 z8$!KD?{pG2`OK|){j2Ue5`QaWW$N-xcnUNZxB4XAPq=m;oB+?H#LLzGLkYtFnO{1c zVh_#Jrl;hoIkMV`i|40JiQ8xNZ}qUhRS~tvHL24N;4dk4OWGnWT(bU64$6~$&q^ux zxEmj$>oN&(iY5Z3}u{mqG^q=1T^S}S!)Am1-x+e9ny1fC~A6yAcKZVR# zmh)B)uI8)_Fb&eiN7KFB9)16T^ov{nBaq>7@atrLy`MD?4us{v*`bnu19mR&mRdQu zOT!o7PSBjhebV0(U@Si&|KtT zcnJJCtL}|2^`tD)FK+#-j;n&!ov(oDJk3>oZ-DJVcB{M{qB5egR>swwRnMlhj?;ZP zv@h#lvQvj(&M|JkW*q*X!#^QS|90*zu3rY5vo6v;Pq&__l%h9V2dgYlzf_j{kbD+v z`!^j}{g`vsAk%hmExZe+^ECHWuAdF-*_%qBW7~SPQ$u zt)PAKH2vE-t)0FHqanPX68o2yf3>;DTUGj(e(YX|bU7S%e`tnJz;vE;wPw&f_8YJw z#O<`&bneT8oK>fPlW%so=V*GAPLF~oAZx!yb5`x2902j0)$*)17a7Z;{~h&P)=C}7 zXf}|$I-CqIK=K~&!eJKx2R%#JE6S^)RZGmt*NtQ^HfPls zu4j9n1&!?$pDX22f|DbB$Np0Me!G#sl;58!{>>2YRhMI4kW~lDa`(gRNp9|`-;1dZ znv*8$;fFP+y#zL!9(~VJ`)|E{P`=&EgYoYQ^TC%>{!;a?{`eU<7PLkQ^%FfaC{#y_ z=evdCrk(fh2d%}_zSYi)%~^H7=$_a8W4cJozO3g=);oEBm2x+R^WjZM>b{u%|4|@~ z(rIn}5F89zv%C6TmROuKrioc5~j1dR(?E4uob8_85p z_*D2CXkID1y8eN&+^uQVW=c6oZ{=v-qxp~KL7ESlTuE2=?Gtbm#Bln7AGM>s;Ze}@Hie?01B0>c6cnT0=eyOaVbN6_SSGUv_heBx^v$8hr`IX)~{P$zl}Ad zSKTz$XuYF-qbzwehF=HUVMjM6b!Sc&jk)T7_ki2sW3X~x!v7c_rZifqZaqz~l`+vIq{mqDI6tM+5Z-Q+!&N#>muj>`?# z;_Ba2?y0b0^bOCT9BV?aoMrLv3%`Y|{Tk`@4N(0K0gWeF{ll4A^~`&XGVC7LoOOG+ z0g63iTtJ*NU@W-wnnUD0{P$io$Fe-+mh(uAeJhav(eQUj-UDjyx58&Y>kxhOL2{{o zxD>RePNwP6bG~Ffy?b7B*0la1S-!&0!%5JsIjiZXr{j9EF3GT!GdKMGw%T>FE}G+P z2(B$Cy*FNt;QYLlG-cJl%GBR4F&S_*XFU*9cGjF#^O(osNEiuW9wp9|Req(9=w3Sr z8IyCbKX7hD+BePt83TARgx2D#GoukmaH&~I(r zA%BF{A8Gy1EY53=<=&^QWX+;^qnnpd-uiGhyavABegAx}O{k+f)^;TSMIakBf!4fsEzOnPo`PdQ{j}XP6|GWS&XMk&7tLAE0kws# zm^Z}jr=mi1Gjn7d2kLLq^l9e`*~hLYUCmF_N2zbqyv5nU{{CP$Ma@k1KdxFiNjEd$ zV!nv1Y9F76dmyb}N$R$g@Wj^r#G%kmfb;f$F;%5FJ(8qSS3)XjQvheTAs1Lgs-h{L?c1~mCWuUQAW2V!q=Pr$Z zZ@3FIZ&W*Y4o-yCp~g{(h$54Ygq?GecyTJ!G*_k->qU+<;(|4rJd zur&-RSopw&SOZO`bh;l#qPHVKdtzE2H8W zz7O_+Wx@4rB@{alHK75`8%>Y6W8rjY0smgmnoV=xiLe5=G}&Q@)|0 z`O}tgBdDKw9U5T`h;3CMo2owDp`*$DAO06C^0#~bR?pw;`GZCNcF*7H`CB}H)1an~ zg^jL=g+XmoM|-{Vw*~e1Yn{I}Xk6qUglu-c#`D*C{`%Pe$JWr_PDXW+zpbI8J@(gk zw8sA0j^@};F|j{r3>Nttg8C?*g+YDL82LMz>IZe0zq!69_P641iDZ+%DUuogLbTxO zf*(yadj5t~e@)Y(^R+G6{+7C$_L}A}PE)P&cm9RJpxDp3xZF1U$kV8D{0Btb24!}IN)pLP?TS958K{DbhfME;<$qoXbK)2=!?7KVQ1(a|y0 zrKNEE9ihKDY{DvwLL1`q?Ll4aj}o{z9btl2+5m;R1kJP!3RP*ejiA1@LsIfzeNfxn zAoX&wp=l5iT8IhF0TGm-g_aZqgIt2vT2zC-19WFNeP|nBV;g7|`AB!h{v;L!M zs#a_0AL_&l{eh%JC%P&jqQeOh`4J*$JSz&|U(2N=+`K&@OMrOh=ZfcVjkK$Xk(6PY zP`RPMDRedPJv33|kzpg#uci?ZZQ!j|HSc*v0+kaN- z@2Oq1ScI!!p^3F$v1j$ate_~pn)O|se}goPww>o)kt=#t$O;ukU2U$WeVH9V^EvJH z`txe7XPyPk^Xz<|u0Qp#dTSnQ@5;UH)lc=*p53D`5wzye`n#n1cj`;3;QbYS*MT}1 zS?_Af-jiWe6kmHmzlT!mFa2u&PkZ97+zK`2T-io4s7~L5zkt=p-1(fF0&9cmNv`_C z((51YyJ!#QF*po{t4%DfgYxx!;5ta|BWZ8^0@!$_*B_dkobOis**~cz?cE^#svr3Q z_0Rm<5>{pWzOPV*uj;ko8@VBkefU>r~3aH ztp0jlp*E3PZ%fbhvelpWuWBq`bNzibJ6Eb}b-=9wjpJPo&L!)Aw&L-7z089+<*NV8 z0`*rvxr{$vWD4Y6l>8COV>7rMPJ=G@8RwGqKgY|epqF$Ntv~rM4_84G)I(UOjJ`6C z6TVZoF5xrbRZ!cW30>|p&Q)vuwa(CUTFI{a^EimdOx+*JgX_|f_S?P-FM{3Eay5_Z zOZ8t7d7t*`pt;v0a46_qf-ZljnsZ0M6OcBReJ$_KetV` zft0wdoVyyffto&6f3;=x*)kKy!Z8)$&WaFLCV@aAS_2hy0vh9QB{>$=kj9hwlJA z>*b?*%l@xGa=ou{Bwm%L}A`&Rb1`m5dV2kuaRH$Chhv<9q- z`b)M5tx0vyn_PduKQg-J1Rrs*m2U`4MO= z^6TdN)qkA@+Pn167pnhi#PjcOOY;HeKS|{AF=3V8>-|4vs=eO;CZD-_ckZ2(^M6d7 zORc|tcTK!{m;x<{G*-rcIX@579^Cs%i!2}dIA3Y|YhL~gm`rN>zk=;xOSlNEAG0{V zn_7Q+ruFd;oE|nIkEx*gTG}`89}6m9x=KAb-=+SA;`$kAEFqrmDJw5^|A*@;|7kCt z`s88YWWrUSbval*NjJ6rCX1KPTKHe~@==~)8QFU+$@)va8l}}ZQt0SG{m*rIM{Ai# zM(uMX>uur8`kQ>DUm5=F<#RUvq6cv_Pk6*jr~6WU#b|K)OZ#m@59&XQe3!AjakZ}f zC0KovZg=aScmGfAcut~x&8O549StjjwbfXkCDz}+|J4_3%`ySh2mAuwhh)7iT+#Yp zf0C=)#*{x5WP2-DuKJS)LH(!h6Q>9LcCv)}>t4SGq*Fg{joliv{d4mBa~6dwT7UcA z0OhD1?gDBH{`^ET$Y!_09-#WD{)(Tc|C7#Eg_Ge$NY>ZF^SRapYbWAVwEothDbv{& zSNG;Va33V+5we-a^3Q_C)1vi{^&5`Y>yqZD@Bvu8+wiNe-gx#mV;St4+-}$ZMt-+d z$}yw!@4Qx^?8D(P_#gN(y+v64*z4Z;)bY>CxAB_zYCrdQ>6HHnaQat#?^=)QuRLj| zB_E!wl4TSe56^)$i;~xWmY=!gu`>AM-(7@Pfv{a?%mFNqq#LIV z^U!^nHP@V#_WUa)Pu0K2`#&l9YM$lSUuz%r-TT9EaC%4^%aeH}!|q_TB&F z+$As$;xwfyLUr}i%b({UUPbJ0WhZ1>jqr)^Joxvj{I9?nFc#vvSU#0j>spY{BwiR(@-&812`EUo%G{F}nXpnJf| zm#ew?<8UO5gs{AB=)I79mM53_EXbnx5<$uF6**crB<; zG&$s|&%7V@gLuqLmRCwx-_;;5Lu+m6*w_EPgp>A9t_QVF(paN0NAkOQUE+IqYQLWW z^?hM^DeWb`7LJ7)6E!xbjhV91=b@n9i|K*1R&VVA?ge*38~8e(O?W9kFTzK_6Oddp zYMi|gHo0-pT45!^$AH%B(v#UwW1%ZQ*&gO}p6AQXvUf9Rz3a=S{H}p*!RZ}0_d#5} z`_KT>A!*b9aPA)1R(A9c&Pi_B^SkgO_%g}g0_VU9@HbH-~7aJ`iWfK z_eVh?y{q5Y)PK*`@+ssJe@%D45&C9GOV zMx9T*Mp`bWoZ039W|aPm|2*kJnZiD6Ea4x+E08Qt^NAB-H3(%)`R2G?!<8PC-zYm* z%6C&O4ek(_0MCKR^dWx9G6E!v>?b{F{bO+^`t>a4C!MPk>p8iaC(3?)pRIhQW6duo z!JOk==Ew1`f%|&j7kHKV+t9$TrSh_o9l#+%r zehOB$+P}sDzx`iC*l+)uV{JO?h(&cNr98#eJ`#Bt8Z*BN4}k2iF<_kn(U`eC z+z9W1-$vAD-3woa#8vY12_g;X%5r&tw5@pB?nao~tSBI@(1v{VUvxaegPq-bD zeYg5hwIj`)l5=V4RQ>vhGKgG}EYj{pzo0~W(S36u%!6cI{k9>U%z>T3tpWW!`=URg zC*3z{>#}(}Sl!j9YP{4vu?$qzy6jw7HgfBo+uqiI`$k9W_vJ3= zXmM^!V^iqX2ccUVwCQ?lZH;r=YwOySeqn80OI_$TDSyIEgF3sXPYJ{3PEojhP#CW@ z2;G)|uWE1}w^jMIOsu1v#wjhWavP#>ovybvwAO@fTQG(6Z4G?oqrC=~&rDE0Zi{p4 zJKCIE+p*BOe8WL*JD-A>huaZ;2BMB{Z|HLnbsepI{z7gW`de6A!v`rE_&i9^#-~v_ zT5E%e4J|bt&9zOG7<4q(w*<8d`HDrOzS7ZD|87USzT%;eN!0SS4Sg;G*L^=`Fxko? z`aZ-w=Qi*^pUDX7+_xuM#)MysXlskU8sWa2;bPS5f9&4Ke;d$9#=}I2zZ0=;Wxf+( z?HpHYoC$CzXq-G5y8O-zk*eZAHg|ts$hsTfF12*Qtn z&F?f(@^ijB z@~a*0?dL^!5X5sw{Hce|d#Eow9i-o-`~-we{`c^2<6rarTq~FSYM0l-@vtg{a-v`V zh|a6em<-Q?bbKEq-{qOSDeE2X=yZPJ&001?xCUB5=X7s<$xlbPn&iJcl2h}^ zoxyCWdqMq`+R}lQlt0=h&>YWiKXL!Ud217w`f^12H~FReH9+!bF6@8{taFOBAn`o?2?5H5@U$!BS}0@^?_s{A)Ws!LU{96AOoYxcg0FXWMeef4RWlHYJpf&b4Kz*Od zVs4uJvZ2tprhZg$Rqx|L$D<2Q%eEDxd{=tfu_N<)q#pJiO>CD74t1C%sa+}*l ze#xb>BP6rpYApDSJC_(QqWF^E?ltM@KydwqOGAEk5YCc+Rj!--cX@eeESL!>^vKT`2F5;3n7=)DOls!=EL;_Lglvtb6cAmU_3|i6xM<(F4}wdc=U`mE>Q2^nfgvmIOq?}C*pH&uR95Z86z+!duMIRCWx zX)bB4|5165!hVq2uO!p!-9O?Q>OXdaTS51w^3&M5S2CT2xu!HF&p$1_T+(j|lH0~{ z+2Ll8EtJ2L&rg%|C-zO%CiaJWK>bsFGQNemRu1_k-zZQ&`W%@3)fTRWZ9#3Mcs)AX zLpIb}x8T9GT=Fwk7CcO@$Xy5bG-v^|U9aP6Sej(Y4osaTZ3b)w(tLrOUnPv;U*J_3Obo zwHdXYYd~X^)#G{mC&FqF=TS^vyY8pKzaBgT%KKsP*T2>Ws1JJ(bWiEtip#_=88?6~ zvUu^-$LU_yJ*|6NWoc~s15APyz~!mASk__ja94t(;VH1OS94~6|3rEF{fg?NzHN6{ z9yHEf3a^9m`4+@>k{o_}=bG-vec^tPPE-%gS$_*Uuj`?1zMWnRu1jD0fckoyxBmzK zZ(t|z_pg+f`jyG>ET}x8enqy@9MaCoRes9%07&jJ`ZgQMb?N4DFx|;6H^HtTd%5_@ zT5w)-pJQP@NbX1AFzB*>sCLeIV{QCXpap!H|4sOIcnnM*t@w9HmaQ=7ebnZ3ALxFt zJe5{+xotqYcJVDQg(I7;kADwX)#AuaJM8-O@ZGlYgr5SH_b=EF)Q7DCSAfoEj`=Es zI6mr+G?!EVWc5(Lbs=mLy=NPX#yLN2r~1W_QaoNd%~Mp4^tPgRUhPKhLGph==dG_w z$*LMKf#~D?^kXiyc@_)CZAl@YjcR( zpvt-_To?H9d=Yx=k7SS?bT3~9uY>Ms^=s;HSBDeeIk5Vv&sCd{K34#ZA-96|A0LJ< zL#O8=Jr*=01IjY}s6XBSB(t3}SN)3ay%w-~=-zxB-t*4AfPW(Nn*3HZ$~0M>n>>F> z*sp9tybIwCuzJXS2^wJxbV_geEN<82!qr$b0qz9pUTtY}|Mzd@mNk4`NG}^T^V*PH z{plNAz4s)yH8{0VZu5v!>zz9}bRTGS?z{6^PaVzq1#%mLV2a#2-Z_%X&SDMjTDWz% z^{4ZeqP()&|OFx zg?V91<;~;eE4j`5g&*fOv^$qSJJq0Y8}D1mZRaH{T;B6)lv{ItbN#{^GOB5@_?`H&U=NM4?pAJJQqGAX6yL$_FY1j+y28~a;2OorG{K8{d z6VJo_IC=c!Yvrns)4ekbG*A6I{0iQLdC;kR!j{gtdCE$QLq0Y}X#8&ljn$Wf#tqFk zG=KUXB+Flhb4S32ko!1b8_FN5MQrSo>+FSUH-xelBQAH!onujk6reXO=(ZA)==uOAE= zt0cn&aMz!BE)P$F+NXOyj;r!-fgw@e($%lwebD?$GQ0&-!1Z;wC6%vyb&zg3 zP~BC&Ta)0b{99qED30#ie}Uxu4D1P-d#FuYnv%;Wt;*LPfy!52c26bC--v4}UpkT9 z)ebZl?RNR3RoSxNF7O(t@7x2lHb|CV&pDN+b}|Rl{&a7sjo7_eQvFG%_M|n#+aSBX z2eR$ekSt$)#a3_wXdUq(oCoWHJ8-Rh_5Z`{d}KehAL+`!|5qcdcDw*|@9hkK0hOIB zU%C=&!7K0}NPc%dDnsSJ2=hVftU+MuvdUL~t~Mx{e+ym8cln3=z$UZwpWM%pJQsoV zC|mv%HUP<>c9pFEikwqFuKwpHP(RlO|AbEUcM9e|&S&+1x~@8?4XItI50fs`hiwMB zzkK~mKN>fG4U$3aN&T^OSFHT|h~xS{8v_!1OVa0o@DThLTHphC6m0yFyh65l3N&_V zOwhSP~%EQV~Tl@+%gVMbSC%`DMIDYx+_n!ydd-uZbVDk^fSAV8*pM`c%o*F0q z1WNM+tfvfdeWasF(RHO$ydyy8UA!ob?j_}a2qfoUety!CY@>eZv#=t}1ledi@bmQ3 zDgP5edXVh*!@gkiKR<4qN35Vl^^$$n-@FAXPvy@5)lKyp0-e%ZIC9s)Jsn;F?SpM= zW#I3W&d&QTacbbF@DWJ=>Z_!Gp*DIss2`2n0_!dL2SZrLNzPrL@Osec{`bpD^GA6M z$K4+u1dW-e!9E~6NG|npDReIJ%^tO(wPuHVK+1Dz*bD9k zyZ0oEWV#9Tv#7EhCu-n%1g0tA!K#iC#<1c(#7(JKD8f(c0jCPsX-{+pCE8DHsW|_9;}?h)r;R z32H0WpSo*FWkm5U-717n0=20h!(Fhl(%`NMy6-Hm%PI=14XCXj4_Ct&SRH1A+oQ+* z4tyPSFDbs|GZFteAY1(q6i3%o-W;&3_2{WcAb2Bla1R|2I``I3KV z<+4{&P(Ux zp*{+0T$=@Jf$r4_ptx#(l27(g9+`RormuR4tb zwY@b!ajg9)&voDtI2;~^ox#deUXx%Zd1tEh6P5$jO~mQk z4HVv6I;VUk+c4M|E`>jUWL10Xl1}N{8rdWLBKxH~2f>gDoDjQ5F8+1#j%DX$%-MLN Rp7?Y|pC(o=%?&~D{{dv=3{wCA literal 0 HcmV?d00001 diff --git a/packages/essnmx/docs/_static/logo-dark.svg b/packages/essnmx/docs/_static/logo-dark.svg new file mode 100644 index 00000000..fc42a002 --- /dev/null +++ b/packages/essnmx/docs/_static/logo-dark.svg @@ -0,0 +1,162 @@ + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + diff --git a/packages/essnmx/docs/_static/logo-text.svg b/packages/essnmx/docs/_static/logo-text.svg new file mode 100644 index 00000000..1f21b025 --- /dev/null +++ b/packages/essnmx/docs/_static/logo-text.svg @@ -0,0 +1,665 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + essnmx + + diff --git a/packages/essnmx/docs/_static/logo.svg b/packages/essnmx/docs/_static/logo.svg new file mode 100644 index 00000000..cb6f702b --- /dev/null +++ b/packages/essnmx/docs/_static/logo.svg @@ -0,0 +1,166 @@ + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + From f3668409d5065b583a07a3fa29e7049919f4f889 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 10 Jan 2024 09:16:03 +0100 Subject: [PATCH 032/403] Relocate resource files. --- packages/essnmx/{docs/_static => resources}/logo-text.svg | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/essnmx/{docs/_static => resources}/logo-text.svg (100%) diff --git a/packages/essnmx/docs/_static/logo-text.svg b/packages/essnmx/resources/logo-text.svg similarity index 100% rename from packages/essnmx/docs/_static/logo-text.svg rename to packages/essnmx/resources/logo-text.svg From a9e6d4b2f74a86f4cea550ccfc1549c68dd057da Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Mon, 8 Jan 2024 18:45:57 +0100 Subject: [PATCH 033/403] Geometry parsing draft --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 273 ++++++++++++++++++- packages/essnmx/src/ess/nmx/rotation.py | 70 +++++ 2 files changed, 337 insertions(+), 6 deletions(-) create mode 100644 packages/essnmx/src/ess/nmx/rotation.py diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 90855426..103fd341 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -1,19 +1,128 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) -from typing import Iterable, NewType, Optional +from dataclasses import dataclass +from types import MappingProxyType +from typing import Iterable, NamedTuple, NewType, Optional, Protocol, Tuple +import numpy as np import scipp as sc import scippnexus as snx +from typing_extensions import Self PixelIDs = NewType("PixelIDs", sc.Variable) InputFilepath = NewType("InputFilepath", str) -NMXData = NewType("NMXData", sc.DataArray) +NMXData = NewType("NMXData", sc.DataGroup) # McStas Configurations MaximumProbability = NewType("MaximumProbability", int) DefaultMaximumProbability = MaximumProbability(100_000) +AXIS_TO_VECTOR = MappingProxyType( + { + 'x': sc.vector([1.0, 0.0, 0.0]), + 'y': sc.vector([0.0, 1.0, 0.0]), + 'z': sc.vector([0.0, 0.0, 1.0]), + } +) + + +class _XML(Protocol): + """XML element type. Temporarily used for type hinting. + + Builtin XML type is blocked by bandit security check.""" + + tag: str + attrib: dict[str, str] + + def find(self, name: str) -> Optional[Self]: + ... + + def __iter__(self) -> Self: + ... + + def __next__(self) -> Self: + ... + + +class Position3D(NamedTuple): + """3D vector of location.""" + + x: float + y: float + z: float + + +class RotationAxisAngle(NamedTuple): + """Rotation in axis-angle representation.""" + + theta: float + x: float + y: float + z: float + + +@dataclass +class DetectorDesc: + """Combined information of detector and detector type in McStas.""" + + component_type: str # 'type' + name: str + id_start: int # 'idstart' + fast_axis_name: str # 'idfillbyfirst' + position: sc.Variable # 'x', 'y', 'z' + rotation: RotationAxisAngle + num_x: int # 'xpixels' + num_y: int # 'ypixels' + step_x: float # 'xstep' + step_y: float # 'ystep' + # Calculated fields + _rotation_matrix: Optional[sc.Variable] = None + _fast_axis: Optional[sc.Variable] = None + _slow_axis: Optional[sc.Variable] = None + + @property + def total_pixels(self) -> int: + return self.num_x * self.num_y + + @property + def slow_axis_name(self) -> str: + if self.fast_axis_name not in 'xy': + raise ValueError( + f"Invalid slow axis {self.fast_axis_name}.Should be 'x' or 'y'." + ) + + return 'xy'.replace(self.fast_axis_name, '') + + @property + def rotation_matrix(self) -> sc.Variable: + if self._rotation_matrix is None: + from .rotation import axis_angle_to_quaternion, quaternion_to_matrix + + theta, x, y, z = self.rotation + q = axis_angle_to_quaternion(x, y, z, sc.scalar(-theta, unit='deg')) + self._rotation_matrix = quaternion_to_matrix(*q) + + return self._rotation_matrix + + def _rotate_axis(self, axis: sc.Variable) -> sc.Variable: + return sc.vector(np.round((self.rotation_matrix * axis).values, 2)) + + @property + def fast_axis(self) -> sc.Variable: + if self._fast_axis is None: + self._fast_axis = self._rotate_axis(AXIS_TO_VECTOR[self.fast_axis_name]) + + return self._fast_axis + + @property + def slow_axis(self) -> sc.Variable: + if self._slow_axis is None: + self._slow_axis = self._rotate_axis(AXIS_TO_VECTOR[self.slow_axis_name]) + + return self._slow_axis + + def _retrieve_event_list_name(keys: Iterable[str]) -> str: prefix = "bank01_events_dat_list" @@ -37,13 +146,142 @@ def _copy_partial_var( return var -def _get_mcstas_pixel_ids() -> PixelIDs: +def _get_mcstas_pixel_ids(detector_descs: Tuple[DetectorDesc, ...]) -> PixelIDs: """pixel IDs for each detector""" - intervals = [(1, 1638401), (2000001, 3638401), (4000001, 5638401)] + intervals = [ + (desc.id_start, desc.id_start + desc.total_pixels) for desc in detector_descs + ] ids = [sc.arange('id', start, stop, unit=None) for start, stop in intervals] return PixelIDs(sc.concat(ids, 'id')) +def _pixel_positions(detector: DetectorDesc, sample_position: sc.Variable): + """pixel IDs for each detector""" + pixel_idx = sc.arange('id', detector.total_pixels) + n_rows = sc.scalar( + detector.num_x if detector.fast_axis_name == 'x' else detector.num_y + ) + steps = { + 'x': sc.scalar(detector.step_x, unit='m'), + 'y': sc.scalar(detector.step_y, unit='m'), + } + + pixel_n_slow = pixel_idx // n_rows + pixel_n_fast = pixel_idx % n_rows + + fast_axis_steps = detector.fast_axis * steps[detector.fast_axis_name] + slow_axis_steps = detector.slow_axis * steps[detector.slow_axis_name] + + return ( + (pixel_n_slow * slow_axis_steps) + + (pixel_n_fast * fast_axis_steps) + + (detector.position - sample_position) + ) + + +def _get_mcstas_pixel_positions( + detector_descs: Tuple[DetectorDesc, ...], sample_position +): + """pixel IDs for each detector""" + positions = [ + _pixel_positions(detector, sample_position) for detector in detector_descs + ] + return sc.concat(positions, 'panel') + + +def _read_mcstas_geometry_xml(file_path: InputFilepath) -> bytes: + """Retrieve geometry parameters from mcstas file""" + import h5py + + instrument_xml_path = 'entry1/instrument/instrument_xml/data' + with h5py.File(file_path) as file: + return file[instrument_xml_path][...][0] + + +def _select_by_type_prefix(components: list[_XML], prefix: str) -> list[_XML]: + """Select components by type prefix.""" + return [comp for comp in components if comp.attrib['type'].startswith(prefix)] + + +def _check_and_unpack_if_only_one(xml_items: list[_XML], name: str) -> _XML: + """Check if there is only one element with ``name``.""" + if len(xml_items) > 1: + raise ValueError(f"Multiple {name}s found.") + elif len(xml_items) == 0: + raise ValueError(f"No {name} found.") + + return xml_items.pop() + + +def _retrieve_attribs(component: _XML, *args: str) -> list[float]: + """Retrieve ``args`` from xml.""" + + return [float(component.attrib[key]) for key in args] + + +def find_location(component: _XML) -> _XML: + """Retrieve ``location`` from xml component.""" + location = component.find('location') + if location is None: + raise ValueError("No location found in component ", component.find('name')) + + return location + + +def _retrieve_3d_position(component: _XML) -> sc.Variable: + """Retrieve x, y, z position from xml.""" + location = find_location(component) + + return sc.vector(_retrieve_attribs(location, 'x', 'y', 'z'), unit='m') + + +def _retrieve_detector_descriptions(tree: _XML) -> Tuple[DetectorDesc, ...]: + """Retrieve detector geometry descriptions from mcstas file.""" + + def _retrieve_rotation_axis_angle(component: _XML) -> RotationAxisAngle: + """Retrieve rotation angle(theta), x, y, z axes from location.""" + location = find_location(component) + return RotationAxisAngle( + *_retrieve_attribs(location, 'rot', 'axis-x', 'axis-y', 'axis-z') + ) + + def _find_type_desc(det: _XML, types: list[_XML]) -> _XML: + for type_ in types: + if type_.attrib['name'] == det.attrib['type']: + return type_ + + raise ValueError( + f"Cannot find type {det.attrib['type']} for {det.attrib['name']}." + ) + + components = [branch for branch in tree if branch.tag == 'component'] + detectors = [ + comp for comp in components if comp.attrib['type'].startswith('MonNDtype') + ] + type_list = [branch for branch in tree if branch.tag == 'type'] + + detector_components = [] + for det in detectors: + det_type = _find_type_desc(det, type_list) + + detector_components.append( + DetectorDesc( + component_type=det_type.attrib['name'], + name=det.attrib['name'], + id_start=int(det.attrib['idstart']), + fast_axis_name=det.attrib['idfillbyfirst'], + position=_retrieve_3d_position(det), + rotation=RotationAxisAngle(*_retrieve_rotation_axis_angle(det)), + num_x=int(det_type.attrib['xpixels']), + num_y=int(det_type.attrib['ypixels']), + step_x=float(det_type.attrib['xstep']), + step_y=float(det_type.attrib['ystep']), + ) + ) + + return tuple(sorted(detector_components, key=lambda x: x.id_start)) + + def load_mcstas_nexus( file_path: InputFilepath, max_probability: Optional[MaximumProbability] = None, @@ -59,6 +297,20 @@ def load_mcstas_nexus( The maximum probability to scale the weights. """ + from defusedxml.ElementTree import fromstring + + tree = fromstring(_read_mcstas_geometry_xml(file_path)) + detector_descs = _retrieve_detector_descriptions(tree) + components = [branch for branch in tree if branch.tag == 'component'] + sources = _select_by_type_prefix(components, 'sourceMantid-type') + samples = _select_by_type_prefix(components, 'sampleMantid-type') + source = _check_and_unpack_if_only_one(sources, 'source') + sample = _check_and_unpack_if_only_one(samples, 'sample') + sample_position = _retrieve_3d_position(sample) + + slow_axes = [det.slow_axis for det in detector_descs] + fast_axes = [det.fast_axis for det in detector_descs] + origins = [det.position - sample_position for det in detector_descs] probability = max_probability or DefaultMaximumProbability @@ -76,6 +328,15 @@ def load_mcstas_nexus( weights = (probability / weights.max()) * weights loaded = sc.DataArray(data=weights, coords={'t': t_list, 'id': id_list}) - grouped = loaded.group(_get_mcstas_pixel_ids()) + grouped = loaded.group(_get_mcstas_pixel_ids(detector_descs)) + da = grouped.fold(dim='id', sizes={'panel': len(detector_descs), 'id': -1}) + da.coords['fast_axis'] = sc.concat(fast_axes, 'panel') + da.coords['slow_axis'] = sc.concat(slow_axes, 'panel') + da.coords['origin_position'] = sc.concat(origins, 'panel') + da.coords['position'] = _get_mcstas_pixel_positions( + detector_descs, sample_position + ) + da.coords['sample_position'] = sample_position - sample_position + da.coords['source_position'] = _retrieve_3d_position(source) - sample_position - return NMXData(grouped.fold(dim='id', sizes={'panel': 3, 'id': -1})) + return NMXData(da) diff --git a/packages/essnmx/src/ess/nmx/rotation.py b/packages/essnmx/src/ess/nmx/rotation.py new file mode 100644 index 00000000..98beab8c --- /dev/null +++ b/packages/essnmx/src/ess/nmx/rotation.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +# Rotation related functions for NMX +import numpy as np +import scipp as sc +from numpy.typing import NDArray + + +def axis_angle_to_quaternion( + x: float, y: float, z: float, theta: sc.Variable +) -> NDArray: + """Convert axis-angle to queternions, [x, y, z, w]. + + Parameters + ---------- + x: + X component of axis of rotation. + y: + Y component of axis of rotation. + z: + Z component of axis of rotation. + theta: + Angle of rotation, with unit of ``rad`` or ``deg``. + + Returns + ------- + : + A list of (normalized) queternions, [x, y, z, w]. + + Notes + ----- + Axis of rotation (x, y, z) does not need to be normalized, + but it returns a unit quaternion (x, y, z, w). + + """ + + w: sc.Variable = sc.cos(theta.to(unit='rad') / 2) + xyz: sc.Variable = -sc.sin(theta.to(unit='rad') / 2) * sc.vector([x, y, z]) + q = np.array([*xyz.values, w.value]) + return q / np.linalg.norm(q) + + +def quaternion_to_matrix(x: float, y: float, z: float, w: float) -> sc.Variable: + """Convert quaternion to rotation matrix. + + Parameters + ---------- + x: + x(a) component of quaternion. + y: + y(b) component of quaternion. + z: + z(c) component of quaternion. + w: + w component of quaternion. + + Returns + ------- + : + A 3X3 rotation matrix (3 vectors). + + """ + from scipy.spatial.transform import Rotation + + return sc.spatial.rotations_from_rotvecs( + rotation_vectors=sc.vector( + Rotation.from_quat([x, y, z, w]).as_rotvec(), + unit='rad', + ) + ) From 2530cbd473bc195233a0c69f36cf451bc3dfe26d Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 9 Jan 2024 14:21:28 +0100 Subject: [PATCH 034/403] McStas data loader with mcstas geometry xml parser. --- packages/essnmx/docs/examples/workflow.ipynb | 33 +- packages/essnmx/src/ess/nmx/mcstas_loader.py | 276 +------------ packages/essnmx/src/ess/nmx/mcstas_xml.py | 393 +++++++++++++++++++ 3 files changed, 431 insertions(+), 271 deletions(-) create mode 100644 packages/essnmx/src/ess/nmx/mcstas_xml.py diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index 5a0551c0..5b475717 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -91,11 +91,40 @@ "da = nmx_workflow.compute(NMXData)\n", "da" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Instrument View\n", + "\n", + "Pixel positions are not used for later steps, but it is included in the coordinates for instrument view.\n", + "\n", + "All pixel positions are respect to the sample position, therefore the sample is at (0, 0, 0).\n", + "\n", + "**It might be very slow or not work in the ``VS Code`` jupyter notebook editor.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import scippneutron as scn\n", + "\n", + "\n", + "unnecessary_coords = list(coord for coord in da.coords if coord != 'position')\n", + "instrument_view_da = da.drop_coords(unnecessary_coords).flatten(['panel', 'id'], 'id').hist()\n", + "view = scn.instrument_view(instrument_view_da)\n", + "view.children[0].toolbar.cameraz()\n", + "view" + ] } ], "metadata": { "kernelspec": { - "display_name": "nmx-dev-39", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -113,5 +142,5 @@ } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 103fd341..92dbf556 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -1,13 +1,9 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) -from dataclasses import dataclass -from types import MappingProxyType -from typing import Iterable, NamedTuple, NewType, Optional, Protocol, Tuple +from typing import Iterable, NewType, Optional -import numpy as np import scipp as sc import scippnexus as snx -from typing_extensions import Self PixelIDs = NewType("PixelIDs", sc.Variable) InputFilepath = NewType("InputFilepath", str) @@ -18,111 +14,6 @@ DefaultMaximumProbability = MaximumProbability(100_000) -AXIS_TO_VECTOR = MappingProxyType( - { - 'x': sc.vector([1.0, 0.0, 0.0]), - 'y': sc.vector([0.0, 1.0, 0.0]), - 'z': sc.vector([0.0, 0.0, 1.0]), - } -) - - -class _XML(Protocol): - """XML element type. Temporarily used for type hinting. - - Builtin XML type is blocked by bandit security check.""" - - tag: str - attrib: dict[str, str] - - def find(self, name: str) -> Optional[Self]: - ... - - def __iter__(self) -> Self: - ... - - def __next__(self) -> Self: - ... - - -class Position3D(NamedTuple): - """3D vector of location.""" - - x: float - y: float - z: float - - -class RotationAxisAngle(NamedTuple): - """Rotation in axis-angle representation.""" - - theta: float - x: float - y: float - z: float - - -@dataclass -class DetectorDesc: - """Combined information of detector and detector type in McStas.""" - - component_type: str # 'type' - name: str - id_start: int # 'idstart' - fast_axis_name: str # 'idfillbyfirst' - position: sc.Variable # 'x', 'y', 'z' - rotation: RotationAxisAngle - num_x: int # 'xpixels' - num_y: int # 'ypixels' - step_x: float # 'xstep' - step_y: float # 'ystep' - # Calculated fields - _rotation_matrix: Optional[sc.Variable] = None - _fast_axis: Optional[sc.Variable] = None - _slow_axis: Optional[sc.Variable] = None - - @property - def total_pixels(self) -> int: - return self.num_x * self.num_y - - @property - def slow_axis_name(self) -> str: - if self.fast_axis_name not in 'xy': - raise ValueError( - f"Invalid slow axis {self.fast_axis_name}.Should be 'x' or 'y'." - ) - - return 'xy'.replace(self.fast_axis_name, '') - - @property - def rotation_matrix(self) -> sc.Variable: - if self._rotation_matrix is None: - from .rotation import axis_angle_to_quaternion, quaternion_to_matrix - - theta, x, y, z = self.rotation - q = axis_angle_to_quaternion(x, y, z, sc.scalar(-theta, unit='deg')) - self._rotation_matrix = quaternion_to_matrix(*q) - - return self._rotation_matrix - - def _rotate_axis(self, axis: sc.Variable) -> sc.Variable: - return sc.vector(np.round((self.rotation_matrix * axis).values, 2)) - - @property - def fast_axis(self) -> sc.Variable: - if self._fast_axis is None: - self._fast_axis = self._rotate_axis(AXIS_TO_VECTOR[self.fast_axis_name]) - - return self._fast_axis - - @property - def slow_axis(self) -> sc.Variable: - if self._slow_axis is None: - self._slow_axis = self._rotate_axis(AXIS_TO_VECTOR[self.slow_axis_name]) - - return self._slow_axis - - def _retrieve_event_list_name(keys: Iterable[str]) -> str: prefix = "bank01_events_dat_list" @@ -146,142 +37,6 @@ def _copy_partial_var( return var -def _get_mcstas_pixel_ids(detector_descs: Tuple[DetectorDesc, ...]) -> PixelIDs: - """pixel IDs for each detector""" - intervals = [ - (desc.id_start, desc.id_start + desc.total_pixels) for desc in detector_descs - ] - ids = [sc.arange('id', start, stop, unit=None) for start, stop in intervals] - return PixelIDs(sc.concat(ids, 'id')) - - -def _pixel_positions(detector: DetectorDesc, sample_position: sc.Variable): - """pixel IDs for each detector""" - pixel_idx = sc.arange('id', detector.total_pixels) - n_rows = sc.scalar( - detector.num_x if detector.fast_axis_name == 'x' else detector.num_y - ) - steps = { - 'x': sc.scalar(detector.step_x, unit='m'), - 'y': sc.scalar(detector.step_y, unit='m'), - } - - pixel_n_slow = pixel_idx // n_rows - pixel_n_fast = pixel_idx % n_rows - - fast_axis_steps = detector.fast_axis * steps[detector.fast_axis_name] - slow_axis_steps = detector.slow_axis * steps[detector.slow_axis_name] - - return ( - (pixel_n_slow * slow_axis_steps) - + (pixel_n_fast * fast_axis_steps) - + (detector.position - sample_position) - ) - - -def _get_mcstas_pixel_positions( - detector_descs: Tuple[DetectorDesc, ...], sample_position -): - """pixel IDs for each detector""" - positions = [ - _pixel_positions(detector, sample_position) for detector in detector_descs - ] - return sc.concat(positions, 'panel') - - -def _read_mcstas_geometry_xml(file_path: InputFilepath) -> bytes: - """Retrieve geometry parameters from mcstas file""" - import h5py - - instrument_xml_path = 'entry1/instrument/instrument_xml/data' - with h5py.File(file_path) as file: - return file[instrument_xml_path][...][0] - - -def _select_by_type_prefix(components: list[_XML], prefix: str) -> list[_XML]: - """Select components by type prefix.""" - return [comp for comp in components if comp.attrib['type'].startswith(prefix)] - - -def _check_and_unpack_if_only_one(xml_items: list[_XML], name: str) -> _XML: - """Check if there is only one element with ``name``.""" - if len(xml_items) > 1: - raise ValueError(f"Multiple {name}s found.") - elif len(xml_items) == 0: - raise ValueError(f"No {name} found.") - - return xml_items.pop() - - -def _retrieve_attribs(component: _XML, *args: str) -> list[float]: - """Retrieve ``args`` from xml.""" - - return [float(component.attrib[key]) for key in args] - - -def find_location(component: _XML) -> _XML: - """Retrieve ``location`` from xml component.""" - location = component.find('location') - if location is None: - raise ValueError("No location found in component ", component.find('name')) - - return location - - -def _retrieve_3d_position(component: _XML) -> sc.Variable: - """Retrieve x, y, z position from xml.""" - location = find_location(component) - - return sc.vector(_retrieve_attribs(location, 'x', 'y', 'z'), unit='m') - - -def _retrieve_detector_descriptions(tree: _XML) -> Tuple[DetectorDesc, ...]: - """Retrieve detector geometry descriptions from mcstas file.""" - - def _retrieve_rotation_axis_angle(component: _XML) -> RotationAxisAngle: - """Retrieve rotation angle(theta), x, y, z axes from location.""" - location = find_location(component) - return RotationAxisAngle( - *_retrieve_attribs(location, 'rot', 'axis-x', 'axis-y', 'axis-z') - ) - - def _find_type_desc(det: _XML, types: list[_XML]) -> _XML: - for type_ in types: - if type_.attrib['name'] == det.attrib['type']: - return type_ - - raise ValueError( - f"Cannot find type {det.attrib['type']} for {det.attrib['name']}." - ) - - components = [branch for branch in tree if branch.tag == 'component'] - detectors = [ - comp for comp in components if comp.attrib['type'].startswith('MonNDtype') - ] - type_list = [branch for branch in tree if branch.tag == 'type'] - - detector_components = [] - for det in detectors: - det_type = _find_type_desc(det, type_list) - - detector_components.append( - DetectorDesc( - component_type=det_type.attrib['name'], - name=det.attrib['name'], - id_start=int(det.attrib['idstart']), - fast_axis_name=det.attrib['idfillbyfirst'], - position=_retrieve_3d_position(det), - rotation=RotationAxisAngle(*_retrieve_rotation_axis_angle(det)), - num_x=int(det_type.attrib['xpixels']), - num_y=int(det_type.attrib['ypixels']), - step_x=float(det_type.attrib['xstep']), - step_y=float(det_type.attrib['ystep']), - ) - ) - - return tuple(sorted(detector_components, key=lambda x: x.id_start)) - - def load_mcstas_nexus( file_path: InputFilepath, max_probability: Optional[MaximumProbability] = None, @@ -297,21 +52,10 @@ def load_mcstas_nexus( The maximum probability to scale the weights. """ - from defusedxml.ElementTree import fromstring - tree = fromstring(_read_mcstas_geometry_xml(file_path)) - detector_descs = _retrieve_detector_descriptions(tree) - components = [branch for branch in tree if branch.tag == 'component'] - sources = _select_by_type_prefix(components, 'sourceMantid-type') - samples = _select_by_type_prefix(components, 'sampleMantid-type') - source = _check_and_unpack_if_only_one(sources, 'source') - sample = _check_and_unpack_if_only_one(samples, 'sample') - sample_position = _retrieve_3d_position(sample) - - slow_axes = [det.slow_axis for det in detector_descs] - fast_axes = [det.fast_axis for det in detector_descs] - origins = [det.position - sample_position for det in detector_descs] + from .mcstas_xml import read_mcstas_geometry_xml + geometry = read_mcstas_geometry_xml(file_path) probability = max_probability or DefaultMaximumProbability with snx.File(file_path) as file: @@ -328,15 +72,9 @@ def load_mcstas_nexus( weights = (probability / weights.max()) * weights loaded = sc.DataArray(data=weights, coords={'t': t_list, 'id': id_list}) - grouped = loaded.group(_get_mcstas_pixel_ids(detector_descs)) - da = grouped.fold(dim='id', sizes={'panel': len(detector_descs), 'id': -1}) - da.coords['fast_axis'] = sc.concat(fast_axes, 'panel') - da.coords['slow_axis'] = sc.concat(slow_axes, 'panel') - da.coords['origin_position'] = sc.concat(origins, 'panel') - da.coords['position'] = _get_mcstas_pixel_positions( - detector_descs, sample_position - ) - da.coords['sample_position'] = sample_position - sample_position - da.coords['source_position'] = _retrieve_3d_position(source) - sample_position + coords = geometry.to_coords() + grouped = loaded.group(coords.pop('pixel_ids')) + da = grouped.fold(dim='id', sizes={'panel': len(geometry.detectors), 'id': -1}) + da.coords.update(coords) return NMXData(da) diff --git a/packages/essnmx/src/ess/nmx/mcstas_xml.py b/packages/essnmx/src/ess/nmx/mcstas_xml.py new file mode 100644 index 00000000..4ba79efa --- /dev/null +++ b/packages/essnmx/src/ess/nmx/mcstas_xml.py @@ -0,0 +1,393 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +# McStas instrument geometry xml description related functions. +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Iterable, Optional, Protocol, Tuple, TypeVar, Union + +import numpy as np +import scipp as sc +from typing_extensions import Self + +T = TypeVar('T') + + +_AXISNAME_TO_UNIT_VECTOR = MappingProxyType( + { + 'x': sc.vector([1.0, 0.0, 0.0]), + 'y': sc.vector([0.0, 1.0, 0.0]), + 'z': sc.vector([0.0, 0.0, 1.0]), + } +) + + +class _XML(Protocol): + """XML element or tree type. + + Temporarily used for type hinting. + Builtin XML type is blocked by bandit security check.""" + + tag: str + attrib: dict[str, str] + + def find(self, name: str) -> Optional[Self]: + ... + + def __iter__(self) -> Self: + ... + + def __next__(self) -> Self: + ... + + +def _check_and_unpack_if_only_one(xml_items: list[_XML], name: str) -> _XML: + """Check if there is only one element with ``name``.""" + if len(xml_items) > 1: + raise ValueError(f"Multiple {name}s found.") + elif len(xml_items) == 0: + raise ValueError(f"No {name} found.") + + return xml_items.pop() + + +def select_by_tag(xml_items: _XML, tag: str) -> _XML: + """Select element with ``tag`` if there is only one.""" + + return _check_and_unpack_if_only_one(list(filter_by_tag(xml_items, tag)), tag) + + +def filter_by_tag(xml_items: Iterable[_XML], tag: str) -> Iterable[_XML]: + """Filter xml items by tag.""" + return (item for item in xml_items if item.tag == tag) + + +def filter_by_type_prefix(xml_items: Iterable[_XML], prefix: str) -> Iterable[_XML]: + """Filter xml items by type prefix.""" + return ( + item for item in xml_items if item.attrib.get('type', '').startswith(prefix) + ) + + +def select_by_type_prefix(xml_items: Iterable[_XML], prefix: str) -> _XML: + """Select xml item by type prefix.""" + + cands = list(filter_by_type_prefix(xml_items, prefix)) + return _check_and_unpack_if_only_one(cands, prefix) + + +def find_attributes(component: _XML, *args: str) -> dict[str, float]: + """Retrieve ``args`` as float from xml.""" + + return {key: float(component.attrib[key]) for key in args} + + +@dataclass +class SimulationSettings: + """Simulation settings extracted from McStas instrument xml description.""" + + # From + length_unit: str # 'unit' of + angle_unit: str # 'unit' of + # From + beam_axis: str # 'axis' of + handedness: str # 'val' of + + @classmethod + def from_xml(cls, tree: _XML) -> Self: + """Create simulation settings from xml.""" + defaults = select_by_tag(tree, 'defaults') + length_desc = select_by_tag(defaults, 'length') + angle_desc = select_by_tag(defaults, 'angle') + reference_frame = select_by_tag(defaults, 'reference-frame') + along_beam = select_by_tag(reference_frame, 'along-beam') + handedness = select_by_tag(reference_frame, 'handedness') + + return cls( + length_unit=length_desc.attrib['unit'], + angle_unit=angle_desc.attrib['unit'], + beam_axis=along_beam.attrib['axis'], + handedness=handedness.attrib['val'], + ) + + +def _position_from_location(location: _XML, unit: str = 'm') -> sc.Variable: + """Retrieve position from location.""" + x, y, z = find_attributes(location, 'x', 'y', 'z').values() + return sc.vector([x, y, z], unit=unit) + + +def _rotation_matrix_from_location( + location: _XML, angle_unit: str = 'degree' +) -> sc.Variable: + """Retrieve rotation matrix from location.""" + from .rotation import axis_angle_to_quaternion, quaternion_to_matrix + + theta, x, y, z = find_attributes( + location, 'rot', 'axis-x', 'axis-y', 'axis-z' + ).values() + q = axis_angle_to_quaternion(x, y, z, sc.scalar(-theta, unit=angle_unit)) + return quaternion_to_matrix(*q) + + +@dataclass +class DetectorDesc: + """Detector information extracted from McStas instrument xml description.""" + + # From + component_type: str # 'type' + name: str + id_start: int # 'idstart' + fast_axis_name: str # 'idfillbyfirst' + # From + num_x: int # 'xpixels' + num_y: int # 'ypixels' + step_x: sc.Variable # 'xstep' + step_y: sc.Variable # 'ystep' + start_x: float # 'xstart' + start_y: float # 'ystart' + # From under + position: sc.Variable # 'x', 'y', 'z' + # Calculated fields + rotation_matrix: sc.Variable + slow_axis_name: str + fast_axis: sc.Variable + slow_axis: sc.Variable + + @classmethod + def from_xml( + cls, component: _XML, type_desc: _XML, simulation_settings: SimulationSettings + ) -> Self: + """Create detector description from xml component and type.""" + + def _rotate_axis(matrix: sc.Variable, axis: sc.Variable) -> sc.Variable: + return sc.vector(np.round((matrix * axis).values, 2)) + + location = select_by_tag(component, 'location') + rotation_matrix = _rotation_matrix_from_location( + location, simulation_settings.angle_unit + ) + fast_axis_name = component.attrib['idfillbyfirst'] + slow_axis_name = 'xy'.replace(fast_axis_name, '') + + length_unit = simulation_settings.length_unit + + return cls( + component_type=type_desc.attrib['name'], + name=component.attrib['name'], + id_start=int(component.attrib['idstart']), + fast_axis_name=fast_axis_name, + slow_axis_name=slow_axis_name, + num_x=int(type_desc.attrib['xpixels']), + num_y=int(type_desc.attrib['ypixels']), + step_x=sc.scalar(float(type_desc.attrib['xstep']), unit=length_unit), + step_y=sc.scalar(float(type_desc.attrib['ystep']), unit=length_unit), + start_x=float(type_desc.attrib['xstart']), + start_y=float(type_desc.attrib['ystart']), + position=_position_from_location(location, simulation_settings.length_unit), + rotation_matrix=rotation_matrix, + fast_axis=_rotate_axis( + rotation_matrix, _AXISNAME_TO_UNIT_VECTOR[fast_axis_name] + ), + slow_axis=_rotate_axis( + rotation_matrix, _AXISNAME_TO_UNIT_VECTOR[slow_axis_name] + ), + ) + + @property + def total_pixels(self) -> int: + return self.num_x * self.num_y + + @property + def slow_step(self) -> sc.Variable: + return self.step_y if self.fast_axis_name == 'x' else self.step_x + + @property + def fast_step(self) -> sc.Variable: + return self.step_x if self.fast_axis_name == 'x' else self.step_y + + @property + def num_fast_pixels_per_row(self) -> int: + """Number of pixels in each row of the detector along the fast axis.""" + return self.num_x if self.fast_axis_name == 'x' else self.num_y + + +def _collect_detector_descriptions(tree: _XML) -> Tuple[DetectorDesc, ...]: + """Retrieve detector geometry descriptions from mcstas file.""" + type_list = filter_by_tag(tree, 'type') + simulation_settings = SimulationSettings.from_xml(tree) + + def _find_type_desc(det: _XML) -> _XML: + for type_ in type_list: + if type_.attrib['name'] == det.attrib['type']: + return type_ + + raise ValueError( + f"Cannot find type {det.attrib['type']} for {det.attrib['name']}." + ) + + detector_components = [ + DetectorDesc.from_xml(det, _find_type_desc(det), simulation_settings) + for det in filter_by_type_prefix(filter_by_tag(tree, 'component'), 'MonNDtype') + ] + + return tuple(sorted(detector_components, key=lambda x: x.id_start)) + + +@dataclass +class SampleDesc: + """Sample description extracted from McStas instrument xml description.""" + + # From + component_type: str + name: str + # From under + position: sc.Variable + rotation_matrix: sc.Variable + + @classmethod + def from_xml(cls, tree: _XML, simulation_settings: SimulationSettings) -> Self: + """Create sample description from xml component.""" + source_xml = select_by_type_prefix(tree, 'sampleMantid-type') + location = select_by_tag(source_xml, 'location') + + return cls( + component_type=source_xml.attrib['type'], + name=source_xml.attrib['name'], + position=_position_from_location(location, simulation_settings.length_unit), + rotation_matrix=_rotation_matrix_from_location( + location, simulation_settings.angle_unit + ), + ) + + def position_from_sample(self, other: sc.Variable) -> sc.Variable: + """Position of ``other`` relative to the sample. + + All positions and distance are stored respect to the sample position. + + Parameters + ---------- + other: + Position of the other object in 3D vector. + + """ + + return other - self.position + + +@dataclass +class SourceDesc: + """Source description extracted from McStas instrument xml description.""" + + # From + component_type: str + name: str + # From under + position: sc.Variable + + @classmethod + def from_xml(cls, tree: _XML, simulation_settings: SimulationSettings) -> Self: + """Create source description from xml component.""" + source_xml = select_by_type_prefix(tree, 'sourceMantid-type') + location = select_by_tag(source_xml, 'location') + + return cls( + component_type=source_xml.attrib['type'], + name=source_xml.attrib['name'], + position=_position_from_location(location, simulation_settings.length_unit), + ) + + +def _construct_pixel_ids(detector_descs: Tuple[DetectorDesc, ...]) -> sc.Variable: + """Pixel IDs for all detectors.""" + intervals = [ + (desc.id_start, desc.id_start + desc.total_pixels) for desc in detector_descs + ] + ids = [sc.arange('id', start, stop, unit=None) for start, stop in intervals] + return sc.concat(ids, 'id') + + +def _pixel_positions( + detector: DetectorDesc, position_offset: sc.Variable +) -> sc.Variable: + """Position of pixels of the ``detector``. + + Position of each pixel is relative to the position_offset. + """ + pixel_idx = sc.arange('id', detector.total_pixels) + n_row = sc.scalar(detector.num_fast_pixels_per_row) + + pixel_n_slow = pixel_idx // n_row + pixel_n_fast = pixel_idx % n_row + + fast_axis_steps = detector.fast_axis * detector.fast_step + slow_axis_steps = detector.slow_axis * detector.slow_step + + return ( + (pixel_n_slow * slow_axis_steps) + + (pixel_n_fast * fast_axis_steps) + + position_offset + ) + + +def _detector_pixel_positions( + detector_descs: Tuple[DetectorDesc, ...], sample: SampleDesc +) -> sc.Variable: + """Position of pixels of all detectors.""" + + positions = [ + _pixel_positions(detector, sample.position_from_sample(detector.position)) + for detector in detector_descs + ] + return sc.concat(positions, 'panel') + + +@dataclass +class McStasInstrument: + simulation_settings: SimulationSettings + detectors: Tuple[DetectorDesc, ...] + source: SourceDesc + sample: SampleDesc + + @classmethod + def from_xml(cls, tree: _XML) -> Self: + """Create McStas instrument from xml.""" + simulation_settings = SimulationSettings.from_xml(tree) + + return cls( + simulation_settings=simulation_settings, + detectors=_collect_detector_descriptions(tree), + source=SourceDesc.from_xml(tree, simulation_settings), + sample=SampleDesc.from_xml(tree, simulation_settings), + ) + + def to_coords(self) -> dict[str, sc.Variable]: + """Extract coordinates from the McStas instrument description.""" + slow_axes = [det.slow_axis for det in self.detectors] + fast_axes = [det.fast_axis for det in self.detectors] + origins = [ + self.sample.position_from_sample(det.position) for det in self.detectors + ] + detector_dim = 'panel' + + return { + 'pixel_ids': _construct_pixel_ids(self.detectors), + 'fast_axis': sc.concat(fast_axes, detector_dim), + 'slow_axis': sc.concat(slow_axes, detector_dim), + 'origin_position': sc.concat(origins, detector_dim), + 'sample_position': self.sample.position_from_sample(self.sample.position), + 'source_position': self.sample.position_from_sample(self.source.position), + 'sample_name': sc.scalar(self.sample.name), + 'position': _detector_pixel_positions(self.detectors, self.sample), + } + + +def read_mcstas_geometry_xml(file_path: Union[Path, str]) -> McStasInstrument: + """Retrieve geometry parameters from mcstas file""" + import h5py + from defusedxml.ElementTree import fromstring + + instrument_xml_path = 'entry1/instrument/instrument_xml/data' + with h5py.File(file_path) as file: + tree = fromstring(file[instrument_xml_path][...][0]) + return McStasInstrument.from_xml(tree) From fc45a1972c7e55299cb64fb6cb97df9630e20b8a Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 9 Jan 2024 14:34:14 +0100 Subject: [PATCH 035/403] Use string-based type-hinting instead of self. --- packages/essnmx/docs/examples/workflow.ipynb | 3 ++- packages/essnmx/src/ess/nmx/mcstas_xml.py | 21 +++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index 5b475717..1f8977b7 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -9,7 +9,8 @@ "## Collect Parameters and Providers\n", "### Simulation(McStas) Data\n", "There is a dedicated loader, ``load_mcstas_nexus`` for ``McStas`` simulation data workflow.
\n", - "``MaximumProbability`` can be manually provided to the loader to derive more realistic number of events.
\n", + "``MaximumProbability`` can be manually provided to the loader
\n", + "to derive more realistic number of events.
\n", "It is because ``weights`` are given as probability, not number of events in a McStas file.
" ] }, diff --git a/packages/essnmx/src/ess/nmx/mcstas_xml.py b/packages/essnmx/src/ess/nmx/mcstas_xml.py index 4ba79efa..1464f6fd 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas_xml.py @@ -8,7 +8,6 @@ import numpy as np import scipp as sc -from typing_extensions import Self T = TypeVar('T') @@ -31,13 +30,13 @@ class _XML(Protocol): tag: str attrib: dict[str, str] - def find(self, name: str) -> Optional[Self]: + def find(self, name: str) -> Optional['_XML']: ... - def __iter__(self) -> Self: + def __iter__(self) -> '_XML': ... - def __next__(self) -> Self: + def __next__(self) -> '_XML': ... @@ -94,7 +93,7 @@ class SimulationSettings: handedness: str # 'val' of @classmethod - def from_xml(cls, tree: _XML) -> Self: + def from_xml(cls, tree: _XML) -> 'SimulationSettings': """Create simulation settings from xml.""" defaults = select_by_tag(tree, 'defaults') length_desc = select_by_tag(defaults, 'length') @@ -157,7 +156,7 @@ class DetectorDesc: @classmethod def from_xml( cls, component: _XML, type_desc: _XML, simulation_settings: SimulationSettings - ) -> Self: + ) -> 'DetectorDesc': """Create detector description from xml component and type.""" def _rotate_axis(matrix: sc.Variable, axis: sc.Variable) -> sc.Variable: @@ -246,7 +245,9 @@ class SampleDesc: rotation_matrix: sc.Variable @classmethod - def from_xml(cls, tree: _XML, simulation_settings: SimulationSettings) -> Self: + def from_xml( + cls, tree: _XML, simulation_settings: SimulationSettings + ) -> 'SampleDesc': """Create sample description from xml component.""" source_xml = select_by_type_prefix(tree, 'sampleMantid-type') location = select_by_tag(source_xml, 'location') @@ -286,7 +287,9 @@ class SourceDesc: position: sc.Variable @classmethod - def from_xml(cls, tree: _XML, simulation_settings: SimulationSettings) -> Self: + def from_xml( + cls, tree: _XML, simulation_settings: SimulationSettings + ) -> 'SourceDesc': """Create source description from xml component.""" source_xml = select_by_type_prefix(tree, 'sourceMantid-type') location = select_by_tag(source_xml, 'location') @@ -350,7 +353,7 @@ class McStasInstrument: sample: SampleDesc @classmethod - def from_xml(cls, tree: _XML) -> Self: + def from_xml(cls, tree: _XML) -> 'McStasInstrument': """Create McStas instrument from xml.""" simulation_settings = SimulationSettings.from_xml(tree) From c13233d30ab6ca9be108d6bd614cebe02d1ba2e3 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 9 Jan 2024 15:13:46 +0100 Subject: [PATCH 036/403] Add instrument view in the document. --- packages/essnmx/docs/examples/workflow.ipynb | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index 1f8977b7..1921a981 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -48,7 +48,9 @@ "source": [ "from typing import get_type_hints\n", "param_reprs = {key.__name__: value for key, value in params.items()}\n", - "prov_reprs = {get_type_hints(prov)['return'].__name__: prov.__name__ for prov in providers}\n", + "prov_reprs = {\n", + " get_type_hints(prov)['return'].__name__: prov.__name__ for prov in providers\n", + "}\n", "\n", "# Providers and parameters to be used for pipeline\n", "sc.DataGroup(**prov_reprs, **param_reprs)" @@ -99,9 +101,11 @@ "source": [ "## Instrument View\n", "\n", - "Pixel positions are not used for later steps, but it is included in the coordinates for instrument view.\n", + "Pixel positions are not used for later steps,\n", + "but it is included in the coordinates for instrument view.\n", "\n", - "All pixel positions are respect to the sample position, therefore the sample is at (0, 0, 0).\n", + "All pixel positions are respect to the sample position,\n", + "therefore the sample is at (0, 0, 0).\n", "\n", "**It might be very slow or not work in the ``VS Code`` jupyter notebook editor.**" ] @@ -114,7 +118,6 @@ "source": [ "import scippneutron as scn\n", "\n", - "\n", "unnecessary_coords = list(coord for coord in da.coords if coord != 'position')\n", "instrument_view_da = da.drop_coords(unnecessary_coords).flatten(['panel', 'id'], 'id').hist()\n", "view = scn.instrument_view(instrument_view_da)\n", From 7a440a9e39bd471658cd6d93d2997b3b0cbfd650 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 9 Jan 2024 15:21:56 +0100 Subject: [PATCH 037/403] Remove instrument view from the cell temporarily. --- packages/essnmx/docs/examples/workflow.ipynb | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index 1921a981..ac93f677 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -107,22 +107,19 @@ "All pixel positions are respect to the sample position,\n", "therefore the sample is at (0, 0, 0).\n", "\n", - "**It might be very slow or not work in the ``VS Code`` jupyter notebook editor.**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ + "You can plot the instrument view like below.\n", + "\n", + "```python\n", "import scippneutron as scn\n", "\n", "unnecessary_coords = list(coord for coord in da.coords if coord != 'position')\n", "instrument_view_da = da.drop_coords(unnecessary_coords).flatten(['panel', 'id'], 'id').hist()\n", "view = scn.instrument_view(instrument_view_da)\n", "view.children[0].toolbar.cameraz()\n", - "view" + "view\n", + "```\n", + "\n", + "**It might be very slow or not work in the ``VS Code`` jupyter notebook editor.**" ] } ], From 05ba5693e77a6f23450d28e2a83447183e250675 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 9 Jan 2024 15:36:49 +0100 Subject: [PATCH 038/403] Add expected coordinate values. --- packages/essnmx/tests/loader_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index cef49f20..bcf5dc5e 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -30,8 +30,23 @@ def test_file_reader_mcstas() -> None: assert isinstance(da, sc.DataArray) assert da.shape == (3, 1280 * 1280) + assert sc.identical( + da.coords['sample_position'], sc.vector(value=[0, 0, 0], unit='m') + ) assert da.bins.size().sum().value == data_length assert sc.identical(da.data.max(), expected_weight_max) + # Expected coordinate values are provided by the IDS + # based on the simulation settings of the sample file. + assert sc.identical( + da.coords['fast_axis'], + sc.vectors( + dims=['panel'], values=[[0, 0, -0.01], [-0.01, 0, -1], [0.01, 0, 1]] + ), + ) + assert sc.identical( + da.coords['slow_axis'], + sc.vectors(dims=['panel'], values=[[0, 1, 0], [0, 1, 0], [0, 1, 0]]), + ) @pytest.fixture From 39c417e8645397bf75bd2ebbac51da5face1b3d6 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 9 Jan 2024 15:44:36 +0100 Subject: [PATCH 039/403] Add expected coordinate values. --- packages/essnmx/tests/loader_test.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index bcf5dc5e..84be0de6 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -40,12 +40,15 @@ def test_file_reader_mcstas() -> None: assert sc.identical( da.coords['fast_axis'], sc.vectors( - dims=['panel'], values=[[0, 0, -0.01], [-0.01, 0, -1], [0.01, 0, 1]] + dims=['panel'], + values=[(1.0, 0.0, -0.01), (-0.01, 0.0, -1.0), (0.01, 0.0, 1.0)], ), ) assert sc.identical( da.coords['slow_axis'], - sc.vectors(dims=['panel'], values=[[0, 1, 0], [0, 1, 0], [0, 1, 0]]), + sc.vectors( + dims=['panel'], values=[[0.0, 1.0, 0.0], [0.0, 1.0, 0.0], [0.0, 1.0, 0.0]] + ), ) From 22fab9fcbd29b4233766f11613bbb49192ceca72 Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Wed, 10 Jan 2024 09:05:43 +0100 Subject: [PATCH 040/403] Fix typos and update names. Co-authored-by: Simon Heybrock <12912489+SimonHeybrock@users.noreply.github.com> --- packages/essnmx/docs/examples/workflow.ipynb | 2 +- packages/essnmx/src/ess/nmx/mcstas_xml.py | 4 ++-- packages/essnmx/src/ess/nmx/rotation.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index ac93f677..709bd1fc 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -104,7 +104,7 @@ "Pixel positions are not used for later steps,\n", "but it is included in the coordinates for instrument view.\n", "\n", - "All pixel positions are respect to the sample position,\n", + "All pixel positions are relative to the sample position,\n", "therefore the sample is at (0, 0, 0).\n", "\n", "You can plot the instrument view like below.\n", diff --git a/packages/essnmx/src/ess/nmx/mcstas_xml.py b/packages/essnmx/src/ess/nmx/mcstas_xml.py index 1464f6fd..2a266c6b 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas_xml.py @@ -264,7 +264,7 @@ def from_xml( def position_from_sample(self, other: sc.Variable) -> sc.Variable: """Position of ``other`` relative to the sample. - All positions and distance are stored respect to the sample position. + All positions and distance are stored relative to the sample position. Parameters ---------- @@ -374,7 +374,7 @@ def to_coords(self) -> dict[str, sc.Variable]: detector_dim = 'panel' return { - 'pixel_ids': _construct_pixel_ids(self.detectors), + 'pixel_id': _construct_pixel_ids(self.detectors), 'fast_axis': sc.concat(fast_axes, detector_dim), 'slow_axis': sc.concat(slow_axes, detector_dim), 'origin_position': sc.concat(origins, detector_dim), diff --git a/packages/essnmx/src/ess/nmx/rotation.py b/packages/essnmx/src/ess/nmx/rotation.py index 98beab8c..c3dd04e1 100644 --- a/packages/essnmx/src/ess/nmx/rotation.py +++ b/packages/essnmx/src/ess/nmx/rotation.py @@ -25,7 +25,7 @@ def axis_angle_to_quaternion( Returns ------- : - A list of (normalized) queternions, [x, y, z, w]. + A list of (normalized) quaternions, [x, y, z, w]. Notes ----- @@ -57,7 +57,7 @@ def quaternion_to_matrix(x: float, y: float, z: float, w: float) -> sc.Variable: Returns ------- : - A 3X3 rotation matrix (3 vectors). + A 3x3 rotation matrix. """ from scipy.spatial.transform import Rotation From 20b04d58b0a45d8979ec9cd33b0c4891896aabe6 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 10 Jan 2024 09:37:11 +0100 Subject: [PATCH 041/403] Remove rounding and make rotation function arguments keyword-only. --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 2 +- packages/essnmx/src/ess/nmx/mcstas_xml.py | 46 +++++++++++++------- packages/essnmx/src/ess/nmx/rotation.py | 4 +- packages/essnmx/tests/loader_test.py | 10 +++-- 4 files changed, 39 insertions(+), 23 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 92dbf556..c56ed287 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -73,7 +73,7 @@ def load_mcstas_nexus( loaded = sc.DataArray(data=weights, coords={'t': t_list, 'id': id_list}) coords = geometry.to_coords() - grouped = loaded.group(coords.pop('pixel_ids')) + grouped = loaded.group(coords.pop('pixel_id')) da = grouped.fold(dim='id', sizes={'panel': len(geometry.detectors), 'id': -1}) da.coords.update(coords) diff --git a/packages/essnmx/src/ess/nmx/mcstas_xml.py b/packages/essnmx/src/ess/nmx/mcstas_xml.py index 2a266c6b..f73a6fb0 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas_xml.py @@ -6,7 +6,6 @@ from types import MappingProxyType from typing import Iterable, Optional, Protocol, Tuple, TypeVar, Union -import numpy as np import scipp as sc T = TypeVar('T') @@ -122,11 +121,14 @@ def _rotation_matrix_from_location( """Retrieve rotation matrix from location.""" from .rotation import axis_angle_to_quaternion, quaternion_to_matrix - theta, x, y, z = find_attributes( - location, 'rot', 'axis-x', 'axis-y', 'axis-z' - ).values() - q = axis_angle_to_quaternion(x, y, z, sc.scalar(-theta, unit=angle_unit)) - return quaternion_to_matrix(*q) + attribs = find_attributes(location, 'axis-x', 'axis-y', 'axis-z', 'rot') + x, y, z, w = axis_angle_to_quaternion( + x=attribs['axis-x'], + y=attribs['axis-y'], + z=attribs['axis-z'], + theta=sc.scalar(-attribs['rot'], unit=angle_unit), + ) + return quaternion_to_matrix(x=x, y=y, z=z, w=w) @dataclass @@ -155,12 +157,16 @@ class DetectorDesc: @classmethod def from_xml( - cls, component: _XML, type_desc: _XML, simulation_settings: SimulationSettings + cls, + *, + component: _XML, + type_desc: _XML, + simulation_settings: SimulationSettings, ) -> 'DetectorDesc': """Create detector description from xml component and type.""" def _rotate_axis(matrix: sc.Variable, axis: sc.Variable) -> sc.Variable: - return sc.vector(np.round((matrix * axis).values, 2)) + return matrix * axis location = select_by_tag(component, 'location') rotation_matrix = _rotation_matrix_from_location( @@ -226,7 +232,11 @@ def _find_type_desc(det: _XML) -> _XML: ) detector_components = [ - DetectorDesc.from_xml(det, _find_type_desc(det), simulation_settings) + DetectorDesc.from_xml( + component=det, + type_desc=_find_type_desc(det), + simulation_settings=simulation_settings, + ) for det in filter_by_type_prefix(filter_by_tag(tree, 'component'), 'MonNDtype') ] @@ -246,7 +256,7 @@ class SampleDesc: @classmethod def from_xml( - cls, tree: _XML, simulation_settings: SimulationSettings + cls, *, tree: _XML, simulation_settings: SimulationSettings ) -> 'SampleDesc': """Create sample description from xml component.""" source_xml = select_by_type_prefix(tree, 'sampleMantid-type') @@ -288,7 +298,7 @@ class SourceDesc: @classmethod def from_xml( - cls, tree: _XML, simulation_settings: SimulationSettings + cls, *, tree: _XML, simulation_settings: SimulationSettings ) -> 'SourceDesc': """Create source description from xml component.""" source_xml = select_by_type_prefix(tree, 'sourceMantid-type') @@ -318,10 +328,10 @@ def _pixel_positions( Position of each pixel is relative to the position_offset. """ pixel_idx = sc.arange('id', detector.total_pixels) - n_row = sc.scalar(detector.num_fast_pixels_per_row) + n_col = sc.scalar(detector.num_fast_pixels_per_row) - pixel_n_slow = pixel_idx // n_row - pixel_n_fast = pixel_idx % n_row + pixel_n_slow = pixel_idx // n_col + pixel_n_fast = pixel_idx % n_col fast_axis_steps = detector.fast_axis * detector.fast_step slow_axis_steps = detector.slow_axis * detector.slow_step @@ -360,8 +370,12 @@ def from_xml(cls, tree: _XML) -> 'McStasInstrument': return cls( simulation_settings=simulation_settings, detectors=_collect_detector_descriptions(tree), - source=SourceDesc.from_xml(tree, simulation_settings), - sample=SampleDesc.from_xml(tree, simulation_settings), + source=SourceDesc.from_xml( + tree=tree, simulation_settings=simulation_settings + ), + sample=SampleDesc.from_xml( + tree=tree, simulation_settings=simulation_settings + ), ) def to_coords(self) -> dict[str, sc.Variable]: diff --git a/packages/essnmx/src/ess/nmx/rotation.py b/packages/essnmx/src/ess/nmx/rotation.py index c3dd04e1..4ec91c84 100644 --- a/packages/essnmx/src/ess/nmx/rotation.py +++ b/packages/essnmx/src/ess/nmx/rotation.py @@ -7,7 +7,7 @@ def axis_angle_to_quaternion( - x: float, y: float, z: float, theta: sc.Variable + *, x: float, y: float, z: float, theta: sc.Variable ) -> NDArray: """Convert axis-angle to queternions, [x, y, z, w]. @@ -40,7 +40,7 @@ def axis_angle_to_quaternion( return q / np.linalg.norm(q) -def quaternion_to_matrix(x: float, y: float, z: float, w: float) -> sc.Variable: +def quaternion_to_matrix(*, x: float, y: float, z: float, w: float) -> sc.Variable: """Convert quaternion to rotation matrix. Parameters diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 84be0de6..2b969c8a 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -10,6 +10,7 @@ def test_file_reader_mcstas() -> None: + import numpy as np import scippnexus as snx from ess.nmx.mcstas_loader import ( @@ -37,12 +38,13 @@ def test_file_reader_mcstas() -> None: assert sc.identical(da.data.max(), expected_weight_max) # Expected coordinate values are provided by the IDS # based on the simulation settings of the sample file. - assert sc.identical( - da.coords['fast_axis'], - sc.vectors( + # The expected values are rounded to 2 decimal places. + assert np.all( + np.round(da.coords['fast_axis'].values, 2) + == sc.vectors( dims=['panel'], values=[(1.0, 0.0, -0.01), (-0.01, 0.0, -1.0), (0.01, 0.0, 1.0)], - ), + ).values, ) assert sc.identical( da.coords['slow_axis'], From 8a48be6566295736fa9a16f6a750efe51ecd84ec Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 10 Jan 2024 11:47:38 +0100 Subject: [PATCH 042/403] Remove unnecessary helper. --- packages/essnmx/src/ess/nmx/mcstas_xml.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_xml.py b/packages/essnmx/src/ess/nmx/mcstas_xml.py index f73a6fb0..d981a30e 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas_xml.py @@ -165,9 +165,6 @@ def from_xml( ) -> 'DetectorDesc': """Create detector description from xml component and type.""" - def _rotate_axis(matrix: sc.Variable, axis: sc.Variable) -> sc.Variable: - return matrix * axis - location = select_by_tag(component, 'location') rotation_matrix = _rotation_matrix_from_location( location, simulation_settings.angle_unit @@ -191,12 +188,8 @@ def _rotate_axis(matrix: sc.Variable, axis: sc.Variable) -> sc.Variable: start_y=float(type_desc.attrib['ystart']), position=_position_from_location(location, simulation_settings.length_unit), rotation_matrix=rotation_matrix, - fast_axis=_rotate_axis( - rotation_matrix, _AXISNAME_TO_UNIT_VECTOR[fast_axis_name] - ), - slow_axis=_rotate_axis( - rotation_matrix, _AXISNAME_TO_UNIT_VECTOR[slow_axis_name] - ), + fast_axis=rotation_matrix * _AXISNAME_TO_UNIT_VECTOR[fast_axis_name], + slow_axis=rotation_matrix * _AXISNAME_TO_UNIT_VECTOR[slow_axis_name], ) @property From 924a07d708aa939ab06ca23fdf2e6c72f5212916 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 11 Jan 2024 10:57:01 +0100 Subject: [PATCH 043/403] Time binning reduction and export method. --- packages/essnmx/docs/examples/workflow.ipynb | 44 +++++- packages/essnmx/src/ess/nmx/__init__.py | 3 + packages/essnmx/src/ess/nmx/mcstas_loader.py | 22 ++- packages/essnmx/src/ess/nmx/reduction.py | 151 +++++++++++++++++++ packages/essnmx/tests/loader_test.py | 24 ++- packages/essnmx/tests/workflow_test.py | 18 ++- 6 files changed, 234 insertions(+), 28 deletions(-) create mode 100644 packages/essnmx/src/ess/nmx/reduction.py diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index 709bd1fc..763a120b 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -29,11 +29,13 @@ " DefaultMaximumProbability\n", ")\n", "from ess.nmx.data import small_mcstas_sample\n", + "from ess.nmx.reduction import bin_time_of_arrival, NMXReducedData, TimeBinSteps\n", "\n", - "providers = (load_mcstas_nexus, )\n", + "providers = (load_mcstas_nexus, bin_time_of_arrival, )\n", "\n", "file_path = small_mcstas_sample() # Replace it with your data file path\n", "params = {\n", + " TimeBinSteps: TimeBinSteps(50),\n", " InputFilepath: InputFilepath(file_path),\n", " # Additional parameters for McStas data handling.\n", " MaximumProbability: DefaultMaximumProbability,\n", @@ -71,9 +73,10 @@ "source": [ "import sciline as sl\n", "from ess.nmx.mcstas_loader import NMXData\n", + "from ess.nmx.reduction import NMXReducedData\n", "\n", "nmx_pl = sl.Pipeline(list(providers), params=params)\n", - "nmx_workflow = nmx_pl.get(NMXData)\n", + "nmx_workflow = nmx_pl.get(NMXReducedData)\n", "nmx_workflow.visualize()" ] }, @@ -91,8 +94,35 @@ "outputs": [], "source": [ "# Event data grouped by detector panel and pixel id.\n", - "da = nmx_workflow.compute(NMXData)\n", - "da" + "dg = nmx_workflow.compute(NMXData)\n", + "dg" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Binned data.\n", + "\n", + "binned_dg = nmx_workflow.compute(NMXReducedData)\n", + "binned_dg" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Export Results\n", + "\n", + "``NMXReducedData`` object has a method to export the data into nexus or h5 file.\n", + "\n", + "You can save the result as ``test.nxs`` for example.\n", + "\n", + "```python\n", + "binned_dg.export_as_nexus('test.nxs')\n", + "```" ] }, { @@ -112,9 +142,9 @@ "```python\n", "import scippneutron as scn\n", "\n", - "unnecessary_coords = list(coord for coord in da.coords if coord != 'position')\n", - "instrument_view_da = da.drop_coords(unnecessary_coords).flatten(['panel', 'id'], 'id').hist()\n", - "view = scn.instrument_view(instrument_view_da)\n", + "da = dg['weights']\n", + "da.coords['position'] = dg['position']\n", + "view = scn.instrument_view(da.flatten(['panel', 'id'], 'id').hist())\n", "view.children[0].toolbar.cameraz()\n", "view\n", "```\n", diff --git a/packages/essnmx/src/ess/nmx/__init__.py b/packages/essnmx/src/ess/nmx/__init__.py index af30da5d..f8440287 100644 --- a/packages/essnmx/src/ess/nmx/__init__.py +++ b/packages/essnmx/src/ess/nmx/__init__.py @@ -13,10 +13,13 @@ from .data import small_mcstas_sample from .mcstas_loader import InputFilepath, NMXData, load_mcstas_nexus +from .reduction import NMXReducedData, TimeBinSteps __all__ = [ "small_mcstas_sample", "NMXData", "InputFilepath", "load_mcstas_nexus", + "NMXReducedData", + "TimeBinSteps", ] diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index c56ed287..5dca05da 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -5,9 +5,11 @@ import scipp as sc import scippnexus as snx +from .reduction import NMXData + +_PROTON_CHARGE_SCALE_FACTOR = 1 / 10_000 # Arbitrary number to scale the proton charge PixelIDs = NewType("PixelIDs", sc.Variable) InputFilepath = NewType("InputFilepath", str) -NMXData = NewType("NMXData", sc.DataGroup) # McStas Configurations MaximumProbability = NewType("MaximumProbability", int) @@ -73,8 +75,16 @@ def load_mcstas_nexus( loaded = sc.DataArray(data=weights, coords={'t': t_list, 'id': id_list}) coords = geometry.to_coords() - grouped = loaded.group(coords.pop('pixel_id')) - da = grouped.fold(dim='id', sizes={'panel': len(geometry.detectors), 'id': -1}) - da.coords.update(coords) - - return NMXData(da) + grouped: sc.DataArray = loaded.group(coords.pop('pixel_id')) + da: sc.DataArray = grouped.fold( + dim='id', sizes={'panel': len(geometry.detectors), 'id': -1} + ) + # Proton charge is proportional to the number of neutrons, + # which is proportional to the number of events. + # The scale factor is chosen by previous results + # to be convenient for data manipulation in the next steps. + # It is derived this way since + # the protons are not part of McStas simulation, + # and the number of neutrons is not included in the result. + proton_charge = _PROTON_CHARGE_SCALE_FACTOR * da.bins.size().sum().value + return NMXData(sc.DataGroup(weights=da, proton_charge=proton_charge, **coords)) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py new file mode 100644 index 00000000..16665b69 --- /dev/null +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -0,0 +1,151 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +import pathlib +from typing import NewType, Union + +import h5py +import scipp as sc + +TimeBinSteps = NewType("TimeBinSteps", int) + + +class NMXData(sc.DataGroup): + @property + def weights(self) -> sc.DataArray: + return self['weights'] + + @property + def origin_position(self) -> sc.Variable: + return self['origin_position'] + + @property + def sample_name(self) -> sc.Variable: + return self['sample_name'] + + @property + def fast_axis(self) -> sc.Variable: + return self['fast_axis'] + + @property + def slow_axis(self) -> sc.Variable: + return self['slow_axis'] + + @property + def proton_charge(self) -> float: + return self['proton_charge'] + + @property + def source_position(self) -> sc.Variable: + return self['source_position'] + + @property + def sample_position(self) -> sc.Variable: + return self['sample_position'] + + +class NMXReducedData(NMXData): + @property + def counts(self) -> sc.DataArray: + return self['counts'] + + def _create_root_data_entry(self, file_obj: h5py.File) -> h5py.Group: + nx_entry = file_obj.create_group("NMX_data") + nx_entry.attrs["NX_class"] = "NXentry" + nx_entry.attrs["default"] = "data" + nx_entry.attrs["name"] = "NMX" + nx_entry["name"] = "NMX" + nx_entry["definition"] = "TOFRAW" + return nx_entry + + def _create_sample_group(self, nx_entry: h5py.Group) -> h5py.Group: + nx_sample = nx_entry.create_group("NXsample") + nx_sample["name"] = self.sample_name.value + return nx_sample + + def _create_instrument_group(self, nx_entry: h5py.Group) -> h5py.Group: + nx_instrument = nx_entry.create_group("NXinstrument") + nx_instrument.attrs["nr_detector"] = self.origin_position.sizes['panel'] + nx_instrument.create_dataset("proton_charge", data=self.proton_charge) + + nx_detector_1 = nx_instrument.create_group("detector_1") + counts = nx_detector_1.create_dataset( + "counts", data=self.counts.values, compression="gzip", compression_opts=4 + ) + counts.attrs["units"] = "counts" + t_spectrum = nx_detector_1.create_dataset( + "t_bin", + data=self.counts.coords['t'].values, + compression="gzip", + compression_opts=4, + ) + t_spectrum.attrs["units"] = "s" + t_spectrum.attrs["long_name"] = "t_bin TOF (ms)" + pixel_id = nx_detector_1.create_dataset( + "pixel_id", + data=self.counts.coords['id'].values, + compression="gzip", + compression_opts=4, + ) + pixel_id.attrs["units"] = "" + pixel_id.attrs["long_name"] = "pixel ID" + return nx_instrument + + def _create_detector_group(self, nx_entry: h5py.Group) -> h5py.Group: + nx_detector = nx_entry.create_group("NXdetector") + # Position of the first pixel (lowest ID) in the detector + detector_origins = nx_detector.create_dataset( + "origin", + data=self.origin_position.values, + compression="gzip", + compression_opts=4, + ) + detector_origins.attrs["units"] = "m" + # Fast axis, along where the pixel ID increases by 1 + nx_detector.create_dataset("fast_axis", data=self.fast_axis.values) + # Slow axis, along where the pixel ID increases + # by the number of pixels in the fast axis + nx_detector.create_dataset("slow_axis", data=self.slow_axis.values) + return nx_detector + + def _create_source_group(self, nx_entry: h5py.Group) -> h5py.Group: + nx_source = nx_entry.create_group("NXsource") + nx_source["name"] = "European Spallation Source" + nx_source["short_name"] = "ESS" + nx_source["type"] = "Spallation Neutron Source" + nx_source["distance"] = sc.norm(self.source_position).value + nx_source["probe"] = "neutron" + nx_source["target_material"] = "W" + return nx_source + + def export_as_nexus(self, output_file_name: Union[str, pathlib.Path]) -> None: + import h5py + + file_name = pathlib.Path(output_file_name) + if file_name.suffix not in (".h5", ".nxs"): + raise ValueError("Output file name must end with .h5 or .nxs") + + with h5py.File(file_name, "w") as out_file: + out_file.attrs["default"] = "NMX_data" + # Root Data Entry + nx_entry = self._create_root_data_entry(out_file) + # Sample + self._create_sample_group(nx_entry) + # Instrument + self._create_instrument_group(nx_entry) + # Detector + self._create_detector_group(nx_entry) + # Source + self._create_source_group(nx_entry) + + +def bin_time_of_arrival( + nmx_data: NMXData, time_bin_step: TimeBinSteps +) -> NMXReducedData: + """Bin time of arrival data into ``time_bin_step`` bins.""" + + return NMXReducedData( + counts=nmx_data.weights.flatten(dims=['panel', 'id'], to='id').hist( + t=time_bin_step + ), + **{key: nmx_data[key] for key in nmx_data.keys() if key != 'weights'}, + ) diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 2b969c8a..16fd5a13 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -20,7 +20,7 @@ def test_file_reader_mcstas() -> None: ) file_path = InputFilepath(small_mcstas_sample()) - da = load_mcstas_nexus(file_path) + dg = load_mcstas_nexus(file_path) entry_path = "entry1/data/bank01_events_dat_list_p_x_y_n_id_t" with snx.File(file_path) as file: @@ -29,25 +29,23 @@ def test_file_reader_mcstas() -> None: expected_weight_max = sc.scalar(DefaultMaximumProbability, unit='1', dtype=float) - assert isinstance(da, sc.DataArray) - assert da.shape == (3, 1280 * 1280) - assert sc.identical( - da.coords['sample_position'], sc.vector(value=[0, 0, 0], unit='m') - ) - assert da.bins.size().sum().value == data_length - assert sc.identical(da.data.max(), expected_weight_max) + assert isinstance(dg, sc.DataGroup) + assert dg.shape == (3, 1280 * 1280) + assert sc.identical(dg.sample_position, sc.vector(value=[0, 0, 0], unit='m')) + assert dg.weights.bins.size().sum().value == data_length + assert sc.identical(dg.weights.max().data, expected_weight_max) # Expected coordinate values are provided by the IDS # based on the simulation settings of the sample file. # The expected values are rounded to 2 decimal places. assert np.all( - np.round(da.coords['fast_axis'].values, 2) + np.round(dg.fast_axis.values, 2) == sc.vectors( dims=['panel'], values=[(1.0, 0.0, -0.01), (-0.01, 0.0, -1.0), (0.01, 0.0, 1.0)], ).values, ) assert sc.identical( - da.coords['slow_axis'], + dg.slow_axis, sc.vectors( dims=['panel'], values=[[0.0, 1.0, 0.0], [0.0, 1.0, 0.0], [0.0, 1.0, 0.0]] ), @@ -79,7 +77,7 @@ def test_file_reader_mcstas_additional_fields(tmp_mcstas_file: pathlib.Path) -> del file[entry_path] file[new_entry_path] = dataset - da = load_mcstas_nexus(InputFilepath(str(tmp_mcstas_file))) + dg = load_mcstas_nexus(InputFilepath(str(tmp_mcstas_file))) - assert isinstance(da, sc.DataArray) - assert da.shape == (3, 1280 * 1280) + assert isinstance(dg, sc.DataGroup) + assert dg.shape == (3, 1280 * 1280) diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index 7f00b28e..3eaa9dc5 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -14,12 +14,14 @@ def mcstas_workflow() -> sl.Pipeline: MaximumProbability, load_mcstas_nexus, ) + from ess.nmx.reduction import TimeBinSteps, bin_time_of_arrival return sl.Pipeline( - [load_mcstas_nexus], + [load_mcstas_nexus, bin_time_of_arrival], params={ InputFilepath: small_mcstas_sample(), MaximumProbability: DefaultMaximumProbability, + TimeBinSteps: TimeBinSteps(50), }, ) @@ -36,4 +38,16 @@ def test_pipeline_mcstas_loader(mcstas_workflow: sl.Pipeline) -> None: from ess.nmx.mcstas_loader import NMXData nmx_data = mcstas_workflow.compute(NMXData) - assert isinstance(nmx_data, sc.DataArray) + assert isinstance(nmx_data, sc.DataGroup) + assert nmx_data.sizes['panel'] == 3 + assert nmx_data.sizes['id'] == 1280 * 1280 + + +def test_pipeline_mcstas_reduction(mcstas_workflow: sl.Pipeline) -> None: + """Test if the loader graph is complete.""" + from ess.nmx.reduction import NMXReducedData + + nmx_reduced_data = mcstas_workflow.compute(NMXReducedData) + assert isinstance(nmx_reduced_data, sc.DataGroup) + assert nmx_reduced_data.sizes['panel'] == 3 + assert nmx_reduced_data.sizes['t'] == 50 From cbeeef7fcae35d5ee1f1d579593e33dbad7acac0 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 11 Jan 2024 11:29:19 +0100 Subject: [PATCH 044/403] Split shared and separate fields and add docstring to each field. --- packages/essnmx/src/ess/nmx/reduction.py | 29 ++++++++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 16665b69..0f48ea8f 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -9,13 +9,17 @@ TimeBinSteps = NewType("TimeBinSteps", int) -class NMXData(sc.DataGroup): - @property - def weights(self) -> sc.DataArray: - return self['weights'] +class _SharedFields(sc.DataGroup): + """All shared fields between NMXData and NMXReducedData. + + ``weights`` is only present in NMXData due to potential memory issues. + """ @property def origin_position(self) -> sc.Variable: + """Position of the first pixel (lowest ID) in the detector. + + Relative position from the sample.""" return self['origin_position'] @property @@ -24,28 +28,43 @@ def sample_name(self) -> sc.Variable: @property def fast_axis(self) -> sc.Variable: + """Fast axis, along where the pixel ID increases by 1.""" return self['fast_axis'] @property def slow_axis(self) -> sc.Variable: + """Slow axis, along where the pixel ID increases by > 1. + + The pixel ID increases by the number of pixels in the fast axis.""" return self['slow_axis'] @property def proton_charge(self) -> float: + """Accumulated number of protons during the measurement.""" return self['proton_charge'] @property def source_position(self) -> sc.Variable: + """Relative position of the source from the sample.""" return self['source_position'] @property def sample_position(self) -> sc.Variable: + """Relative position of the sample from the sample. (0, 0, 0)""" return self['sample_position'] -class NMXReducedData(NMXData): +class NMXData(_SharedFields, sc.DataGroup): + @property + def weights(self) -> sc.DataArray: + """Event data grouped by pixel id and panel.""" + return self['weights'] + + +class NMXReducedData(_SharedFields, sc.DataGroup): @property def counts(self) -> sc.DataArray: + """Binned time of arrival data from flattened event data.""" return self['counts'] def _create_root_data_entry(self, file_obj: h5py.File) -> h5py.Group: From 7b79dcb50daef6314bab6bc77d84cc7e01cf4f27 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 11 Jan 2024 13:47:22 +0100 Subject: [PATCH 045/403] Load and export crystal rotation angle. --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 21 +++++++++++++++++++- packages/essnmx/src/ess/nmx/reduction.py | 12 +++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 5dca05da..60aeefc7 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -39,6 +39,15 @@ def _copy_partial_var( return var +def _retrieve_crystal_rotation(file: snx.File, unit: str) -> sc.Variable: + """Retrieve crystal rotation from the file.""" + + return sc.vector( + value=[file[f"entry1/simulation/Param/XtalPhi{key}"][...] for key in "XYZ"], + unit=unit, + ) + + def load_mcstas_nexus( file_path: InputFilepath, max_probability: Optional[MaximumProbability] = None, @@ -79,6 +88,9 @@ def load_mcstas_nexus( da: sc.DataArray = grouped.fold( dim='id', sizes={'panel': len(geometry.detectors), 'id': -1} ) + crystal_rotation = _retrieve_crystal_rotation( + file, geometry.simulation_settings.angle_unit + ) # Proton charge is proportional to the number of neutrons, # which is proportional to the number of events. # The scale factor is chosen by previous results @@ -87,4 +99,11 @@ def load_mcstas_nexus( # the protons are not part of McStas simulation, # and the number of neutrons is not included in the result. proton_charge = _PROTON_CHARGE_SCALE_FACTOR * da.bins.size().sum().value - return NMXData(sc.DataGroup(weights=da, proton_charge=proton_charge, **coords)) + return NMXData( + sc.DataGroup( + weights=da, + proton_charge=proton_charge, + crystal_rotation=crystal_rotation, + **coords, + ) + ) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 0f48ea8f..63eca145 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -22,6 +22,12 @@ def origin_position(self) -> sc.Variable: Relative position from the sample.""" return self['origin_position'] + @property + def crystal_rotation(self) -> sc.Variable: + """Rotation of the crystal.""" + + return self['crystal_rotation'] + @property def sample_name(self) -> sc.Variable: return self['sample_name'] @@ -79,6 +85,12 @@ def _create_root_data_entry(self, file_obj: h5py.File) -> h5py.Group: def _create_sample_group(self, nx_entry: h5py.Group) -> h5py.Group: nx_sample = nx_entry.create_group("NXsample") nx_sample["name"] = self.sample_name.value + crystal_rotation = nx_sample.create_dataset( + 'crystal_rotation', data=self.crystal_rotation.values + ) + crystal_rotation.attrs["units"] = str(self.crystal_rotation.unit) + crystal_rotation.attrs["long_name"] = 'crystal rotation in Phi (XYZ)' + return nx_sample def _create_instrument_group(self, nx_entry: h5py.Group) -> h5py.Group: From acfe0e3fa034470fc705fb508a18a06a0cfeb446 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 11 Jan 2024 14:54:13 +0100 Subject: [PATCH 046/403] Reshape detector counts. --- packages/essnmx/src/ess/nmx/reduction.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 63eca145..53c56947 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -93,6 +93,19 @@ def _create_sample_group(self, nx_entry: h5py.Group) -> h5py.Group: return nx_sample + def _create_compressed_dataset( + self, nx_entry: h5py.Group, name: str, var: sc.Variable, *, long_name: str + ) -> h5py.Dataset: + dataset = nx_entry.create_dataset( + name, + data=var.values, + compression="gzip", + compression_opts=4, + ) + dataset.attrs["units"] = str(var.unit) + dataset.attrs["long_name"] = name + return dataset + def _create_instrument_group(self, nx_entry: h5py.Group) -> h5py.Group: nx_instrument = nx_entry.create_group("NXinstrument") nx_instrument.attrs["nr_detector"] = self.origin_position.sizes['panel'] @@ -100,7 +113,7 @@ def _create_instrument_group(self, nx_entry: h5py.Group) -> h5py.Group: nx_detector_1 = nx_instrument.create_group("detector_1") counts = nx_detector_1.create_dataset( - "counts", data=self.counts.values, compression="gzip", compression_opts=4 + "counts", data=[self.counts.values], compression="gzip", compression_opts=4 ) counts.attrs["units"] = "counts" t_spectrum = nx_detector_1.create_dataset( From 6ef48b0878ca95c3956f77555ccb80a8cefce26f Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 11 Jan 2024 15:24:03 +0100 Subject: [PATCH 047/403] Replace h5 data group/dataset creation functions with helper functions. --- packages/essnmx/src/ess/nmx/reduction.py | 132 +++++++++++++++-------- 1 file changed, 85 insertions(+), 47 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 53c56947..37ec7127 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) import pathlib -from typing import NewType, Union +from typing import NewType, Optional, Union import h5py import scipp as sc @@ -73,6 +73,49 @@ def counts(self) -> sc.DataArray: """Binned time of arrival data from flattened event data.""" return self['counts'] + def _create_dataset_from_var( + self, + *, + root_entry: h5py.Group, + var: sc.Variable, + name: str, + long_name: Optional[str] = None, + compression: Optional[str] = None, + compression_opts: Optional[int] = None, + ) -> h5py.Dataset: + compression_options = dict() + if compression is not None: + compression_options["compression"] = compression + if compression_opts is not None: + compression_options["compression_opts"] = compression_opts + + dataset = root_entry.create_dataset( + name, + data=var.values, + **compression_options, + ) + dataset.attrs["units"] = str(var.unit) + if long_name is not None: + dataset.attrs["long_name"] = long_name + return dataset + + def _create_compressed_dataset( + self, + *, + root_entry: h5py.Group, + name: str, + var: sc.Variable, + long_name: Optional[str] = None, + ) -> h5py.Dataset: + return self._create_dataset_from_var( + root_entry=root_entry, + var=var, + name=name, + long_name=long_name, + compression="gzip", + compression_opts=4, + ) + def _create_root_data_entry(self, file_obj: h5py.File) -> h5py.Group: nx_entry = file_obj.create_group("NMX_data") nx_entry.attrs["NX_class"] = "NXentry" @@ -85,70 +128,62 @@ def _create_root_data_entry(self, file_obj: h5py.File) -> h5py.Group: def _create_sample_group(self, nx_entry: h5py.Group) -> h5py.Group: nx_sample = nx_entry.create_group("NXsample") nx_sample["name"] = self.sample_name.value - crystal_rotation = nx_sample.create_dataset( - 'crystal_rotation', data=self.crystal_rotation.values + # Crystal rotation + self._create_dataset_from_var( + root_entry=nx_sample, + var=self.crystal_rotation, + name='crystal_rotation', + long_name='crystal rotation in Phi (XYZ)', ) - crystal_rotation.attrs["units"] = str(self.crystal_rotation.unit) - crystal_rotation.attrs["long_name"] = 'crystal rotation in Phi (XYZ)' - return nx_sample - def _create_compressed_dataset( - self, nx_entry: h5py.Group, name: str, var: sc.Variable, *, long_name: str - ) -> h5py.Dataset: - dataset = nx_entry.create_dataset( - name, - data=var.values, - compression="gzip", - compression_opts=4, - ) - dataset.attrs["units"] = str(var.unit) - dataset.attrs["long_name"] = name - return dataset - def _create_instrument_group(self, nx_entry: h5py.Group) -> h5py.Group: nx_instrument = nx_entry.create_group("NXinstrument") nx_instrument.attrs["nr_detector"] = self.origin_position.sizes['panel'] nx_instrument.create_dataset("proton_charge", data=self.proton_charge) nx_detector_1 = nx_instrument.create_group("detector_1") - counts = nx_detector_1.create_dataset( - "counts", data=[self.counts.values], compression="gzip", compression_opts=4 + # Detector counts + self._create_compressed_dataset( + root_entry=nx_detector_1, + name="counts", + var=self.counts.fold( + 'id', sizes={'panel': 1, 'id': self.counts.sizes['id']} + ), ) - counts.attrs["units"] = "counts" - t_spectrum = nx_detector_1.create_dataset( - "t_bin", - data=self.counts.coords['t'].values, - compression="gzip", - compression_opts=4, + # Time of arrival bin edges + self._create_dataset_from_var( + root_entry=nx_detector_1, + var=self.counts.coords['t'], + name="t_bin", + long_name="t_bin TOF (ms)", ) - t_spectrum.attrs["units"] = "s" - t_spectrum.attrs["long_name"] = "t_bin TOF (ms)" - pixel_id = nx_detector_1.create_dataset( - "pixel_id", - data=self.counts.coords['id'].values, - compression="gzip", - compression_opts=4, + # Pixel IDs + self._create_compressed_dataset( + root_entry=nx_detector_1, + name="pixel_id", + var=self.counts.coords['id'], + long_name="pixel ID", ) - pixel_id.attrs["units"] = "" - pixel_id.attrs["long_name"] = "pixel ID" return nx_instrument def _create_detector_group(self, nx_entry: h5py.Group) -> h5py.Group: nx_detector = nx_entry.create_group("NXdetector") # Position of the first pixel (lowest ID) in the detector - detector_origins = nx_detector.create_dataset( - "origin", - data=self.origin_position.values, - compression="gzip", - compression_opts=4, + self._create_compressed_dataset( + root_entry=nx_detector, + name="origin", + var=self.origin_position, ) - detector_origins.attrs["units"] = "m" # Fast axis, along where the pixel ID increases by 1 - nx_detector.create_dataset("fast_axis", data=self.fast_axis.values) + self._create_dataset_from_var( + root_entry=nx_detector, var=self.fast_axis, name="fast_axis" + ) # Slow axis, along where the pixel ID increases # by the number of pixels in the fast axis - nx_detector.create_dataset("slow_axis", data=self.slow_axis.values) + self._create_dataset_from_var( + root_entry=nx_detector, var=self.slow_axis, name="slow_axis" + ) return nx_detector def _create_source_group(self, nx_entry: h5py.Group) -> h5py.Group: @@ -187,9 +222,12 @@ def bin_time_of_arrival( ) -> NMXReducedData: """Bin time of arrival data into ``time_bin_step`` bins.""" + counts: sc.DataArray = nmx_data.weights.flatten(dims=['panel', 'id'], to='id').hist( + t=time_bin_step + ) + counts.unit = 'counts' + return NMXReducedData( - counts=nmx_data.weights.flatten(dims=['panel', 'id'], to='id').hist( - t=time_bin_step - ), + counts=counts, **{key: nmx_data[key] for key in nmx_data.keys() if key != 'weights'}, ) From c6ccc4e0edc410052e1d6a425b532dad40a29f94 Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 18 Jan 2024 13:45:35 +0100 Subject: [PATCH 048/403] do not plot all pixels to reduce docs output size and fix error with line being too long --- packages/essnmx/docs/examples/workflow.ipynb | 25 ++++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index 709bd1fc..3ce0b131 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -107,19 +107,19 @@ "All pixel positions are relative to the sample position,\n", "therefore the sample is at (0, 0, 0).\n", "\n", - "You can plot the instrument view like below.\n", - "\n", - "```python\n", + "The instrument view is shown using" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "import scippneutron as scn\n", "\n", - "unnecessary_coords = list(coord for coord in da.coords if coord != 'position')\n", - "instrument_view_da = da.drop_coords(unnecessary_coords).flatten(['panel', 'id'], 'id').hist()\n", - "view = scn.instrument_view(instrument_view_da)\n", - "view.children[0].toolbar.cameraz()\n", - "view\n", - "```\n", - "\n", - "**It might be very slow or not work in the ``VS Code`` jupyter notebook editor.**" + "# Plot one out of 100 pixels to reduce size of docs output\n", + "scn.instrument_view(da['id', ::100].hist(), pixel_size=0.0075)" ] } ], @@ -138,8 +138,7 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.18" + "pygments_lexer": "ipython3" } }, "nbformat": 4, From df2a8438f0bf5ef58983cb22c6f50205cf24ddfe Mon Sep 17 00:00:00 2001 From: Justin-Bergmann <48309261+Justin-Bergmann@users.noreply.github.com> Date: Tue, 23 Jan 2024 09:34:55 +0100 Subject: [PATCH 049/403] Add files via upload --- packages/essnmx/docs/about/NMX_work_flow.png | Bin 0 -> 97506 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 packages/essnmx/docs/about/NMX_work_flow.png diff --git a/packages/essnmx/docs/about/NMX_work_flow.png b/packages/essnmx/docs/about/NMX_work_flow.png new file mode 100644 index 0000000000000000000000000000000000000000..51edeaf21b8e362216817fade98402c3d4096ce9 GIT binary patch literal 97506 zcmeFZcTkgE^gjrQC@7#J@}hJEq)YEjr1##52%&`zp?3rXl-|42Tj+!qg0DyqU?>4X zQ+h{wXQQa^cXoGX|JmKynf);nczB-YwsX%t_ngn?+(f9W$`KMgCcwhNB2hbzRkMWm8vXm4@rv>r7^BUm4Laf`NzUsg%_-u6vtM;Pjy1UrVGNO}=6#JfJ7}Z?ee89-{PU8(J!m`B9J(TU(|E$PQ z%^|&|xV!mP*$2zKM|U{zjOuq()+^zTvINZ)U(ckTd`(@%Lb5r#&zc1jzhz<*jy!%@ z^=&up)##!uDq#@#wlbot{-Y+1|GhNZ7>9kQrn)k}X&MOe%F=22kd1UVee}(T)#1-H zHmDsxt|M}B!ld=bgQNl|y5-~kRS#Bey-V_V=^kw%+vJ;-V-JsP-I5zU0uGWrj?=0KZ{_AdJ1DM?SXD)G@kZ$4z9wUVsyXW5XM~pyv<2R^XnBikQkl5iaL$7qYIFR zpW`{lb2@PX8X6i=7Yj>a&6l#ji(~#1qqBB%a}ws{^ziWD@ZjNabg|;(5)u;Pe9q0u z&CQN^gWc83!OhH*-NBXqx{^Qjyac+MyVy9n**H4T{H)jPwIkR~jE?SSLw|p+`w8^4 z`L87h*Wbg!43P8Z6HYFU=bV4*#uOF(c~@B7#uI2~@Y2Q}BOXi};#^z;&qaR~_&+@Q zugU*WRR6!C{DS=dv*iDH^1m;s;|g?J0$*K+=K z7b9qK0#VMtlO|4J5sh-j!ji;NcqygriM=s>%U4^5s{KbmAl8SY8J`3qWCo3ldR(b~ zl}E?0h>z3)kS3s6Hwf!eq4Y05iXlJGyJ&5-SFo1UFwnwW-e zBQj0~_KvM)`bFWm32)wD;oPPP{Pim^{LaIjom~x4+`mwyXA z!m0DqX%um=YegS22Z_-#S=G{?&2g5eQ3tzsXy2Y?H=swVT8RMuftKjtgtJ?E9BSvx5`(U9<2JWdf zT72iaaWeepoda%TcWE=KJzdVz$g39#x zn3AOaC{(LnVEd>Kn%{^oN|=ljPAL&rVGi4*F#&sc?cfVhdJLdTW&N%@=9G^qd?TIx zsj%)PkwnrwgUQ2(&KlVy0+g|)5*2Z?_@=<>AFJYy?^~NUhh$Vws%a|@dt+RjRImHI zs}$JI2Vbaln)xC}Z}BOvij5-f2c&DDGc|ByPDUiUR7+zEWGa`Q!r}haqpjhCR%pD7 zN9`nczN}YF@u>uA^YE>U`t3h;<*NHs+Ef4#3l91;vcf;!02uB#kaLhA~9- zh!K{_|9Ek8)1!rcuvbm$!vhz7RVq7Ul{W5^uJrzGkiJUcwiu(@In0QKl+|{RjmDq* zqwdX!w-E`I#CTClK7N~ujDuJwTk3Cdvqf1+qmKJV`FeXOT-)*rp$!3}HAskYk461G zgMS`!1mNZ*mVO~ng7*R!;VseKKWeMFE41jD=ay4m15n}P$G9i`iE`_v)z)TBFjHWO z*2iMIcY^FKba@*NT$VXs>sfofd5m8Vmejf%)d&9ju?vDP!oX!|yk&pD+BfyttZ>7b zyU|<7Cb_h0eA+8HjrM-JoRn6p86Ru+)geWaE>NkPZ?vy(WLl14JP~f+^uVQ{WIJfs zYEjNL`qjK^2J>=U@joXMBTa^q_nVp=7>poZg;u_M5yQw@P4YdhNgb6l;aT4z*s@{H zy2_w?3PAy~-}T#9rxLT>)(@%-%!XBg=dYu^d@oEh!cA|aduOzkk@ZbTSlV@-zxbuG z?IAb+VVT2z&%?0uwwx#;k<4f;uBT#s{FQb{T`eO`nI4@a3n}qELI=L$_o{E93}cez z7w7PNySNv-E*Z@=UvQP!moFHEn-qtI6k-wItQJDyHx|c|L>LAr&UYg185{4AUg6x# z*GPNdvgMn7YCD@@ewY)PygY`dn9FJ{nnE+I{QM}Jd0q_UK0#BV{OtwuGF+#o3%3ih zUkUbEEFnCzNW_0moE4pnSDVt;C_DPJANMoU>v`jr0?+sSz|CsUkczX6%{8S2%`Xcs zT16pN1L<=dGpg?%ANF~BZJz%#O!%!xMq*I-9-UQx_2ai!cEO*R#x`QP0ybhNzeFjX z?K0%g#ve4WK@ti)s1WAa__BbRW&qVG2>K?y{b*8Vy&qIA)FqVGY8FGtZ{oEnPRc-X zNs!nzww;g@{iZM1c5&geiIhtQ7uDXf`+BidZ8XVP?s(Fr*`>Ot!?3?stAR>Wo)js0 zvs;37Y#B2+`$k!E!WAA~k6))zLIc={L>a{WrCwq(`|0&c^i{#wj~j;P;UQ+4<8CKO z&!L4;X;i~!D(!kBxG6d+m1vbhCeC&{l&OSYrC3n^SEdzDb(leWC63;FpmZt`gjPVERb~9@k8>z~ zW&#u8%kWfJh}dHm$A$_kAD?X-_0g@GWBu3Ma z>4K8}jmej%+8pKq4i$hemLhpZBuyU`zc-{z^2P>Kq8TPXmoUTjti_>eqS2Xx^zr2{?Pn3|`0*`~gQNLa!Adj;S8f)KO)# zpPCGk4?|k1R8|+B6jG$~r%L*F^>nxkTZWOd@qgy#ZR*=|(?0b-eC2<=f6hr`c0H=0 zap9Tj719b|L(|R6T5_$^ljsoc0uYY~6(9JATfAy0UFhZ#asTGR#I5MQ+LX~a4aRYA z@Mlwze3}C;m45w+{dkC0DXeg^Z-v7~&1ubI>lIqN19uS0SHWPIt!g>7#Kx1Y>q7ya zvh!ZG*bpMQdrN+K4;Il}*~Xc)Vf=VR4CF24lWMg68VA+?baS-a(gAf#HoLmTYDu)& zT(|RG^0UNUOBoxIon{<$J#w&Tcp*t)cDgsy9rya|D?8oK4jvbZSw$cp!^+7UDwQBZ zZae8;u8?I=;?=3QtBP2oLte6DLYea59NE$%&!HWTUF$OM-7hnVuvo}_WChgzgDmfZfctv6~8t16I zW70L%X(~`vLHTd2H+&%n?1FSAc`LwYZj^AfvcRbq$tyLxXK6p-c-D}7)UZ(t>w|1h z!E9+D&`YBxhu2ddgiyiKpJ)_$FxW_%~Ufc%Tp1K2h*gx)M5L+zXJN)HENw%|M zykW6ia`0g*sG;$2vYy!pSumHEOjQ`0~_aAj-j{S?fury~5Q-mIsCE5T}Db_qqLuZQ$CMuRiL zI=(WlD2hux>&Hc8sE9w#9~JOg^&ENkV!5TC<+iY@9@D#~_<`meGjBUSo9*3ExMQ4L zx^Bt`K9=sX6oI5Ch*$#HVz%%FxeK?=u#yq6!4ne|DJ|J$A<7JWy{z11;Vdrq9`=s| z85qqd-9Cs3Tj(k~Ik7`Hh1N9OI?c+Gi?2bA5>Nr4gZ}O*MgFTS8uNRwCE*6I1pHp` zJ8t}I{|rF#syOggUE_>xj*YhhpY^b$c1q%D2OI68;(UKcsnONA<&EO_CH`hOxrY}V zzjrJxce@#Aw8%{|9+nC)A?yZ!!bfrUrNx~KZKm)$f~#+T``8wD}sM<1uh_r=w%Y($~7NpX>Ae=nSqpF$4itWkU(m zeE)_I;mFLCe>=wqzMlQMwFjCpl{03JUqOyVj4+cc^U7oep6@2h?yxl%!zjGE6nx7D z7zeN*VK{i8qdPoBAhE}YsP~Bx@%SDQM0cith73Ji-eqdj+2YP~kTwlVV`cT~VtZ0( zt`L*@>WGu1gOh4V*ESQt!2;WBtvkp)_gFT|P=m@mu_!lKT65qzQP$)8r_n7c>mT>_ zyFx=;Wfx`13cFr2f@NNB@g*fD68?#GH*K7OoSnX{MpIC5bRpJm$1%3&j5SrWZMmjR zHL{PxPfPBthn9?L%_E72w{`gw>Umd}k(Nn8{Z9!84$}+D*3yj59^bVR@OzaUa_WiF zFc>eAK@#8?nwrol-KZPAKwG39Xz&=6yfrZRG);iJ#hnVK(U)bPpusmkV?3Z~VM7Wt zwk#{KR&Ll0e;~eJU2Q27p17Hs*19h-0<*-gA&|M#L>cx?z`ojmBkC%jXTC@L*GL9P zx{cq=w|Wbec0d#~nt9_}@|8?i)zbUZZmH~)t-oU= z7OpS9fe69!S8GVN@m?-@P4z4`YoXb^61qBkN8iND>PQqa&v~Ge(^()fp>SFx1Je#@ zv|yr+zK$;1OK{#fgKhh~)SJ2lN;uGn&HW6$jjDtqf9~}naZ7Kb9R8?%_;i~TBx(Oi zZRSXZ2+_p3BZ`)IIj1xU#PrnscdXN`huPc~?gw~F-`p*}-Q@q+ZxZERt6HKuxu)5s z4LKrOGq+dk_>$IDf0`Dd{sY-A3RLd;cHiCPNmiOOH-~S5(?8;#Roq#DJ)<2P1pyVS=uvUA!bL+plPv|A*(j~SsN&f_E zk~C46oy-8YC2<;c^CqD88%5NgsO`88(<&8TKmw!6(>7~Ks;u|J#~<5!V-t{#FUSGq z!eH57d&T`YrEoW(>+{}qoT?ixs~D9&>Q}usLjEgEmcy9gz2XloZmV<>kV_^0 zmze{3JT8v`B*(tn&Z}c}A1ZY+PU(J*_&D}gv+nC-6^m&_IdRTt^5t~71#xhCKn0qW zv@bdCM*?84#QP6W@L)&jc@MNS6JEbLo!8PqJ)LS*i%Rm?KN9$XN(s%aV>|j0e{$M-z(aGC_3fS(X2V_sL$_YK`S0 z!9?5DtC_4+9&ISg8i* zZ%u7{i7^vOxC=X(WR2TxXHR)~d1X3h{f@F$h<`gKxW0P1vD>;?={_F=;c$4J_171Z zbvE=DXGc&O3IT`A{wxta_l@yy5i*6!zo#o!4#yv#aq(-DAR$iLgQ0IY@1F=HV+tsJ zi6mVduQulf_CEZYt^eS+MLdRJN#FE^Zi+Q5O|E;ROgt2PeOn0)&ki?Pp)=lPUCAlI zztJ(tnV+!QdmgKvkL{^k=K0d$M2E%q>1E|5B_$uZzVi69{P?ju8@-L9ycPUx$I6+_ zyP}g~WAAwz!SYo~@U9z8VVO(if@Fz!Bqy=zncDgZQDNhX>bDOL4ieq}F-ND#`?IPb z--k}xG3#&3y!*=aIv455ie#~ZrXJkJk4cmbNexUfYFqfZz&%3(^KeivogPW;gx!e| zrinFdcWX%Rs5%?r@cm_r8)@3%56IX&!~d9Ynz>IucObvcC|OJKe#O(rM;cy}tsv z%k%58HetkLtA2H|W@Urq;K$w!mKZBKhUiP0d2I!Kbvb(L&)l$w9kh=d9hx1hCvidx zGvU6NPUIx(-U{jBh6PxPc%yeC@Ma}!yp zXguDpuyR#9^E-;kLHSkkEt7V4H#y6`z}2P5$0`#>ij6sXo*qsKrEzT&Pc&^;)!7>Y zY<1b0996CqCV4AZW2U&D+DM?D=*^fhsVb~9f;Uc}TjI~6RGOseReQzgD-%me`+D{E zt`aY#J}nuODx1~=+nCKG0zbNdw+}T7UXbg#lfjyI)l3eLk3GqK>^V#H8xqfd91$6` z+UD_A7)i_SZ?oMbIA&PPgOS&c!^pTpzBb znkme#Hsh7WcJ3NdK3!upDNxB7;?sa(dT>t`S^$d?0!2y%F2Id??e@yRCWvrEz^7n^ z6Q+Q?Ppc^s>?wR1?i2GFqm|6;2tJbm;}DXW3Zwe2q@Q`KfyPD+K zOTJ*>;#yvqWEGY_a#z`8{*0(!+%oh?vbn4Q*LZ$mbgk&>JqT%5Na25P0*Ht{%05HS z9`3zm73o9W(d%Y0=l&6-Q8RjeRCq<%k+$Augl0T6%(6OsA03e}-Y7`!4`gE2A6mY0 zSFKqf{JAtFJ8?{ZREAy7pNyb+EY1;nJw|F%wp~{?L-{smGxW&QW;vB1b;B92UH1vs zonm8W^qoVIi>_MxVedm2-an@#WMrIH=C2D(5xtrvFAToYynxDOVCf{Z`P&B1#k3h% zNUT?CX;>}VkfgSroQQf|!YVf%lyqJ znQ9asFns@|Gzpxxx>=A3I_5kjkuPYrFJ|VMkvnC4GD__ARq+rd5>(kgBETYC-IgG6 zsycIK`+@Dsr)BDN|Dn&;es9u0#hgh;mIEn1rpM}~IxL{dD1Ow1G-KP<`&RSIw=3vX z?vt$)9d5hxGJc7Rq#3@40UR^u*mW9=ts+{T7iR$v9#WWH2HLO)h#waE3NH(|ZoWM} zBTJM8m-aOsyB-=MQLanEt|^86=LWhNbK76SW&0Mw9qR(H%4e$wk7fE6*`G{&9!T3` zgmVn@ob0TB+1=^O>E1hfAOGz_q$l?5ioIDy%Ny0~e-)D=p0r%pAK$gzs=M{ij9!@d zeofOOk{hE!OYmw|ha4IcLxJQ%#z2SXbEYL)6(4y^^iQg@l4A1;zVtxdFO1u7DIdw zw{I2rMV-2!!7!jF9KyY_74Z?4V^gr-OVxN3s|`_~9IF7KJ6(V&x`hH<&JWN%!&b^e z7ulkMma&^9<;HDw%Dy|^!$!>=cv_QD(5oG;Cz**Xrpnn(KB@IXRH`alldQD|Ft9bJ z(QMK>G_8FL&ucqPWMP_VJN8{KU+?}$qSaBUv2@cp0cvWHPgOn2aCjR8?M182SZ+GB zHK(u{G|{fcMDIgrH~m^Yv;#8aDv1z_1K;%IX9rT!uxhJUvjA2;#RrC^j+ggJ4GLZ$ z!Mb&C;bM#x00}k&`%2bLJ{0Kia{Wx{>%XD)gKJKKg8?)Y;xO%59kuyoZj-U;a~R}?k6Tsr>1!?>y|*D7VrV{(O*$Zg z`leML$$E|Vyn~SQZXGTqv6{>K=g{7m@HZKA7lGD+wdR)(nH|DFzFZzn8+=mbg>Sn% zgj+zIs2*;kS?~JN2`a5k6;wX7mWM@8if2SBrp%Pt`F40ilqvC~J474e_Wd3qZC8?1mQgq-NWQStFX+chvg3AO=|8NBCn57X0%ayjH>!<+EuZM zEE>Gb)RY{~Y;1I^wQWt?r>wwkSi;Nj#+BG`wPm&>C>5Oji@5JKIzdy5V|u@Ftkh6y ztl#^~AuhW~t-%&QEU0;ZC?Kb4SlBuSv^*9erq%onWkZr*im!`QJd1fVZi*V2U4056 zSX6AHOesDi<1`dJFjIiJPpedF598G@6qL+nnD&=htU;A5Msl~@<^PqrwKx>Uud+6o z=}vjK_jy-cZ?>;;sio6aX4AI4X2(hHvdgd!>VwR8^*htzPFv_gtLjL~M?XI4MOhqv zB;DPB(E~@?A(pZiM-OshJ$C3rKGi#nGMeRdNynvBmLv3i@8SAdM1hg;Hw zzbLi7%#Nj#@Q4NSTPzORX+nU)U2; zvgob#=<~}X@o`Vs&N=l~vHy7HfOeIa6*~=c32%+MJN+<$h1KO`(j}4&;s(-&9WgdlPmM!Hb$7^)+m) z8A-E5DS;S@hTZ$31awJD$PEiANrW|)IJ0KLTOsB~DWj(OhqGwcT45=*N*Rzm5o;S~CQg~`uZml8Y#iA(A1w&V=otgTl zowice3{~lpqDe}D$4F&WpQWrcn}(DYDn*Y26_cWOy-D6z{k~%~tgI+?%Uxrj1A6dG ztXhDl;zCYrH_Ujnw!-%ZQu@LEde`oXy|l{U!M4q&ZD)>u(vmw$mhzSlB7}3IM_>4< zOt>Fz=G6_Q8!L|e7Ih;%;EMG(EGmRK-+rt(icc+Ew@twquF z=UoVCi-C9@=4>~<9~YGgW1sAdq3L3(T|fu!gw{1wX=b7t3CTN3z65*TO)+0lo3OT- z7t!yPGVQ)kp15h(WpbfSR^O#n6)Wswh;CTz8&^=Nq;T}i4JgO1z)WxCIp z*^mldPg37pVIY{QA}U4adS*V9M!s_dvHPz(+U5`%M<R*J^*| z=9zfb&E`|rrL4I6>|)znL$_j!IYWu5`W=eMo%g=^7n|24Q z5|@@%5o#y9OkR7n)G9Pkjj*(WOk>z9@0pR3LUMv1hx=X+Y;fAB5ijeImYEy7@P^(O zJQN?}@#(rp4iTdqRt;tv}O^bScaEn6@P8QtHI z1M@7{;FBiu=a~nF@7p>i%dFYn7-8md<2uhP6tIWDlBUPlHg(YbY**&fsMdk<_o_`mNq@9TDQ-3XwM~6WGEd&=<#L`Q!gK7s%Mcr|Kq=iuy|6FVrnyk< z7Id+>ul)PYd-67?ZZ^t>5#?mi*Q&z10mtA8*wtGTAsSvfl+6pIL?@qMPeg*0f}^JF9C+>0`0F6^zSrI>%ymj;4OZ$a`uKX2F)w@X1PPSdYX z$^j0DUl7Jq`3$qvu~4f;1p3Iz#87;!~CG z=A+OUWhWrnd$8aF>`@WW~kUN9D;_nE~_aWBGMKjGOqQ(%a8ktzn?s zqatQ~5)fWtz1;wcPXo2I=%M%^0op5e{+r~Z_C0z6^4o?nq9i~j)F>}@Bclnj9Ic|( zQ<}!9WL~RU$yhOax3+U@oM|Rr3uKkwco`P$BU7kGmGU}6?AZiog~#V0VtM_S1L6bx z6U?5!5b-%D?8?;40;w`a%9zxd4q`$GQt|P_mF5Z?20{z1TeUWuDPJ2tTjI5GxaIhM z^~%c6hpMWjaW#@Nu$%AhwglfqIiC&%Ho8VYZ4$PZPsO38_2NC@dQT-{@Jg05IvvIa z)47LD$wyI^i363G$Zf!k+#k%`&^)EL=B~8<2BKi(?boQZDyC0YVXg_7FS+s@Hb=IM z9RyLH&MS7kP)nD<>5Ga6xHDAbfF>I4Erse^vd<*j{MK_UQ&+AB5wpBoEUr_Ml2+iy$RhjbvRaBiJ#-xIH6N!`8cg&)6-1?;Yferb zFn=da(7`Pd>m;6g`4ukmUgJltJF5b8Hh~Dc% z!70;)^WY+QqDy@cZ;b`XHRMH3;Q2*8=q_lLR%E3@+mE6!I&Z|e!304Pj$Ef5u&X{` zHkr&TlsMHNa7U~G+|c5+Ue(k%2KR>+8~000X)Te-(V$UMHe=#P<7S;Ot?`9AjY7dy+Ks0V>aXxv(#~j8g1e*wKkv2v2DP1L?9 z!r(+v-TgyvN4kFe;+C+mJ#cK)K<-ch^k32g_g*&>25|NbUatEO{xkTQdznt=^FN5S z1PpN_{>jzwH=v7wV#_h3QNUHUTfU)?89F~pkM zPUfS(I4ce4C$YBo+VQ`-eIH;tz~rg=m#v|R#Sl2Ca<6KE|LPY1nIA|ZT6CSi@RMTe zPRI0etabb^Un5DAgG;bD>{Sb%WC(yH3iV#o^`Q|^qW%(!Ls{o!>N7GRjFsnR(0t0cEt z*Ev=LHb=eOWVZzoO^t%g`o*yx+Kefn>eX$qW{baA+QI5_lP_b<&pZW;$;XsTBv+cty>BR#2c8hD*+xbV>Yc zn|8P`D~KB8RQlJ*3F=^0X$iYr)8AG4GYS6xx|-zAySsRR(^2RH7r~W}hNom4Z0{%Q zlf&$dA_C(6CG*>^c=E`fUyH~NOM<8_QbQER4@9O=i+YwUly-^WHdk1v9<)}B4&UDh?Qgk3r~09Aey^T(|i zYZn8+@skBGk(TMid2~uJF2A1VuBi4KomAdlXBCm9AWrf_dz>nQOPH?Bkr&5LJz+X( zC2@Z|0D^HqCpdamwUxw5;^l;;gJ$QOx1Yk@d-eB^-dH$4ET8hz1e{6pKKHZP@T;*J)+N8F49~Zn7Uo{<6k*s{an6_+ zDFP%?f>1rha$RXO)gMKL=Y*g5HV?SD4Ptw_kE^&$DXXu$Z|b<5TI91GpR=M+hkV~X z-zv9W>OQfK@!y~=F@-~vAG^vSckl_v^#;xilKV5W+%oYdW?T0#@^7AYh>D6znpAnh znRyXPU|hQyHDofOg|&r${N(tT0=ji6!$iDC}tAha9?MJM8> zee6H2)!R7$)WT%C`%`8Ui?ysQc1)o1XqIVU4{A1TXXktM2X9HY5rc@R=xVVzDWAih zvnyZS$8}B0-h}Y61?OI(8&%|1-}RD&>oqq>!E1Gi6~HSKwC$(^h^<$d&_3*n+wu@J2=I6gNDm)B^?z=H;`8?OmJ9j^2~GYNmD zU7;uPWe}ey@mnK4i-f03@H@|Ae{z>Wdw0zwR&&4Zz^fD5d*8Z3Ofoh>FKT)W3VM^` zSolDS`s(VDelJ3zCk98KJUx7|E{IxC@QDKphPEU+ahUp_&yjclYAJtj#WffgGM$jO zi1LBGw=|#4t2EJ@n-iRsGZ(SiB|6d&0g@bJn;iVp-i#k*!VcQQ4SW&A&xU$rg+l;Z zqu$_+7ni4|)OPAkx++5Ck2q+%q7@kSW|A7p^)csPf_3uo#fdkZ;M$3J4e753Wm|~q zD}@~n#=p_Ke@tGFvS+=D*V!ZM5wGnHWE6ukBFs^p?3S9^eRnkk+5MC&w{sYu zqiC@)r8~QmNy}8!c5KcJ0#+6A0y0P+s%Gfm@Rkg3*DlN7*{cc`W>25AFX&4VD5H9V zJ~~wjXS9xtRNkl_JrXukNgyg2s%{e|O8s-%P>4x=$1A~wL&U{}sS1SRFgRVWME*z* z+H&tn+62JGW!eIc=!Nb-vUK#>=9MbuK=CfaN@_-Nlh|84;?MwJ*~mw5ClT1o5@gL5 z6zC%86k~Vl7y3|uq>vz^>tEaFs7W?Qo!Pt0&p?(tSK?=KLXGt5hpDfe=j;V0(MOLx zFJYb~tDi&}1I~96>M&`A_JZDZ&s2%Y)_?vxy2fz2eHWn+cwc8~U!g&x4^7~<2{i?*8AX?XkE`!7>z2NGi2uyiht~Y#$9(x3EIn%ddv?ib$lQVYurgg$ zA2#WJ^^jeiQu&}4H|=tZQ_?_-K^r(LPq_mE0+@B1Eh5?KD1<56>l;A(t2~{5RRET72Td6e99=O+;;Wa(jhC@_qW@mhs|iU$O^a0F=mo! zipLA;3DKQ~NaL4|G(BgdIca;`-yF}ul|)kF7V2VSnYZY4v!Ypg&1H?Bkd4oJG#|U{ zGC#@qEdLsiw4i`c&d!Bf%tK#3-E~p%n zpJX}xj{d(0aaRfx+*1z;G5n41B*%Vc%{p}c@#`_<$q6RiC!KgD^S=|dnC95RlfXP( zlpkVbP4-XpT)NpHv#lT|a^GbHl|guD4&tGoHmHPP5x784>}f&H{P1@J3pp6a^u#KM z$!aY02+*=l%M7R}<+VjAE@c$GDDdA0Wj}mM!enDfRqaj3tK3j=%1Yt`7zeIL@{x%z z47E&k^Mpsr@?Sf-$C#tPn4<%Jv0l8Z)t}}1^YnZ&vd4@TXI(AxJwGNg>A+?4P8k~= zvKHwO-WSts)Cpg)o8qiwmlu3~%S0X!_yK42Oz;w2tW{)BX{?saV#2#wq$X7V@qU*y zaP}}u$nYG~rraI*-={*4i!p7I3A9F&P1n4xulUyMb?3|MECA!SG5J6pQgCh($hWk? zfG0;1iU?W4eDL;`J3O`6+JMU=h+>NB8D9o(BEO$^s(J6^_MzQ4U5kKyTyeL2(^|BV zMPE*t#b9BD=5~!Gvc^hJh|>AR7@IMR)nH~EKO?-ip3|Sg&elO}_Ot-5G}LxOv&>mK z?#-K#mhO_B+XmvxOhU~b@DXc$Kr=C=9rb$i^j+rTQ^e86rJ;n1if~;LchUoD9Utpu z2ZI);h)mwk#ObCpGyB3v3SZH!{;S*~Y#&~xX(F5TJd9@F?J5si9h`r8;J)AJch@0J zkyHhitblG4E6`1lZLs^DpMVvlggH#nUU1^I|Ml?=#?&jDNO2N|A3uJ$Y)uJG9G_9w z%q4Ix^re^zNPWZia-D)J4|!?QMTd8HtUD$(j_Z?Oj7w|6{7uq$mx#Uw>SzV`IH7y_ zb~k;f`%)5SW8cQA+f21p`0gy+*`02o?yVy;t?9DHI{TNzD2r#b`Q~RhcJqfuqcZ`Lgh?0At60g?N zOtZ;PI%K=LD**wV5-M37O7K%%9F%?Hl^756YG&*2^Sv0k&^W<(C1S*z-D^9MANLj; z-eut(BoN(z#4=K%6^}sS^4-;6G)?3<3pAW zPQuBCM1O(z$-hIPeE;=FNYH|M24McsKEJD~tiYzo9ofF$$dS!uw3WG^$;(U9jwzdu z|3a;I27yK%oUD}KYZe)hwDd}y*y`k?hsyG$;WNz9C$D!wn)Rmkt1btU%oOS`W#7R8 zI06oFpDjb*Ji>hNDmazOXXh@FJ?GZqR*0RqYK+RstPpB5AC3Ah&tiPjtK)A>B3$+d zYEcV=Igfgct)~3{d>4Q{Nlv_r4vRwEM;SXWk4xAm`)u#{cfQT0BJ$~ATTT%HyF|13 z?Sv;e-r5en-{T%0e7o`6G+6vxj1y4Tjkue4Nb7)(;|)mf zGBAvn$o#F)3S3G4AUDX4mS;**>qhN#Qe+Yf!$G=0bx?r=tHbWdL)XJ{Kb3VojLSIz z^D4vV2roKrez~ftkTd`KH86pEZRp=}{;Km9F9md8`eKdSV$ho9xTK-Y_R=Q*J+b=~ z@orOTT19qZymKZ;>n>kKjit#kUN-wSJHVP9*{cQxjuiz0uuC*+3s-3|M--md1JA4b>4lArPD>W1HG^mhxz zr37RMGh{}kP)i(l+|`pe)?}UWb*Id$3`N8S))uLuX^vMug*>26;jR*o?Oel!i0(}? zfv1nJ6Em@J=rG6R{Yg7y<#!&CG4;toVi^Xq=`TI!hU#aWNz zY=5=6D_D+DQX=EHCN1+3XVxv^#P61#pDSm5w|g~lt|n<%{f+1N)h_6G#J_&ELM$}g z8B<`I`<3g5L4=HJBtI>yG%px;(?|!Iw7B;m0Rc(TsJs16vze3MjwU3!p<3dJ@;rTj z6fa?rHwU=zY%s^4xd;)ji`IpZZYGtRcNW16+EiB4?{lIZJdaA-+GhQ5NAnJX7zOrkmWev>^u4GgqkELOI?J(Kc|A zNknd2k3hx9MqOWl} zRbJ)U6)qXAixX>e&d{xSHR(111-1|-dtBD5%8KpgrX}j?Pc5d!4c@x)R&7YvA$y?` z3-cQ_EYU1WPUK5~@3P!X?*T6NuigLY019O4Eia2+tLj2SU``XC&$k&nha5!s9-!(~ zkm%WztIO30l;T5fLXjInFRLd9t-vH6n(?ag#rx6*sGg4_I8}1Zxz>=8GNp`_>Z#f! z4~5H9dBt?|$mD0)=d9?ACy(8ZZPDxvj@jgeyK5uEWU*m$TE|xHLE9E>SRb2ku z8Y7XAI`{nKGAeC#_ATa4M8@Z!9Z+>&mfPIzC9=G4W5az{6aHhZ;1|g)+RZN})gCBusv-JJ45qTe|1Q5(95N3n6YuPW>vWV~7`kLM#v@x2 zWG4R);$H~E*bj%|_}>s0rVsz;FD2BvNRO&y$64Z+vImsA?XhRvZ{!V{Us|;UYKq<8 zK)NhgUt1*6pWgsMjLx&@iRV5FvGMYhRb2$(=%Ar{^(oy6uUi^<(r`#303!*AvCl&;RMLJc z2Mrm9PyDN0oL1a)e-vwn0E4DiTFbIre|ZRI&3M8ws&)kB;`{sMJgq-jFxK=q#lNA? z3@1kUF6@;F|0-W4hUYqnwA{~D?C9+DRs$B}|50p=I}eI>i8oNaJ>@a$_u%p4bh@P^N0}-@Orss+=o{8q)7nqg#bMvT8>fqYBA~i7ki{zEzOmCmr^Wck z1)#^4sBITvZC$?4U(}NsNJM5m)mXhT3kR_1LD;0Er2fBgFh|GSSmJbr-Y4orQ$z6Q zxg0G?pKp4VrkiP@dD_2hUlYfiju-^P3dK8Lc^RGiOvkP}Wnu`EUER5&B8MbPN|B{a z1)(QDWf>@k?}_*a6>^KXH!_K}U?A`-)lFrydy;d=o3976liPt3efH*#ZTtP**guh_ z_G%ncl1Bf-2`eOQ7E8!?3{<)>9u*l(A3KbLUP2@mW_c?@W$RspLA$@GT&LJT)Jv~g ztHI>48(|`Evp>=WN?P8;N$HQnmJV0KhWAr_^8eN4nP+pPLF?&!AWZg;IH!--_k`ru z>JP;vIBowMdv6&P1sAn}3W9>52#81{DXnya3KBy%1I(b5ba#u2h=8;-(%s#@B3%MQ zcS?8HJu~=z@4bKTy=&e3^LDotXAljdM-iIZrQ)<+zMLq1ic zGH1}!(LH>(eNEL~6t!CL9@u!y2pY8Dt^nn<0EvCiO;;wsTI+sp3jQ$Nws>(HlAaQL z)tCc$J)?#E0%&mXf9$ovO)~h{%gdsIkWd`=QjB(0aWZx3+@C(`?r0OcD_|@L@ z{#s@tWqb*Ou!u*h|LCj;xMW4_o9qlt3@_n=!aiIWOJ&g6NS!#MV!S79hYYFCBz6aN7S|#TI;QX(@(=z!gGkTLM5|5;b4k;XYdO}M|ago zK?!MiQ?YNZXY-mk-_{2~Uws#^F&wcxl)3y+;b za?incW6wk;4hB9S7!TM*vb86Vm7a(xxMeR7O4UtWo@UzZs&MV@&ZpA6kRZ0{l{4%2 zPJ6(n$8I`2Qfgv2sksF?Zb-1()QxXW{2W2LQ48rbwN^kFnWOcDcz<~@2Cqj4m1cIVd^KRe3JI7|ZT4h1LVU61q-duTV=iXcZe z%&0^5#pF=KgjGzA`}SsjiN`7n0fnoItXdv>dd^Tq(y+&2R73%6*hF+gWbx5AGGzUF{Au`Y^$8R#mg0x*$WtG9wyP+I zo2BYtw4q>G{LJNHW|a_xc_%rLpGo&Oq<3h1eRugG-D>#KNyl$00PTgER?$V?^OcEK zYoPKWlL_%>pr^>J&$~!;(=?u%()2nXepJ0wUT7gIBNXqQYHn?I=iEM2_jTekc-#J4 zV_piay|}m5Ch0BFMu+z+&mPICB?m@1K;V>j1#nPyqnuBG4lPhq<1)nm|Q1Li~2ico1% z7AXM#n*%SaOOcxO0R2zDobnAC(m$q9Be%CdgUKTbjzpY)+mhxvHk_G`)*8p;B^~^b zoHp-GKYV)Vw(S}q>iXjFvUq#Wvwzg?&(`9KM_hSfPom&sbsdMc?c~*bL5P~fo4(6V z$b7om$>)CKK@nmK&AJyvruN-zC2z*a>>N7b#GXkWKdVO`ofW@MGN<1i`LO`s5xwav z(&-iT+rX+zpQGVAcwQY%3bTNiatDgg9A|+X6pU1*t7Q$d{xo17vSueDV>qz>cymM2ds_WY3-*dSJCo4(?>uHWK!uX5VR$-}a2Wk>g*QPXhJIZ1ZApu1%9 zOXp;zoy_NR=PO=CFYh%b?Y&Vlf(mbw&s)e{wdGi$$(_FyW}Q21|8>QZjV1F%aK~)^ znq|GC)^ljK-2kuC-XSJD^T4scqB>;KaQcjxjY0m_j?WjmZ@5z!>@l>9Ie%zj9Q(Q^ zHdt^J>OKDElL`VaXkkLzpN8e&DPwWpVjd`E)cpy$M~Ed*rO7;Oxxo8p09#j5HnL(~ zi*RmUbv^SZMl9|lCVIIo!wud9qt-&P_Q*^{nb|q|ezsTfBr3_3^W~KTT^OYX3iAf3 z8_(Q=T@`~O|M?{=-RsW?=~|JFIa*-rAB@jLe5##T(rIj^uAHX7deNF{HfHjX_bszd zt}l`ViP@X}urY1>dw8st!gaj}JLe|7Vee4dmU;PF=zb#K3eLLjdBQ$8kXLATkSed4 zdGz2FTSt+(a;K{`&$@QI!c&tMA>&;*-<}n%q|+=nhVbRytqN-ptFYAT-*6SWIqgw> zcH~;1H_`Br1FjvrPxbi2=LeCWgLFoZ4r=hq&a6)_2}h5PsGP#I@VOew3>R0IE`&;s z4;UpG-z}>3Q)ku&MB$tPw64gNuge256@3lSQz5UT{|UZLJ!rkJ) zYW^lwx2bzsIw5)(g+u0g9qKF2jCOP-n>qHZQ4Cy2wc}EC4D3=7(~CPTd(l~H-5Mg35r>}vdB5JP+QNQqdNxrDb1e6T<_mWC-p$?k zrdYq2B={i#&YSxtT!X1$tojdGt(z~)F@odekyW@Jix%G(j0Y-{+DHaUb1&BQ8e!VVUswg=;jmjP7 z{&fgBNuIY6n&j>Y-f;|amQPH&_hDrJf_r>m!oucFeVBLBxhJis5*pJ0I{wlaHm~;r za^Hj((;^pODS9xL?gqdXcv17p#4T#47J+(iejR9dcg3h$*=G-=C+VIXo0piK);e{W z?NS^SUaIrPksOF@d24haOyl#N6pwO<4fL=W!7%i4jUs}8JX(gQzab6 z4CdueAxbMZ@4l4pZ+us56+6KLePL^{#f#@|aB;%%TrYV5jNX^>9@z=>o zJfxdAe$XivT5pV%FW%exG)#!#@O) zIbs=Q5Ve2G;upFmpfrq#_Q(y&krX3BwCwSYw|aw zCYVI7cBtYsM;a(fZ00VU0+=S1@?{{Vd`r(|3i<4&KmP(5+5A!nOYb~jeNs>5`D>}X z7ne^U6I;q+^wX(jVL@bW| zi3m8{UGA_DN|XO_+SH!~gZ<=(@FkD*so2qUnvWjUiQrLyj+{0+eLwDpCsqu>k$fUe zYy}PNQDTk_9!?xjFIud2dx_y;Qm^0}UW^TYbJk$F01#}_L+-jdx@_3Y*tlDkC){8- zpOU)pr{aKs$Z0|ty*xW2*NA~WMk;2>`17+)6I1I<_K--B6bhn1^fUz8z3MVLcw)Fz z8Z|p|7-ZMOictTBQFD3&U8M&H9Lmw(#58y}CVmZ+*d<1jBhgbK;r4Np|A(Zo^goLzv%e~PbFG?HZRTywb zFo7u~qaUJO>$o4pw=^(%t&#N^zz{}ZY1mLk0zlrY+?pLg4Mu`RCpuSNqdV|Hs?J#a z*DyF0ZlEt?UKa(hTMoeP0ezJ90U%wXUt#3jWwnsiSmy}J{FoU)rzEQa$D8BD&=@Y^ z_6kl!hkK?EdwUdsL&a0t&&@s{DW$J0$YNsv3JECPOG@Ws(1a$+;kQZoT8ec?EG4)@ zqCWHaV>E1;Lxt#TN71%R@|U#3-9G5L1}lCgGXE zM_tiv(ts}l%fHsu?xIaU7%AQ+t3K~|tdgD8dI1ly=EW*U!VhwW8RaXtDsWi8y#d-a z;wWK@-xkr7xQMfG3t&qo7AKTGxJs)j6-MCRl`m>Q6GQR@kW}_eJ~EGptNCJ9bSWl-Li`gdujB9J!Y+NIMBn#O(*V{L zEjlGv3y+l9ONPInfYl3Q0Ypo=JmlB^IFz@5#QUJ&Fzv607F>M8AXAf@<$%gF)N%uV znrZ>tQ=co6{V$77JyQ;3tg!JiHC*I0f|0zx%?@~SvF8P4>8Rlgp`ky2n zb%57I)Q*LZ{SWW+zs>o-H9Gq>W5*{au6hYs_mNNqEKoy5KpDGRk)uu&z%s?Cay08R}<(Z3*Y?M82#z7ULE2&~h{S zgH=R$U!Rh@$JSK6(NLbYc2D5Vzx`@*2f4Bj5-av&W4SHI)Lw;9vH{}oD@d8cq@42p z3zgWE`KpxU-@^ZPB}@0=BObGe2oCsEqEXw|$NlM$XkAfSz$hW7#m6TTnR3Pr=j83l z|5Z+XnuU%)*;s%wS^(M*`GU|7M~b2+Y0bZW8O>rjb?1m{X?@dstv}P`t1S|7v8R>n zd3MNYx6mdm%94rNWZOKxFB&vvQaDEj^uN{S8?UXE`JDOUUa>rIEeJn!flW+^0_5C6}Yti|v;*1_|=OzJl zt%d%N6%mCu{`<|g6N~(tlJ%V>Zqa#%J>(}HQEcOy!!?Ut7Zjo;n<7TWiCenyt@g|D zuN(2O*2ngIB_x|q#UXD|kr+NJ=!Sai``ax+WJaU?2u`}w&ec$P_6JS$SdVCw^mJ0J zlHH$M)vPGIq@lq@tqJ%;{Tv^h`cj+O7coNi`P_(L`&l#g=S}3k7yj_bjEXm-ap#Ew z&+$?GuutzD8+#4D+E-!*-1PL4w_3Y4mLlVWeO%E=1Zs^o?()SDKf*_??Blc>0p@(Q zi!mmNp_qG`vM>cnzQP3uj4pyF%rcQ&CT2!I>*WZa=9&Z`(}Zv9p%DyQ9d>8P#~JP( zFHY8d9>9O9NX=#+*A-hWH%Ca*jT8I{wGKQ~e45M2CUHthcDgfA)z% z*RF0~#n<{$5j$Edf!wCY8ORo#E}kqWMZ4N-$ePMn@)gXfVRR9;bwHwu8ba;%@`t|> zlmBgR9rcqiO^9MN=Ne`%P9aD73A$3d3yn?7w1>%nK#Sb2&xB20T=v#f2KqXuU zwu2mzO^SIQJFJ~TXEILvO6{|7Wq$#0mYe%tt(oTOMM-l z6bi^qaDc%v`T}>E#f-^~a$y6_>3pJhssx*ToL#^a=6fLpJ5AdQCRe;rY6*cp4#oG& zuP^pv_~bQGFWuy7;&^1_!;>SJeATBw98B7v)WeOCx$+7~INZxHN2_l(chO3l_=liS z?`H8ecB4gh?_;%Rz_3xYfZxk#!Y69*GiMX?w|QYtc{)}{UfoMz3|yMnav4u;G)VE+ zG4g*evH!L@<6%@rl#5=Kevm|Qb8fbtoG$WEWW#u5TQ9-4LJuPxBq-#G6f7374=7!) z)$x{0OgO5Jl^T2U?Bb)0zRjmE((x>&B;$h~ear!Y39Oc8UTYv^E>^Q$4zlgat36)ww^$EdY@ z$pogA{`4X}8WyI_r6J>);3{LTEg8Hnvi(jj#-^S%-ZMn@K8rd*oP`W`wuF<^ZfVKM zh1BBU$~-5lZNTfe@9op239UcM}^J3OMwF)V^Y)%r>B z@pBuX!BImMygV<<<)P3x!Geud$+$9TtxhZ%VlGJW^W>C_r26Q9=NtQ_k7_~Dti=YIPr@(~Y}MpB_SLr{Ii0M97{i=M)d><8LU z?frP(3E6h>3<;cM-qUG3kY4j2zoh2Mn4qEc@6l}EyAi{;w?;rHl9rm+C#uY8a;hmb zwMN_%kx#h0aI>&e8WH>_OCtB4r=_WDMV|I0A@&MI!8B=Qy2+M)fabhgq!02qbqB|~ z({JkxjvYbc1(kL-eg1HP2uM7Z$Lgsw_L7mWXi>zP;0g9nLauhaq|1L<|= z5xAsJE5c#x$va_D)tW(HDjdePcH`I^oQC1Q*66tNti5x!E?^<+tA zu>rTOW7;&uaRgax4n*;~2FL#zT09;G~jB6cHnn;Hgjd#YTOvi*aSoL(wt;gl8wHl(On zFnw_2uVGG(H8h*&;BvYI414;+RQ&wWhV%|93q5tMs+Jh-I33+vb^D#ddU_S|^VrLr zjzUP8{d$AT62+F)sq)nA^$+U$w~Oo*Z@uM!EWGWgWuDEqsdF6P8bO=SFF80LP>72) zA1921wdp(cCxm^s+Wd@5M#l%g()q1QE;q?(ci1*YX3fhZj2f@|MKQ=0Q5duB)aZX$ zSgsLkuSqrwsYf{PP>D9Ar#3u)mTI)!gl|@0cJ?V-)CQW*jBw%|9pKZYP`r>ZruY2i zX68d#Ou=Q>5HEdJG*5m(=UcX(z-h2r6D+irnmQ*GrhXA=Ki%g=;plYm!h3Q$#p<5D z1zYz9{E9D^tb#mFbouN`x}xSM3SpJt+zNq;oY(;nz1za&7_67`<*=l; z(R5wbXf2Rzko$c}2YCP)zp{umuSSq~l8U=K1JmZ)_>ek)nnJnEFiYJxUk7MXtq{Z_ z{r|`H;xinBOGkH>(L9!grbE*MHzQgExva~6e}9-o7Sq9OhnN`J026RKr0#phtBEu# ze%T_reZ8|&_Oo{>KUI{~scXf&U%kSECtJO(QKe7cxEdk}<(Cq%oOz#S=*3K*PExWp z14*G;6;qvju;@Y;(V)Gu=U14sP|zp)**}-@fkXL9=t!gYxjB<0J7f(F?;fq2?{UY@oKjy(QO(Qb4a#qrEm-g3jY$1n9dSnjBiyAByt?bB9%%x!gB zNrQ^*=B5e%bR*Zahtw}DQ7BusM;8_OuHGu})(oVLkc|@cI?(RL#!*)KtOjLE7xoXByz<9<*2K?bh^M$=}KrT3*gw z&YUt9F=nRGrNUcRQ44XE@szwwRnzq4q1;N^p(h zwFu_yIj`W^sY}Q0A?eM7T(6-W3^YA|F9{7>?TiVB4eNNej)%)4N2iqwl_4TbM-z=h z4>!_7QJ2yZPJE|ESIEI<{8h=@OFb|nhyEOV@(|1ZK=eOJID83TR;dy`y^?IvbEz6Q zJG6fzv>|jAlz6q0FOa-EJc+0q-y2`)T8phASlfZ!6Y~rzpnc}`ku*VcNe;h4ptZPp zt($enEKW4_I}#j-VLfdVUO%vNn{p#I<>P&UAqAPE0e|RN8~~b`u_GhVRWM zrX4u&w9iVhlSd6h(kT;8^~%y0f0%|7+Owr zS0Et#D3kvkQYW~#a`z5y@ltlMdV4KQFGo{+c;IF{Q)jdN{-6CZbFb{BZetlqqlQ>j zom@($3g}W#)p(YT4s-IX$jg^}lwav5>ObE96SkYr( z=o0tYyW%J?cK)EPrRV>~VWbJuiCyOJoD$iZ!AD0C0`jK@eSj z#hL&qQkD6QLwqjhR119UG0LOz)qd0U`;xN=K8;PW_!&W=DeI2gv%BusW{9{?v%2Fu z*qq~OM5JgR1veVn)}1uNReBh4$8kWSw?~XKZRWKARtW3Zb1o;^MP6QBJw)J*RW5~S zQQ7Z$^cY|H5bxg#7!wR%W2@zjdvqCWu|utL{obJv>-{QHv_=JFm27PJ z1a!0oEnm&?pT3|LGD(`S9Zz`G{Ng@pwHe+xMR4u z8R^#w3;!u~Pgbo_Q~~zO6!irx7mcPuDcuXt0L@;n!L06R%>I&!&Xf;u8qZb4=cQ+mB4l)Zc~EP z6~M})=wTaEp=HRZ>Ft|njdHq4ZmUe%#qsp8iW<1dv zcj z6G2n;FFk`xu-z$8aO)sx=Q}qK4+#}5+F#M5k3BkFL63m#L*xiu|8z8_v~K+bn%Z@$ z802R&@izi;?G@txqoihx&IO=lja6YUu-%X0ZztlcR>bj z!D;y~wTg$0k)@n({}W1?6Jd70D&8lN{i)Rie|t|))DOSgkk_xj3-2@&rbxXR{X39N z-k9H*XBs`XN<-I_hi_B5h(mRB^3xmO z$R-#6)%G0+e3kNv%mZ=YOjd0Qvl8c;!v)Hu%5Ik!l?MSq#bv07v!+LAOI0uBI6`Z@wva>1}>$H9SW2`IMXNEsT);0#-La)L8mm7R0 zdXkz2I+Y4@x8x&chN?A{??lsps^^)gq@VAI3b?pLW8B``f}QP?qgv8+f2L8G{m?*C z@`J@xo%TmYR;OYN_Logw=qsZ_3$X}u8R*8aW7#9H?2&Co{IZ&;vQ1^xse#_H>#fp4 zkAAGr;kz0;pyD8?M|pa9^U#bx|GI~-kfjSe79%wc#|DY|hnC);>3;O-#!%s`K-noy z=i+hY@k^+~^&1U15ML-#ih1d(sHhm)T)DFAcGF{xs8Z9KROab82a1p>=KjxcIDRKf za$O?P=J!rtSd6H(;fbc!7xPv9#h9L+uFUq+J??^3G4Q%HJ|N@1vs_=wg*LOYfI>|~ z&yE>4JnUK|AftNSd*dw+>cYbZn|yNpRVzW^4>Z~uSNcp^eYkJlgkkkwaP0=!4K@@Q z03)zmnuQbfr;o#NPgX$+)JY=XzR=drx41paLC4A5ocO%&+HXkt zz6yX|^5QgY1kO9w{oTY^%#q9 zwmQZSkBzay$tMSQE>$>Q;pwm{hZaW*u2)nU`1ch_AA$63(yC0DtNAxj%KgXL-Y8p$O6?#`J2p-8H z_j+LTn0g%_FWsx>EkE2yyy)%7ti9A}35GppA@+thl&#~F@R@z^DB7qD%LvnMeR%)= zlb5#?l^2OR3l^5szkin{m~c z^Cb)Pc2r4(?6Ps?zdbJYW7K~BD|`C1e#(2}GgU4=f{WtEYW3Il+put=z(z*D+h0~j zlAhAZJnZ!ppSn1%3=40Z9-{)j9b3N^$lO=cp|g?ago8W=`AOe*xC~n&bPU_;-18-> zQd` z24tf{pnxCN`X1$yZ!>3cmPA0wJM$SHdUS7-1vpkJ-yyIU3Osw;zlHAQFYLYWoy9IS zlAhpqwO;eik|yIJw-G! z67Ddw3T8Cuz6~w(oPQ|96IX%S=a1NtEx4@%sSDPbZH_UH=X^zP*<&&PioH^yQ^c`P zcK24xm%0Tbzc;Uv0ZZ1d_k zk5cSc$(ZwhBLJH{p4;CLNCfWJHg-;n{vP!rKzy0eYWbC$)toz#m#eONFC|;4-Di@ zJfa3JiaQjU#1!9z+#@5R24S;uS0cJPaAVBrT+JDJeDc=)n;RSx`$4d?d-beIxk(>C zp5oxa_hG%E6uGZH(9+%tOdLvvuO*2eVB8ehItRY(K^=0Xw8#lvkSCwT2OJdGNuS$-UXQwbEm(pOYqUb45c z;z?_1%ijT`r2__;yU%D=YiGkSs=W-OE?S zERm+s^CmrmIT+rR6V6v6PNtk2s5p*oS^)ey9S$KJ^>n4id10qDo0B8FjXXbH74V-C_#-cRX(Ol=^-oW z7TzoYQhnwFE>me;j^D3=1$-}~y84IP0OEY`uoQh^Y*8%g$4#lWo`puIQ$bj-z+~t- zae#(B198`;@aN?xy*p>RK|S@KgJqQh?9OOAWLZiO%sRG1@tPM}1%#$lIdSDxDsc+XX_upAPOg=uV4@!{&LrBiW=r!Ki9x21@I#X8qOu;?s9~I{Nv6!?faZb%p4R>{ewFrq^Z>d!TQ{8gu>2YxMBx z@l&$)t|c22oesH+sV;YikIFbZYM zXq9tQEy!>q7d(tQ--0KFdHwd3yAz~^Iy-Rjv6NC(yox@V9=V&({_}y;c}dVTXTu|0 z@d&cp$ikd^Cq0i-%I)@tjwZBJNe+#f0(r(C>WIvIMqK_x-I*+i7-jCo2sO)h$Mwd+ ztsGKyh2vy)ytQwq`vG6SPR;63^1WV8<~_P=H_(wXY?n+?>Axpx8ljqF$>izeupnfD<2g*U3# zHaY%dJs6RukXt8o&hh*+dvFmB8W$#bTwrf0FU;2u-11$N49rN3feVV^obCXdyCllx zZWul>`79gy(jCL4_3+xpLmvB^bet&6O^Z<_u=dWX4gjA;InPh~IQ2{KXzL`K+ou~N0v6!kKjPtu*I z>N9o0$QQuf+RhEJtrnkYEf;V1>-+kd`39`kaG%W0WZsgJK}Mwjvjaf084E?hC>M&T zqLGLO`g<4!Prinhsut94vS7|4 zQb$#Xb^3hCt#Vir6IChJ{(ErVY5&rMxq<|$dZUCH5x&4@*w0y!-FvP=B3PHiHU) zkwJIGyiLzvRXK5qAd44Bu>adoY-Gv105cLMNs`V!_Z`gZHT-do;MCWL;|lNLPH@s_m;->G`%ET4PAWAi`?XnQ!i1CGq*%er&ox z_YzZYx@aDpxh8%Tv#-?KZrl*RAtUiz4aiLvmX>Ou7{h-V8K39qUf%Xm&plkbtKQ-Tzkd|J9hh zdPQ*mzP_O@#=nHzBr&z^MgxiFUkVRQo6B=xsp9-m`;X1(^90*;GtL(C---0`AqH1% zy^liVzcYCQ{r@i&ML*0cEQ|&zD0LlPsH%pO3p)Hsxmdp@s&1fN0Sovbsi8tL?fdud zyT|h(orT^P=BDQ6rDO4tk@S!#*3g+5eVu0UzvqZ}lL$H*v%uTldlRan=`?+ozVodi z5EYd$@P>g3V2bkBqB-E~9F6C8m7R0ws3RLN15!Ttn)avt-=rDjzEL|**!sr{gQzZ9 zSvp~1T}rRx*9LR6_y3l)kq4vA!&hKe+O$zQ&Hn7!GYvMrDVJ1Mt)ixG9u*Y*3?}8c z25gjMgz; z;|11sX_h@L&kT(+^WNJ8MEK@rF3qTAGs6fjnNuyPX}eA~0t$f_rx>jJ#ZkY>YwrvW zMz!t5H89>j%xr%zW9xL+&*_@xrS7sl_>T8nPcPvda7xkmFNx2Ta^HTQ#QaNt(B(UA zNspm;V_*_Nl6*nN)Q-HEy+sGKkfDQguVQx{PCKX0F*2(X_eUt|9CVpL6Ml;O=k~gy zB9w`Z4X~1f98OjcWtp6K0aX^V&H90!*L{EUd$0m*aNKtLv51J?JqkfOY`>WCRa(6_ z5?uY9mD$SYMg&RsO(ldeQ1Qw$)8J?m?Y~kKyC`G**7(A%q{A~!lKUqHk^nitVtu6} z{|E90HRSg}tsVX|2gC5_c0!F~S^KyCXz1Fm?F2)7Dpm~#q&HI@BPj(iJJPzIb6iE9 zXHMDK1(6Lu3_Mdrxc|ZR-S65cdHW+e4P+)W#*lb`niMaNCN(swV=Uy&%xS$T%Ot8d ze>49(HUm~!1uw+y5apV{p1uE91sMfa2A;?LDYrE?5pw%4t{AnAkD-6QAMA~iRtn|!d!CrvG4?w<#9OvN}_n$Ec8$bynrfHrs| z>dx_9Wlk%D zBnfRJD1@!wn$J;Chtw&Q0u*4OYk`4^%tj3^Y~8ya5H_PY01*Xx>c2q%WJP9OHQLO#P}Ihqpc`d4Fj4g(tpLWKq>svSRK*KEwm{0jLy zib-EHmU80)dh?gKqQ?p^Q#gP5dSMG1jtwd>i z8!tsp440|gQuuYEe-IRc)cK^SC2WJlH7o!b76ISe=OEC3Zv8wuc_{5mLSsE585gZD zxR7{Hbt7FDTO}py7``=yn5V`EANtS;|!mKJ*z<=Jj>iiCA=AFw>wZOT48-G;@YvP%FlI3b%cIAo1xG+Il-Wr9Dl`l*E!~W4p@5@MnO35om&*S-c52toB1}E|{ifmm zCrN@wk0xYzD`$M-+2zZF{Xf2P6wH|B&azX;Y2{Z}TY_Bu`JU4inG(KuuIj%ir&=$V zD(?maGRGrKVQ?4e9`0d?j!qozx|l3;&aPyyEuxk2i#8&^V#RHs#5s$f+34-L91x`u zcxqmzJsXzxm>3KkTgesLFAMyV=eb+kdNrUKCtZ7exK){f9jwdq`Ut0Jo}jRsEhx-m znge72=+&2n1#k9%ml>VEBb*mY>%BMmGpiTB)LqPSEE=9laoYVs z5BTzkRp{s$BOIzqDK zu>u`E(I|z{= zBVvd>@*8PSIQqqym0N;!ncW&zCrF$j&u3-KA2_=FV{y`7LY3`WEMu2*{N9b=c{n$( zM!V(OrTkLw`9~m33lKHYnyap^?5Wt$T0L83!ZtDdXGEQo+U>pi)k+vQ%f~cqKBs0g zf8rHwA+TldQAm=ZV0M8lU}JKQYrJ;?IU2H}T-4zv!@83O={4{@1RTGQrAYNL8XDTr z*<(LO$8|hqMclV<``{auKw)(BzKRYTi~mS@K#|DC;xtq%MgX2wc%idsIL0`n=(cpv zF7=oxZ+c`?FsnCLXt-U7UvWtnN6YA(|?B!^@xQHCjD8$wXD5-(MX~2Uk zFZ5o1I2cp#RvUhh8#?y@sY}`zv@AnqDrv&>wI6+tLu@Z|gweycV-{^vDM*}@7&wN= zSWb?LPAXYSbaK?)GFP!zKX^FFp?vEu`S?z$Rd=cxhC_^~p`{zCua}v)sp}6(nVk9f z0xx)u<>|yu>b}Q_9Z&tN2+tL-eD~-i2he-l~Do2UQ#nbq3EPCH_D$n zlLHR6!q~nO^vlD_Mr}ylL!vSP@6e#$LE4glmkhF&ddnZs#s-5lg7X!n-{XFgDY9zn zSH&h53!U{#E4ZBaV`AzW6%l3BmQua8nA&^!wmZ-?F>{KP@nPWcB1-0U&aD~~9 zSVvkh+^(FzP%l%ee0U0Z9IxGAzP3WXf%~F0OA?%Z6&3N231O`ZbxlN`Mz@)Uy&Oec z!)u83a$yF-!70h>j?WZBunz60>uvR2H>>X-EDq|F>vd4t9w5S zwfMAF2=_TNs4UK$wRx|`IW7AIBXJeOjzljsBiDzthu6fQv2khT8x!wt^_1mHPV#GL zImo5Z%o!HBDOj4D2bzV8P%18AO#tD8$Wc*q?$7trsGD%<4#wpR%cYOYLZN`B^ghUi zw>#!~J&7c`+c5By5t>3(o|c4bQ^P&(rmLXjgEW zjb2_JsiNc{jdTG$E)GvqOll~(+BR2F)yzv|-wptZn3xF72RU(7hca^ePe}wU8r=5P zt=Hl^a?k-Yb+?bUm=H1O<1BVU?C3YlqN1nA3|PvrqQ18R4s(`BNs)GSEy4i28BRVr zxtfhWo|adqG}#yVcy{(?f|<^z9D^h`l^4OWG+#sOb?#A4(wyO0s_=QMcND#kkX2R=hjoytxY{%R)Msz}2bfNT1@rRWJ2F|xKr8MC#dqSM< zV+ov2%>=p37v3M{N*rAGCgE+dMq*!^U8cRu2JH*IAzo)A;dZ?j=9BG`qu0Rx`27u4 zOyR-1NO+hZ<>Gid(CcHa*9#w5ym;fi6^e`;f#|YT!Gj46k=V6Oc}4oH+}x<{`7F!CNI$5~t*lr#*3trJHH z7~RrWy{$%n?O_i9Ch_)dK?-f*qDy%C!eI67E%#(-&Qq-!po~(n5!Aj%Iht=}VbBut zLr?yiMU5bLQb7^yBVdL%)4-6(#rZsuzuP|D-~t>k~l zBch_5YCHxqTtn0%j&g*tJ9sqrif$|yh|5p0g*N3;(2xd;DKcY` z#YoW)lX1w;f1APuKzpKXMwHc^3CCeQPWMBLwHzJmdi57CU)E})XZ}^R`Iw@2+4AH| z-3?%T_Dp%G5Y9nQPyeNYv_JFxwH@AsxO0pkA8tHfl_WK-d#0}YM)af#?}uw~$>SNmfVBZdF}A>m1z~mT zW?XK;PwqdJPOjtB%*jz`ucWLSPR_zYNs*H?G0z!nJvYjS=}dQb6Ax2`G4B6C^NskT zNY7*%fJ^qf@V0C!7V2nF#XP66l~+)RnwujCz)G+%6C)r-uXxzjHOa!~xu~JjDn5_8 zvL`rZhy&J2dqr9HmJ7b7yvV?=x}uf5i{+q5eKq(Rd!2~dd9wR_XtUjx7ipM#qkDx` zHJmKPae8%eUajfTylT3IZW@!+UMG-iI5YEWi0e=vi%6B3b!?JcRP0w0dejQw&vG zDD7FAwU#yBu!19MIXMwNb!#rq&X$>SU8W%6iAY0OQOS+1QK17{$nd0pO&WTUS{r~= zJXM+bCg-yM@W&Yek|4QQes$SE$8X)D6DqMEeXg*%d55H-OO&olF_{+}E;J}CZX?t5 zCmTIivrvQ2h|M@aGg_2X_q~rh@@_N-5m5dh2(G?A0;yj4MytQB1{@k|o0GNlkHHPu zeNsgbAcGDRBIXTmRHUz+j5Ee)286?ziEgqWnm}c@#KkE$PVD7_5b;XP4siL`XO0is zp`n)6)tyIiRX_-wDfcEo^XlR)(6!;DA^!*ASR6V+q$Dnl2tYU#v zJtr>R;c9;=ic3W|jKS8A(dlZgRI{w2qOHf%JN#pdEMk*{)5@XuOU04IxbD47u^$P{owkr!XJukI6ho>&HA3UV6UvcwPE`-$^4i;gCY1*iDFqi7FFaLw%o?=NH zwX(GZ$R&!w;c2E%sHcpQ6)4q|Ri~FRcNQhQdlx}4R0(gTzexGm#sl#l*_le+#yYTI?h}MWIqbV7Lj*&*pj{Xl}NgXAUeIX=`vODl_>=q zM}FJGi!VK=r?LSdaXAhDBo*Gn9nm-Zn^{{cKbag6Ua5GGe3DKm>}pt6w6gBXJ6)On zJ5eO&ZpXNoQoDWx2(bfhdE)jZJn!Gx>q}9uOv5!I4Q^>+uO{}wqjqzI*WxQGG?@4u zrJc6ie=yfJH0aLVQuul#cA+Sv9Kvg7LBjlhu=iGBRX$PMxFAX?64KqBigZYagmjm5 zhe*RlK?wnA>F!3lyF{Ac6ueZTYn$#?Qy7YDlcv*(#vGqYyqUiZ2OdEsX9 zo|W?JP%H?m_l2+8^#i^Z51RgIz1)OG#>U8jjJF;`0cznYZ#aM4T^W(UWwG_#b>`_l zA#^)la>xm+)TYngvW1(=_{|ivx7HnvLK_eD5crwqQvoND}e!{ScSR1#%72Ya4`F|4{t@Dz96E#y`t_JEAftxpZYC+VA}L-FnjI@LQE(bM9&#wQPe6t$}?HC=A?K$1f>NJ)Dsz`#x6P)x8* zL}8f5Zt3S72z_O>Qq`dXZa zRT2qjnaTC~wT!7ky>e)dcVTFi=BCcYH~GxXclVY#3m_d&Kt{)&K^$jbo-EKZ(|P5m zRiV=F5q04B0-YaVHFh+VU( ziIKu4qbY3%>AGly$-EH_Nr4c#N=xHkIx_vgiY!n=cZcqaa^C3FY?tghWA0qY=r&L7 zPmJN8#^hu(Z2389cwP@0hwh4AiE-=kPLL$M98S4ay<8sarndcUUi+z2p=kSYs7RgV zTr|;XC0rwXmWzFE1|H{r`XoW;{`8-3tQ_M37Yi;6o8Q8*IAH6EumUn(WR2kM=@?j$ z?h-4K@6+t5Jl*w`of@6#v=%Cb%$I9Nik+=UqMOn-ZF_j{ulgI&(9mj^OXKxY-y~)3 z*!+f+aj=ay`CGUIO7iO#6ND&N; zYbZ1D;%?2>YjN7Sj3h!QpBr3h*0^|Iy5KHoY-shKHOPSX+RPf^ze&HoS1dQ{sDO!T z3bzo$OHdViob8pxxQ2k>hlQrJchbPn@Z$Dh*Gxk972d+>lq=2MXg$p^0I{gZzAAi6Jw=h zcMaub$*B6^uL%W1jm9-Z5Fzn!q=4$g=~nK`pw&4p+Y7}@faZZ6Pb-cQaEv*&uf4BoX2&K*}^n8XmJ*(^f8tf*6#uy-TKL6);zcPiJkEGl2I<{GFNwnB*D(_sK0mabY)<1xOW(F zhJ6Z!O0nFELGt639WO3ST9D8upWC5YQj=v@JUWbIW1AN{-$0pv;5ibOTBqW1 zx>~(>c`>zvp_)>#jftT)2VHGc8WoECo@l)8qH>jcS+4D}Gtn;7Ia2jGol>4>i?IC@ zjZnd?j|%l)ZB>yQml>BMY!JbqdVJ+#^T#74H`(I7_a8&f*el4*dCDD`p<%0lGF{d~ zDE}f$7(#%-cfOD>hy3+9A&Q^dueP2w|g7xh>GJ}IVB zbaCm5?{Z^GKWdp}5?#}#KPE?F0?pMotoj0r+!aGeL+t75z#o&SBC z;PY%tC|2aRllDSf#f}72$z?(^pRjyBsbZ%N#E&W?)NZweHe_f^al1Bq&$i53BqTaC zP`S3=Y!|7QkX0(SGwnM6ir4$2pdABJuZrsr&vJeQ{sZ~t;HyX-8dpR4#)-tC)0r*Q zwiM|a2z9JSGY9wY$FPHpFMjikvU2DN#52lqjDJSHiu*N2Q?%>t`=(JuVM%K6N*IlF zCI=^C%Vvzu9CiSKKuVCNc^xLeo(qpnn8Zy3@S8UmOESE&+ReXtA>>=hWL8ip5AqaO zZCrGn5>9U8mz@Q6Kx3M&dz*Vksr%Pi34K=G$#wp*DOxn)w1v9Co|@O~?`wMIQ7+Hd zV|hpyQNm&uk2t$sAPSYeks%iPE1a6ifYGm@Cae3f%YKSF?cU)GG-`H^WrHUo4|53S z#r?iQ1fHPf(aNYyX7>4t&h6np`j2MB12?yrxUR9n$^I#PIBk>+0UaHhTc+!{5YH<5 z?*J+shCKboVIMqFpWn?!2M{24iPS=l?mC|R1b}G0N{#x*S%f`hqrc+(WC8Gu1nGXn zH-SU8F3^7voaTBBi`16>f-qyL1D=(IMV^?SeEBZ{8_5<8-vJ%4<1Omyk_5VFp#K*! zN=;U-=ADLn8)_M|a!Y6VEXM%>EA}M-*PcFwMwBdq1D`B~!#{w98cv(0*skd^8=M;k zvLLmfcJqus;-PT^DZgZ%qV4Q12GKoSbJlW*U>#etlgUYuP6|BBf@CygpTdCo*nMhh zZ&YXY!Quv#aB~9Pz`NU=Zl~vY)BXm@0^b0VY7B74`bAm;n8Uq=uOe^|aN79Za5BWl zVfj<*BLo~4J>@}Qyq-zlomjeH82aov3^yAg3<3ZU$SEMp ze0&80Dg>ih4@dlqL`YFK3Q#SQp$XMxunAH2M2mPzzI+IeIDuKB3XW2Tq>mraymRb2vZ0g;QZ6PGTW*1kER2`g0G5WI2u=0kd``ByllVs><%pD z1}AiWW<_0PswGeRFF6HS|CFhPFiYiD>OSpj#c{JsGxQPMU87Zs{~iIKD8Qo4T8UTxAtu0PIxvtrFeLc@e8LZSVLKKM zJO7|}Sf|@d1LTqV4a@&xljuhk0OQT+%zFV_68~phV5-MX*ZePBDcmv!gJW=V{$NWnh zfOS*`7!e^m@}K{^QVC3%|I3A3BP#3bvx4nZE!_{0_Q&&a>$mvz%zu0s_&Z846otHO zf~7=7Mb|-MTde;5)zs+Zq-iA&BO_pgX;QSdwpIoBO8g6uKp-9n@G6eKlZk6`JY~dF z=is};NMkcIsqlFmGgVYnq-14-ne>}<^t>N{uGtZQUsRH$X!a4ew-u2V^{ z%5?qD_D9aZ^!kV7LN-f)eZ`GDR{B`#XDbk`oE&=n;;lYI$XB%bRej}C&Z#h zNT#T!W-4H~JZn|LNk$;DW#J1pUxhirnf#NRmP)Hbz)* z5xz_5Y!1XxhGVdgvN9|_foC~3@l|vFpjOf%tZz$Ub5bg-t6S-L+MA&CUXMD>%fqAQ zQ~pVeFu2;f@cyi5F+bVehEDN^i&oIz;C11@?+L4tY4l~twv-U}%I}w3zeFS?Fy7nW z+hwvf_puCSCjk?rw`|t3_J7Ybp(gxM04A z6)BNHke+!kq};oGUoSU_;Q^j(U^dw8fC8pbWGWeFcaceMHE|vQh4{&S#DLWGK`k1G zV+@kq5QYb@?LG^dVL5_N;y$M4KK)&h%T@lQGQN z#sdLapX{?kkjU(#;eo;w6-9dU{Ku2N0;a(JhYIX57|eA*^Kz0SMl!mj#VvAm_bott z>Z2$bGgyxFY9uH!MCu%{bN*quRIrEtM-=FTXr%P*TPAS0@fUXL$M5ZV7xVnwGb1=- z@4>Mi;lfr{adGj#;FbfRUPa-vO}ir2Qu0J;w7PF=M7rG5E#kk7cd%4dz0-ld_BD}J zm-R;DGP%4){CROJN+tln7%AY`Qr_*H^8Y3UY7rQ^R>35>(*y&-Gs;&So&Mre9T}jS~r%GyZlZZ^;E3XP?fw98!AfmiH=KZ)fy7hAS8L*et zZ`|V{5^E`jmkSyEJKt;_DJ1Z>Y0r(eli9GMGBDHYJzWTt|74!I<>ih8oHli_UGr_# zcapM>RjE5YL+?N2Imm(+r%P*{a`$x1%nR4D1y@M#>~7KR#i& z=cL3(L2%~soBXxg*a4t}E6+h%#y((o?VWO7b?`?MZDJE`O>!Ox zUQP$PWaVe)Ds#Sl+i0>_$Y;{_a{?y*wfSU}1MFRbHC8G&P3T9|5kO{T&&3QfrjXH( zx%k(XeFmCtyRW6Scn55p71`jEUn2cAHc?le`o{~>?Zqrn|FYFq3DS-n^LsACe}{{1 z2H$NTsJZS0_7lrwh5W^uQ#)*UZu4n>BzL;1Qr0M{c4y{OqS4X)0){Yt(c5g3GoDL8ncK> z%1&Af6*v^+jg;e3%M2q)l*{yE_(daCCpAw*frM{CelS&?{^w@_B!-Ps z&Sz)`R+`$_1wyAUKdVg03pA^zaPd2Y`hAtq{8nPTO!sAJ69Jq`-vj1mtO=v$3N#U= zWRJ?8w?rw_TK(h?BY|=2>~Vxgy4c@Q8I$=!at>a4`BU>D_{N{@dXW2-zCP<;eQLJy8TU`S0YF`wZx?yde9eMn9<4MIVv1i=jtlybtT?p zdDuE!UgY&lEiA%|G_&9;c&*^RB2#&eSGR)@VV-HZ%V!g+(+@mZmaJNMg`V(LZ?TY! zhIz$6O06DEh%Tk1@*=jElS+cjnIme`kA07nn6PT0VhIk!p46Z3%h%iW`xYM$yF{f+ zVuSRmpT_HZ(Q%e99R_-`{`Q70^&3V@?a0xF@YarN&&}KeB8%2x{Z&R$(S!_+%b~}6 zkn~un43KNUKJ0cpU8^s*IgA{wwyThN*}j&=Pl4<7CK`Oc^!YZ)tj=6h^cN>R*ITe z?5y)6lX{1HLmOdv!JXyzzZ3p;j!YGOvQc$XK@ARNsJEkQ$8+0RM^qV&Ga-n6gsK76H|LTl> z#*4Bd{tZxEST08`!Q4z`z6;th=VC>-N2;i->dh-eal37H>1$r7mX05KEjt=v_;)hp ztk;LXMpG-E?_M__YgpZ7cIr81hE8&_Vcw3vDlP7*kAsIdAlusb=?3uGrar^Fee*y;IS7-N35H z-=a?f_X)$9={53Q@4_PQFMI3DolcN_;sL# z#jMN+g`T+jLUNv1AL~B>=hksRfVEj2gKkr?Q`>w)K=jw5On{-qS76kZ{9ubi~QZYk5QX!LH^VS?7Ou9!uJnK$| zCj*K|phW!U4bBG|&dP~*fr`>cS#JbJx!vX~lhDGHGE&>IObh(pz@Te`kW6_Zrx52* zl9BxaOgmj$`$Kd_>B$o4r8}SN-t&o2B2BhyJ*%O;pBTa8J!TEY>-#oB-fPdJD?(#{ zPGyl5EcbFVr+0qaMHBiip+f#?2bGkGbK1Kk`!71xE0P>0^U%r4byYVdt(dI6&&Dg$ z6PU!rjNJ-NX2fF-KwZ(Qk%T~K%yMZN?nB{uolQHYnPv&G&))R8dSDQqbtefP9+g9% zHNYd+lft|+BJNFVBYZQ6ysJ^SFl>PhU7EavsF0@B432(#JGMW5Rdlv=b?eVzHi(Cx zujW&KIK{Z$ll*Sxv?E!*Of_b_w=t|ZD%?_jr(B!1H*a2veyF5;D6hyeD*eXMs-sFl z!>;yybU~QUv2?cc4Jzm8Y(C~_X|5m;tlBA&kN$nMI9Z16@-E!({pQB+P^G}F@;MsK z?$fQqhyL44xroeu%hnRggtS*7{6_Z=js$20lx5qLtnKL#5Qsq2Br9@H|ii1zga9?*xI6404cDiuo z#?s11SMGW#4Rt&UqVChIOQLsa7RTn43yr>?@^-h?vnok-@-f>+{qp&7(XM1cxLWrl z7l&8~O0|!2_W8DzNc*sycy5VBlqF|G|5r%$6JDSA#)Wv6ovDA?tt`f38^epK5>kiN z5^tNi8#I$W_UmK4zUC)W5@2(-5crNinan3FanHkOL#;M&-t^3h$C{l`(^VdwIAA;& zbFv2s;kgd#2~FhhX#LFMcT~#>KX}=cG}lcD$!>Vb;tbXf*I75j4=045)N%^E9Ud#n zXNjh_G={b@GdTA6*B9pCHI!f8Yfx4oi$+BC*EYAX@@nqasLZ>E7B9L(dCzehyh(E< z9HDAi5+20bF3*UjE>;f*5*NyC_nGhTc-gN5g;%EWdZn3nUfM{d1mCkv-q%bfBr}vk zqd)v*U2+|iVcGPwsPM}Z8)J`7^|7wGS@GvHX?4cRqRUabyXt?j)=Qd_Ff^W*ezJJV zV;5SuxKF3htalZS>}W3eHBo9P@|M&@O4A`LK8Q_~)JU|Y0A4T!7QyZg(>+k7(l68* zem-&XI_G<{{3O$Tpy$ahYu|4_`iSt#lZZhZOWUZvK1?u5=XoWEpBEMF-|Pu{wa1ds z@9ogA+{SL*wU_9W-z4)c|12OZn9WJa$hX7cyU!_3^YT_Lim$FSA9AN>60{vOY&_|2 z-u)TrQCp>=mK1SbeJ39h;oTlT$4BPw+P!_HS$?%qfLaYC7aQzjp@J0(QJnEQ%G&eW zu7fiOLRy0^CQf=FF=N*qa3V=UUfIyvGt`T8<(6zDEqrQl5LtBN)$~4Em%jF}l1PEWl&#mFcdT_ofg;I7&;AS6@Nrgo zX=%es{;R%)$#J-D2XD${X1El2nDUtWDQhZEtG1w^wp{^15r24{?n2p;AO}IA#On;o zRqE{f0C}6qK*$F^`;U>F&kf3DK0&in#I;lsXB@D?7hU-GMR%=kj(02&+?~Fw?Syx7 zK*u3coUtd42~G8ScAKoisW}L*3o*}TrD}Rt#Ni7?PIJcx?V{wNb%l%Y+DLy;p_wqF0U5|$t@9Y2B_KKv{$w^Kwl{}?D z*-xmuB;WUf8v9aBRE#%Q(+5v=3j}b4lS9$x z>%C-J@KwE)S8FXEneEKo-&ykWVZX(l%mu-z>ae4o^Q&zHTA|@sj7Y>@MNW~z6xZML zAB&%Pp%KC_q9fh-&o;a-d*yuhH?~=YYvx^46^%p@%c-TZ%Xx~_wN%lYXojpEjCU5A ze3{wBDyFmxCnbS;FLcQ7=itzqs1b-qF#(O)v%^eNt$xDz@Xf)~vHfVxVGDPr&9l2cQU zt7Yy$=bWZT2R&2GYj8)!zRHEemO*`qf~**X$k?j;<%tm$`ir%}IQ#Z4oAt#7y9?@i zkHh)6SNEFbOUWJ^1NKR?GAZv)KsJ5!bk&L3I(1{=)ok8Iagt09G{bUAA9>FJ;mM(T zaDk=Kdff{Q@9jhsN$!j>!`>D4LkjAn!{R3lge`83=mnuNhSd02`RW#-#2!BCZ-_?d z>xcCDAuRiclhrG$&XbVZ`~@$|=zD^_#pY*oF1Oho&eY8Ou|>(DsJLOPj^l2L&u5GL zjA^}+m_&jb9M!Uh{;Dsoqccy=RT25>Uf(JGL5_vCG1L~N_=)ZDl&Wkx6%{GDMXM*z zgv^;16vFSK<9O>KU#?oxR+HrFq=nOzUALs{xTac~u{SgXrMpaZJI9whCp7nAe0j|+ z@OS-cQ2)d1Sjlqso8I6*{Fv~FN#rm>Ffk?Rc1%PWRl5ugA02ysf78ACEintE<+hc( z?iG^H;G!>I@?jRZEhx3x@b)4+i@SXH8O3LCF@OnbO?F3a;aS+Tu(q-!-1qr9>=k=@ zgxwP-NmdGZ#8n~=8kJ)f_LLXiq^atcxXfxAx`f$PAp)O+8B`2vc?C}wtA0E&B9+to z{HmK}rL9-0L!5T`CeWVg&XKWLtZVsql~b--3DZVnJ=`~Vc#KcvO6M~cQld#LR;}Md zxEpTy-`rJ_7I-iE3z^xd#yyeQ6*r|3O-kcw&7pL-SR&)U`W=a%y!qA&lP=Zve(@s6 zq?{dG20)@9@RyS%S(fzr1BbG%k$mFcBWX+ebjgU&@Z9R1afdq?gH@nyyV?+Z7g0@~{-&~}&hp?G5)VLl~G!&>nl%9py$-8zbTVuMNZXPtVAiZ%C#m9snh z`oa*Ci_PpArKRdh72FZT3EW2yr6chh_b9kSVCVs4{-~`Hq%=*k0UL~IFsAb^KXo?bQ58(`9(x$~ij*-m3 zutB-&q5OKgF$<=oN~~)aA}6)jy>4WLcLhkz_jexAkooBqXqSx^EnC?;+{LL^Px0CY zC+%0?H1L~JYlK*p7b}yAvRmj_l}JkOunOU0BNLZ0V_s|J8@plmDRQotW>#Si=SFpi zQH%V-C{|kPX(O8oh<$A%J!dz2Vxdo%yi?JA$?E~Xa(p-wqTOJs5X4z1vb+P;KHM2D z#c{vH_MlMmvR{=g3;!!4R`62#8@13Sb8NejVzH6MdRgP2XBg!3*)EQB@qTz~E=-q| z;Zq(gaAPQ7;NN2J&*n1gm2_XZ9=u%lUhAH)xn0=7sgB~j)8Bd`bwHE*(RNm5uypT` zYrzs(A^k$_B1c&4H?~+)oT_CVG~?DuF4$w;dOL#>nH?A z;g+a0cyP;Yu5RZMecYq}v;gYg==oS6yY%<9Qe_uWAkVL2rYkCT*2!qmYx+?UKCAf< z{#rFHR(=cp3(ER#{&(a`=kFzap%Y2n6^ErMT2y$D1rIKVY9^QDEzj?RmTB;qH0X{8 z#LH#B1*@;#pg^Gz38k zf-vt3+4d{2K6c+2z>i{di{v|6#{(ailiOc(5LDRCU_%L2d$MU1A|E0Dc!ccA3pa*5 z#pAS*D!b{;?ek`IZ1d|XEnFCxJ%v4LwJS6rZi5U~F4;Z8J6au+;3?A(_e~#TYhMJZ zmQ#e!pG~^Ux#zvk@y6z49{+p4jzWt{n`)ISGX?T;ZD+ym5`F~)AyIg_ClI>zyZA-E z$|gKI?XoE@l`uS&fg5}_X0g!iDHiXituR?j7YrOsGPrYE`@g4ez7z6|+#>d2L_E}U z+8>hVw+n`y|5Io8SrY?mKTv^r1gfgak9mLb*vm_5pJzG4iWG*=CO07X6Yn0ZK6aEw z4*riuVSl>z^mG}g1TDSa6jsR=0py<&HGMcG`=EjzP5s;qm{OuGU?PI6737`1vi@s` z6qyrzHhOV+=kCr#!TSQsumR?Ed{Dh=a{H|dA@lLl=k8FO^*0|@gn@a%ACW@K%eDzn z$~%LoRmlMo@R5$Za23d%T~@2?w1SJB6M6ja$wE=^BruW9LVAgF?uZg5nJ+(<6qp6? zZI9#40vdk6isO%h^!rN__AL;MgQpT|Ity~-__}3bt^GuIg#If zWME^YiXW5iN2Ux(NoBtYrNsA^rp9mf`|+{5vmYoy2^+MfUUN86%!ePmMJ>`SkUnmY zJ_&bd%`W@)RpX2BOb~Txt6dJ~_pE3hie<+?@sy03y_Coe&`Ir!or02&bb*ra%c;Pa zOyUMWT~edc5*y^M`;q$Gj965;>`p#^PWZ-eP1KAW&V_dQuOgfa;=pnELvPp6!inYr z)kCzoDjQJ{Vj|g!qF0`GY9Yc#{t1Ef0lr|f8Gh2kB))iyR~KB z(cQhf+t|>+=DahmQ`y-E+QCCB{3qT?txy_fG4K~JeEE0XVa}%d&q4j6gt0MI5H@3W z*pkz*K<@jDr}Ur_+yRgL3%<0JYd*XK?wn?McQ@Gk_Udgzq0s%!FOt2jk&Fe8-$ca7 zH9x_V!L*Qte-lb0tff%%7n^YpdWF>#+TGoq`AGT`0xsL{cemHtc5D?t5tzgvZ4)Aq z;>!OsUzVfQ^;$(=2>2nAdZNq5vP_Un^+hwT0w%wGqXU>c^NEjsq10R=IJy3-{TU@0 zdtt<83R&P*X(oRD0(~DsTR^b6F38$_c{MrPb)p*>rut*GR{> zkL2Z9lP94?<0%J!MkD;k(k|V4xEKNDC7r?4*)$%^Bdo==fU2^4-?;ku`?qTRh@N|+ zkpkw7AOha1C26R&KWuPHwLC`#aY7;>F*o5*Xf;_lqKPnX|C0z4S>r1(he>wO*MIYt zJF*2V79k0CP58vTA0c}Ys?FYBW}k+eKp`AtpjY5tl2AN!DEBLqWFn7GIMUl1gv;Wwy3V;5`FO#VF~Phnt;hVDK&M&`1n(4ZnB*~q4gVu__yMA2a;!l^_h+CD z?gJ`LsjyluYa$#pUE6~2?+b&CN5kzJPkQ8 zX+&Tu{y&qtQIf?yhN+kEpZ*rT071|9rDQ_0eFN5aW{t4^_V1BD2&1NGspihyj-SAC1xA%+{&p(+zgos&7;HcW zdXBh}UD4oB)Q97&LZe{6Vv=13VbamK$21Db0|Rg|N#X?Z@RzhLcvCm8j8Xsn7L5vw zYS-POYBKm3Fil&WfT7$E=AG7HUmIA0UNqR*PII7I#srOGX&sV(MG39}d&J6GG9Z7# z4eb04eY)O-wJVG?EFb`h%}Mx5dUL7o^qp(RE7%--wK1cV;R&Mb=mZ4ueH;F$xFX&f zplb6e5M#8hrDh>dPtQ_7+{v!*Gw4%D<`2ZxbA_WYu<1YD9&=dh!I%^El!vz!2i~tJ zd+h29hMiR~4)gD9IiOSJB=KS^kT$smWc7p?wtv<1fe^ug2^sT0Bhz_za(-NPYDl%{BEOVU>eNN-cF>H3xT3>c?pTgOhEkRcx61*M z=};o_aYs3<4_Vy5lx2*5%v5{M)^X;GJ-Lqsx}yj&zYz-1b9ry*b%Mj*wo(YHvB!Oi5JrIu zG>q;9As=@}9A+)QM3iFKscU88BI?tHA3`}rsf#jM%+PF{Dt7WOGBDJCy(5U^`)NXB zjtEBLW8^h1Ct=K=NZr0;?b*^`uSusPCnWy*VQV!11q?Wtj5B%MW3d)kr~UwUb1K$s z#Nh4z^O9>p!eCXG2qg*xBWHU6&SlZfthm_B1m<$skQ8eOs4k=)_5ubj@Ddx3FiHUb zL>T1P>@6Y^0=IHR1V6C4!{am<4qL?_%Aj)|Nl&b;u6_Y_SFGTT7UZLZJEsE6LGnr8 zd)S@@A`9z4Fm&IJV%UI;KH46aytjLaU=lKe*VT2tc0_-ege35+O&96%E>Pe!KOC=g z^2ocGBxC%>1Qq**2|S;s+1F1GFD_mmrCtyvq>EiFuYqq)mQ}4{$#wxj*V0PfAp7t3m`fZb< zIls$oIbWWztdLW2zr$ohgqs_H^Y->8X+*N>OiV)J$NQ|b9&9);UOXfJ<1h@4+llUF z7URqIE&7?M;i|W|Dy5rv6e8&u7zF-WA&hkfRC}X4uni3q|D*D?Hl2f3TQ{M1g3BMU z)pw8DJtyi6*c9qi51y*q&gEgkWTOS)*VuHQ`?~=mdoTBR=i)p+_19?D0tp$&RuheH z+E^w2ET1J#48PBiFOHgG-&=B|Ny^fHchc$8b>ruEvDUo5&2aPx+4wD_jT4odf1MY8 zf-$OfIh0um|9A0w=6Rk>zNZy-F+1G)z)aksV0hQ8$7uhqv{JWTsKXv0dNgjXK@u{n z*3W)w3GGWWOvvXcw(AOV!xuHyczxrIB*m2oho!w*)vLR#nyFal#4ij#gJF;>{K?OC z@ijsaF0{+zMab(Af!BV(i9nbd+&7b4j=&IH1^fp4^FO>FJ)fX(SGeW8PM%{Y8t9;0 z@D}#!d4Y<&<`3)l+~fF{@an4b+bhrEkhmSSx>MieYvK=Ds*7HCRXQg*rD14P2tLYT z@k_S~zS^3rqL;}>e?)J{;5}hCC%DIt8Q?y_U^Ln?q}f$`cY3boCoYj@(nzp1QAa-X zRR_czuhDTZ{2gw2S41?ZCii&1=da7Io{`ur?>n#V6XzFeXYNJqUego8vwk(qySJU^ z$whUqvdrE8!wPo)gb0OEJj<3VoYp#?=;5(Ozqhg(=(Mw_KHYff+{^8_TVX{pt$z8y zH!NzatzM+97MuUsQSQ0Pwy=*#?dfxM>kD+7RoF0`6dFC zJZ>L)!?)_C6NgA`L%I@?p`5?dt(adP(|>Q^2nd#L*)Wl`*PL0b@N&kEcYglOB~g;I zGtX@*{^+cC_U@IWNSW6TS!@*`W|0l9t018;Ra>!Tj5Hx?I<4Wh3a# zu{|*d1~$ew!xt5@_^~=vPM^%Xd7_v$zkKrA-L8#T?LOCODO-3OP)ih+c3#~scrTnd zkNcf>kTG2~j91CO^9vH2O3N3TISkR;Ru`;KL}(o+SWj28_`VwXYpGP8i#dYi-kpn7 z3(^dcxCqzPra#jzEBpsu=N>i^pRZQAFX9vwv&GmOJU7{ho;?$3rY9wTBS^0NYJL9u zlY&y)^FL9%IiK5(T8I1Dy@oh$8do1(Ratx{^}Id4DHKxL3252BUAv}B_uKm+&+KsT z;{&;G`hb_#vG1$Asl0QH3|t8~%B8JpZVsoEyh7BVUQjK>G(Z)cgG8Tv`f#j1cUpAm z)ZKP`u;8;Cd(O3d(G+YxS^E^nvkqC->^^~GBKC8}(MHG8kDvC~tI7MJJJ3F(D`Hf8 z9c1bZge0VHABp?V!Df^rUanYTx}pAu^ZQ*Ew{*^WgeQzL^l5MY%#hT$^1_qy*Kn&Q zB{p1`-RTXuTHkBMkM{7XU z(*=>ic1kRI*5t=WIeJg%Rs28xps**a}Sh@bhU$v2o?+hf?5%EnXwgmJN z|8_S2aC5k>S|_l2tvNDdZQi5BXd+eb1a!D58cpIOh4 zDK^@|r%tV;dO?iJN+mt-GxLbiul#X2 zvw8VOn)QpLVjjYhLK9CU&k{RlGSc#POrR8EZts15-f`J_%Ytm zk(0AvEH%~!j2q!X7wyTZJ2&Z701+ZHKGN(>|c|sOqc{+;_O~L<&VnZg1pr zk==&!tVbtt)TPMVs5X6cu)g_A&FqT1qxhLkmz9EnCd*qwVNJWihF*j0veJ(nFY6;R zsaQ9a2ie-sd7~30qq-L8qawz#G9=_?Yn4q(u!vKD^VI>k-e~;&TveZwTZI;V?w#Ok zwd(p1Hi(f7g0#i_K^N|#QC1^HiCF5(TIMF_b@K0?dtFQPJd29{gnr+|R{ZM3b+a8E zh~_-xQMGx@qA>G`GXknVh20}pY^mT9;*heTWmx+w@tz1npROy1nm~=0oZl{#?uYp6 zj5XmGi|4ySSQDR1eu+_4;v_e^*?cjzQ`WOoH_yRF5NRbFMQ_OV2QoVv$`R*FJ|CcJ^Ayt&5PIDS+3>ncb|RP zn^gY9Rt)uzFQ|hTEKqzmo?j#(gdA5yQJqT zcEnbqvg>>|)6Wv2N!{={Us!VM(mhEFR>qZyN_4kW3ng+-8>34v(X#IAMj;xT#O=ym z&7Wef($e*H3+U3g^754s!jYRZH9opY?rYD)>X*SfpQB12X zJ5|&Exbvx*dlC1zg4VZ}bgARDJ`h14G=(>#SP>Civ-So+wY~QNGfFp|GK9|?zf1R1W$lPA;Z1h@k*uU zFVif}40M)h63q;M-Wm()vqJMfBPz0w zJ+yM!qCvr?OI5p3SJ!-!pPF08$;zYd^FHp8(Mm)k#@}yuotmgn=Awn}ABaZ6hsrxf2iANbdCj~pQh3~Hv%>9Xh z1VwfG`pY<9mo=JC%AAUGe;jYTHMWjyDGrM%%PxEW0iSKhK?m)*@ zI9hKP?js3ekt;ZVh4|GF-tK&{@b0xa;+4xcSO@vgLmhO(8+N4EyEVgTw*-a@ZI63V zlT|{@*Vz}FR_LB8Sc2Ov!f~HUm{E{Cf06BnXszXl#I-c>wtG*8xYyC(`&%O|z8983 zm8fW$9q|2?LQT;`RbkF9_~Gg72>*UVgR zgje0b_P8?$=k28uwPKbIy)~aJCaI7h&T|#_0x@dz`;&d6Js~#|-Gr%b@5;|UQChjw z!RcoyC~Xz28yPxzE9OcC`LnZ}GC3Vr0L(a5Cwvwlc^d3*b^5s|=DF2sh==5IyZ+7T z3QM4R^lP<7Tg%U}W~?2_lsSqyS|;;+74zg`z5 z-emYrWi8y7*}9;rH$A6pqz`=?e3Q>>gi_oko?(DN8m#f|)u0&YY#;-SN%+RPg=@+4 zL!h_@W`>~M;V*`Pt*;ude+Fq>?**DnS3@&#_?YV_BzBW4%eI1xxV`o$EGJ9+{=9JM zxEsvDNL4EirY&*n=5<|bf7_?6`LaAIuTZOGCYN8H7-_Pbrg8jLrxo32=f$~Qd+99C z{Hc5n`ThEUfFF27i_Qd4ZI$+MR=bg@!o+IXoR1NmV&%WoIxVy83$&#*g*<~@YWlN1vLG&UU?ujaMoCUt z>G#yZgH*phcMu0}y6vb}8?>i?(YRYcLr6lKptwC&u$@F*G!BnGDoAgr#%yzVYhqEe zJ0ZwBsmj58aa;3hJsz3i`pe;qdV4xD#pWjj)p<`0I5 zeP^ZjSDpQH^;(Vo7^ZBee3?lIfn`>68B(7xBmCJrdu~fL2E^mO*%+zH02zKN>^>?~ z7dPF$O$nVsn}r7cSdCfRuqISZJQ~#QOsTFf0)4A|2NSI9PP0i{Q#7|l1r4X`16JVF zEj`zG1uk#I1uD*v++4b(gM2duE91!?nVMOvuOd;87*(q@g1)ia)KeI=M@v?I#>`v^ ze;KClP$|V|Q><(uX$#E>k(8U$Q!8EqAC5~(6kr98(%(}_^~aCA6~ovt?SjHb2N=Ym*)L7YyL$$M4QsGXx<3GnvU zH^SRExhe{kW7&%*;~15-7}8aPGPC;%U$SVhQHOHc5Hl;;Bz`@6Su`5!hb+cmXUiHL z(@#f{tUXl{s@2CxM4Yhb@nL+G61j#D;ON3&Cs}4a!!cg0JGb!lXUTB@`apg_D#uPh zDMK>2v~_`6BIzToql0jVz1CZ69Tp>1{^Bzx3PEvn*|}&dle=^N3_*Wc>gX$m3<}hK zDid{i+1U_MM|a{~xBo!cFrvJ*0I$mfZTs%IB_sRp4fCnoH}ZN&tumGGMZQ=@vz3az z3R`P?>p=q7O9YcPGrBnXqfucVz6rM+@QR9zy~1bcN`13_VI?8cZcf%)0|%3h5e9k< zJ=-rQ^yPc{Q7r+pqO16W+4$rya;?%(_c~(Zb_5*EBLw*5M4lF`4KAS%r%K!Hyw5FD zM?Z0nCUgXxNBJWRyaS-#z%P(KE0OwrHY!zp_iMhur6D=oFgcsWDAhm`A1*9!L*8qh zhjLTONkn+X-e$~deT`s}98Mj;c+S*bXAhtc8Z06*U15Q>F%b8K@1Umrtw7mB@XMYA zRCBY;Q~`zzEx-H^t~ZPZ2$OHSJPetZgS0ZmBnuKKz`&zCnA!UKRZZNLIHFer*12|!Wl85w^zH@}exxc_WpA$SXGKqyrU zp1S0y$sDYj%a35~1!7&IBRh|}tRukd6G06Jc~~nR;C>D&Hm8kurf5Asp1lAyaD@l9 zQut{S8vX$M!34=&_sY6Io;?D=f?)b}GGH!8X$X69Ekf7ML-{DM`lIaZMS>q#!%BWb zS@94hum<=$YkkSS8IfR(rCm`(i}ST%Z!qZ z?RWWy)IU@|aN#3(gmDZ*@Yp}kRc|~=l`aO?{&j*}I`S|O{}lgaI{3~=^I*{RY{u+; zjyy5{QOo0_zkmC@9yfeO*b{@Cv+N;63_kR<0L)ARbahS7MJvAg`)le|pbUb$aM6IG zJqIiV0HAcmZzVO!9Lz^zz>3M-yF{;GosT^4&`s&i7EA$&nEj&z&;{yVIFH+Jhy|H#}0w#dOq=#o6BV#i=zj#F-5rEF+6jVb9IO+6? zZn7&&_g@y{B=g71gLeX%g1E*euB8M7>6Da|7U}LTX*qNwjdX*AbVzqecXvriiL`Ws zbV=7;$M66Cd&eE)j3L4~`|Ox|?zz_c%y+FP%sp3eTw}NcNwS_rsR2xq3&YaEgT){n z&CsNC3?FF0p`eqN8AIqGY?(@8ZMV0#@6LrdfmkK}xmcPiQiWz_x;_1;P#R$vouR;)$ig6wlF!LniX&j1TAQfpmyo#B3XoJMDvQ5TYq0B*cGrP7AIUyNpPCk5;Q6*A{ z4Fp%xPneZoH`F;DV?F9*o8>rNzU<&cDR z4ia3_AIMPoJ`igkgpC`&zeG5mVV55aNZw}0@Y>@;B-QDB&B1P_4KO>O8a?cADm1c0 z=D`DT)gGj})rJLIF`zC1Q5kLEunz5WK!BQSe!~P+D)hiV;fox z)kCB(t44`W0PN*C^?S*^iULC2uYzZ!T-V}K@ba_;%$IGuJ0Gfj$aU>+f*7S*P=Svy zKrM#q)HtxscV!623TeL<`XL6JMr0_&*j91=Rag|DoA+_iP=HaXut#G zM^?+JH`WUcFTwh6J%PyBH6=eC@hA)!Ak(6I$+ak$X$IXLg!0bQW|cz6Wn=d;6HL1w+SB4Y?Ozp%I! zuw2$PL2h+shY8b^!sc|n!>QQLB!fT?2oV;mho0{Qx?~r)>k%_6Y$;y9ew~2)FW;et z&8U4?yhZ3M)mE}g49K$|#X?u&O%s-vM{Tg4`&yxWlRXczZ_5TyptLaGzC(mogS?J@ zZn|1!N~-|LnVf?)p$fKW2O7IT`ApWlPNd&Dhi21a!MmW+1^IkWRDf?GX8GLG6)0t8 zzRQlIRmwFV%LGq1c!onR9(y=7fFL3GMQ_hQ33-^A(F#J{`B9Zc2pLVbwoq8vRFv3n$ zM~&;xYTPgUweNVo9%Vhq^tFE(640u%d>^&U(N-zS%A~cf(lKesNj2#JqSXbCgVkYO z9H_riWE(742EYBlYVFXXv9q%6?8W-}x(>8xL`OlHBzzu>N@%1Ar)xd!`*f#|uWb-@ zBql2Q79^B=kh-q7)9jp*L-QE1S&l$1CYk167!B^q5&J?OezZ4R8BL>*nypr>GN)sh zISDSaWV^@3i1B+grs_RUL0;ZhSo?alW(w_1Twi)jIry;br#w5VNNh}0-O0&MxlP-L zay{}b-J9myS?IEZXLCe61If@GkNvj|ou2NzIhQ@Q6maNvGrp8A?O2TH6P{XNyuA4WNo0)H=WVSj&ogLpps{<)@Vli9Pr zxi7)>WDzZ=_32AYceQW*p9~v<${VA1D~5=FHK?Ne^~$dCe%`IY|Jm;;m?Dw1ux-av z;CeU}t{P^ZPuFP{pAgz-iNmu9togPk_^WX3Uj}nZcs*&@WsaAJEX_4?v z%5y!G7Cc!pA8l8!0lvf^`AiIf-=?2O53A}exLc}DHpiWyHANL1cu?~Rn_z=P7ex)t z;VOEA&vBvCrQnq*1!}MJ?Q}{S7&*)sc1@5k8y=kj8zI-7Qw@E~sv$I=8)E-l`|6hf z4In_P?CDN{E4y>zq^V&frVVAMy0D?l(R!KV2$ca0v`FN%T?_%Dx^|ak>VJrfRcTQ9 zMtvJ@0B=7NUj^P5hTDGpu^STqzGNBtCClf-oc4eCIOtyu!0o{yeF7a^ z<%5E!EB|_q2U^^G_zW^0bwQ})S3iJ{Z9lup)Q^gSmgNT$V61gXG3Ww*2HzDe^a!5W zaNxP3^w$j&`bjrDa8AZj?V=LlcG_k~Ebx7%sNq z#`JhsJDDgzic1zLbj4g-_TbK%o&t9!8aWHL7vpD+oo#<{a5Fafrze+X&Fx1hziwJq zxJ%YTc<1F!u43~%k0lMV%x^L$jA~>Do{}srYRuuRvIkhZayNXiNJkVtiwdg$ zT4A-@*o0TTaY8uoXo&{v13UxQq^;uL^)hgQ-HH0LHo!Gwu+6TY&TBu;ysf9u7VLgp z)}uLA(bX@6{yTp(B*p@s^hJZ5wwZO{_vwgQqfcqeS{H((ch2;-|L#To#Y3|kn6$KD`~_eyzhMf{Z`1>M^v=Un3$>g-2_Ymoes&f>7_K{Wg`L_r&r+= zL2c=rQ)CUJVy&q*@8=7hqVBA9iZO#5NF&hd7j~{wBj`04m0>dWoksq|>qaO1We#;X zl-#Jj(EnYpTgy(90_XAfkAJQCm=VaR1bi-KTJq=Id=MG6qX*`V+4z|l1H+r+hY zl}hJ0?JTZ=7~vbqjT`vYu7|%k?vp<0(Vpn^()U%HLspSHD) z!?xqLgEP$`|6k{m@0i3bZb&t5=1yq~nD%D)X!_g{k_kjP#^o*{5$zF?wD*|UWZ{a= zyxAVRC3Nl5{M&n!Ru0_PNV1+u-~DJFl$%W?(xcn_e3g!c#pS7NKRU3@kS!D$bA90? zix5RX+1;^rPsc1tqrf>Tk-T5Tpci>oOFxU=x9QM(zO!-_FV$Ar%lEFiy7zH&#}#QdGM&tvU@cpK zeNt~c!LHY=Yp&^j!+>5c8CX;5bK_y5n#Xb0Ghs1#$@-2Ht90tOl0I*E%a$cw#{;<$ z4;>w{RkZD|#keVi`k@)&jGB7fto{(0kf0U+1leT0z{Vu5-eKORkG?o8Dmq+Dh8P?} z8k(qFlngWtF3Ar!v{_!C=~N5;9KfL9F}`hWwzbPz6x2B%u;qUml@39ak!|(4cweA` z+(>wLAI-U5ufID!HX7L9+uJoTK+Bb6qRnM+fvm&rjZ8g^*Qe0FPqCVXjl}1RU~@Yo zF)E2`f9eF^P{|u?MBP3vv&4L*94*<5;U+VSF5pUdp{*%W_&QQ8`NP>9CEw4r-bVu} z8Xbu%OoeLCB^jM&oNx^R%RERsQ5KJ+Hle_gjw2RgL#9aR#888vbDe;S()?oUDf^|J zV5CugoAVuRNy11`sM>(yQD@O%WVC|#@KODElQ)Mryp-{*^8zobwI99n(5m09iu0}X zn+h7ezL?g90!zxn@I08EYO4o2(@BoHLsi6=3FaEIPA0^UkM8IiEWd+(2BaFD*Yvzm zE_efRDi2~SISg1BqoAii>mju$6bz4hh&yEsJ<+^SEdBL4Re?S?(y~&m*{G#GVq0e2 zMBAx6bY6jBm)Go8BDyrij1o@!YkZ@_&fHRR>y7!yJUh|a_?m@sIfCGaQ`T+kJ3_V@ z-~2#j1@V3fqanL73qz_=DjkJhM>##^+$$u~Kc84EOL$eE+|ar;MPHMS#2NZ0H*VqA z|FL*pQ8KGven-|o5A(+8>L&lGJJa!Qyj4W$n&h4+tMW-DbbG;Gr+hof>Z(XW7*1sjr*#ZDCyY{Z606b>(xE^MzxHcbkBwCr+$ZD zQesCz>3#hQ{+FqwAQ=*xqNZuSs;x~IVsyUZG4D3B=R@=`KE6qaQz-8!{d(S@uhCir6Wzr-&v*MbxE~%L(55dH{&40ZFSpylL>)J0myY>t*^$`s zxr_x2UMS|X3U+a zpxxOT0;{_);}5)i`+UUvJ}c2#Lcg;g?OcSvz+9XR#`%gy$%IDBiTAytR&Efb(`j{4 z5L}EQs3cX7o&6F|>#~-VX0LETE%B(-#&gOgK=yw*AU~vkP^Me# zB3tWjnlj-x8du5rfX*>IExr=m{=Uv0bFmzLuiNp9y{z}^-(@uzI2t-(^wA~h*h((3 zq7rp~R_xsT;blfss1~a<$Uo%Sjc^i0G!Y(2S9;^j9)=&rR{d9sSsPWdR27#Bp<)4!GsElI1K-zV z9V(jv>QPMU18AiPlxN*rR~w3Dx9(iuSqh>8M$5=BVJJ8s)h=Z`Th;Zk!pNgldbc@H zF1xP=o`sk*+02!$U-~@QWBYix3Pc^sKH9%TX~`X+vl(*SH6AQG{9Sb|JVX&6Fcfgv zjD?}ni$X0Fv7{i!wK1E)>nsZ9KgPjeF@lh-pBYvIkEH%c9dY0?v&W4(xtH>{r<$Wp z*P8E`;de)Y_|>w&ibnSSWhUAfC#Jb^2iCV(Yp?gDMr+;p?jk<=D-5HPzhi{xFMJbw z>C{>u>LYm5?r$+s{e1IxQDKYGDOXy;0Q=hwk&-+hNd9R!^3G#1sZMYm{~n@3I?^)~ z&aeggctJnm3oZ6(D-%bx3O&51H)|PMu}|7U@=rvKf>W3m5Ox0~S3@QObX&eJpVVay z{&*L+moT+Vt2op|zO!Q9y4Iq03etwb6(v7x*j;-6y54(1(mi%!$9XXs!`fqs2>V1APTx;ia!xv9uW!t-5+ia3- z1@4U2ujDMG3*g(PY+UISN~xZt5_YKDTmq472UsJ%hsU11(zd#fy-9dD#3c*_b2dLtb>pdt}xmqg(FH*~)f3}*xJ z)YA&fu32_74n{*Y^_1xPIkf6M+l!shu@cGkw(7QiqG2agkp@$C4>(+}8wU0TZpd2i zq0f9rP(du*2{N7c=}T+h5Pvp5^)WKx%cBgBZiBcM(h5xE?YclGdbMJ;UAZWO2iHO9 zW29%E->s`-X}6orpmvdVGc$y4>?WoF&goiGmU2%f;cJ33!Iwi;27#mzm^{hI?_g~b z=j~_A-)o)@1`*p9vEsFZ3en*6w02%4-z^Kh(a)!XTowgAEjJQyyh;S2g9E~`_KKka{@7~2$AeYf zjMrJNe;fZM&@&PYU+4{g{<^Op*8}8)qUl!4p$!6>;vgoHDIun8(tu$B)U~nGx2~($c~Y zw-=|t+R+fS3lZ?Uy)`m6RxYtz&KDJ-ZY!B$tn5qtOD8JN9~8Nusl=M&R-Aj=?1t6u z-RAVBn=kxGyjv9EZ_=CQmn_tfrAJO%JGoYsuu|;@$yn_L6_?V5eF_XVdKfFkKaUKv z^m$6d7y4ycWj(N)U8-!syBnTUo~pgZFc(^c-aI#s>RHm~a(L1*AAes;CNJ071zY*Q z88B`8DA&E;csxnk=UHe7K#Iy(MLph*8zb`3^#;fWOI_RCl#y>qc*hE(feTg&PtlEcvsZN??L>+E{t_+ICx<JyW@{8vKdpRb5$> zdMEtq2JTf}h?1e2+99KIqrB2Vdv?1S#RrMxj{6=0+}fQy-`;)dnVH7s&(DE|1gzzf0;e=`GHHgw{FN$H&$Vu2zK-(~>e=gd zQ+2*`+ez+ymm13gtH8k3A2rE$apr6Y}Orr$g|;|kA<7SRmC$ywi4J)Z<1XL z`D+^eb=E}=pD8kXc<@vzn1mu^<;+u)rNu)wh*p}%el5l)KE3jZKH9O-Bjdu#^9gIr zar6a8U2>xRi#%z)1(aBPR^ug2-oQM$CB>oLxWMCvpX!;%&IG@jSOm4sc{2B(>d?k8 z`1G$bDtP3Of|Wm0e14}BVwTRe4=T*2r^mbTGA*{l>S74TO>~adVSS5k&hEF z+nH_!x$CWh&Ip5MKTFm!e8~uce;ypb7?VWQ=w0Wq-Bdr-j8L-1nl3asd{nYs7|v*@+sefX*TQYqB4@1~<(>T; z$S%#}Az&ie|0W0HENHahN77`b?5gZ_fK4n*-TEn7D=D){I*Pg>flA4&oB-)YTMC&C z@sJtkWUWn{grtg5#F{>shw3Fru%tR5;<76p`oANrJ-`N zMxIpZUEy`F-8{viz+pPt4oFdP-^_Kw);AR~lzuE{vFGzq|-(PQQODl|j6CT5oxP zqp9yN6^c|cm9*uD$)09zLO^EwCoj@Ls&V6d#~fo zvsZW|Y@*TfMyYKomcS}cIuZ6M|BJ&X{TgA;??}BBd97nb_Hm__v?UnryMARqlZ+x{ zI+CY9svt0^M>H#MdfzV$3@$rk4q&~)bXa0z*jVXUFN!MVFq&}oEMXz($kPkK*MxV= zpXWMDl#~8!-&)o-?vE(uq^Z7_IGMwJMgf`!CUC=cH%=V}iLoo~gn#J=Z5>?iG(|Bb zy)f$G<#@!)*vnN6m%U9L*&dvc)E$TKoOn`y=daAZ_T0bo@~#*`%6AWLJ;U=oQ^T#; zQvZspS*PITUQOfMjwJ%G=2N1F+tzYN3@qHwYkOerTgv6GF<`sX&~-?UHzQ?E2A85FP9^E>r= z=6=;S5fGM5DBw<(-kg7*&<+wNF!s5rkGjxurql69;l{cNa^Hw)uGZl9onGb% zm-_kEa7O&4_bYm$+5IZ)W?kP|xd$hU$Ur94_D-hdR^Ob8s|DJ}rZN@#L8IDY{iU!u zmZ!)R)ujE)F;bM-@Np-Ji~U4IzCV4@6fV@b8vB{f)$2RuRXZqe38J5dm3Rh$+SSuYZuR>*u@YGq? zv1qC^n96hjbm^^^Ec7@yim^to|H7=jNjCq1zH(!IOR=wq-bF`NHKxsai(8m=-*8>+)^jV*Q??TsF% zi>)qmEAw2DK96gI>jjUkR$`P6tXyLm>Z1$NdJnyI8Jl6g9=%%c;;h1Ug(pC>{cKor ziqsJzztOfTZd$ORzbf0T;23JppjpdQbp5k%o*=8ChSK$-`t4Tn?Mg2%b=OT;t9zwH zio_!t~N1slJlib&=_-?xrCie;J&yp9U&qyt%ViY){B5t=wcSH(|G{X84{CEU308;!765EMQ}$ zF=;VU(P;A@1RPlhxLstj7Vqep6eQYW+*O$ zQK2f=l`KAd&fs%Kd6+<7I%D{|mte;!(+QF0T4K%evy6Qm@_NR|N94#CVmtFBs^X($<3^mIjCkeA(S6l`fNh15`mCDNPakZ(wb-S!3GvINkD`#pfP^7((#bgBL?T%jVzALPN1DRZRnJDz_|o^zB#v`^pm8R*xTRij&Et(<$mEV$WN((f zT~9@Q;a@~=aq0NW_1}6|5BpC=ID#Z<%mXNv#}|2Ydew*pSF_8MgF*C!FdYwLmRqk& zGHPlQ-4aa(1fDZM|FN z_jpYi!Kp%T0b-%H(`db9m3mvkEs_`ABrKP`;~6`w$4f6;9&=e5*TS%nJ}o%q$$p_m zdB5DekG^=Np+9A#C$j?kaHb`cVY>P=$z#{*de{TrTA=VOzgMG9Pj%Ue4y!a(wL3M@ z&%sLI&;eQbt$1mA?ID+q?!qiZ6&s3iEN3JJ{G3I%OD)0zr*+nFGJ{6vY2R=NQDZsf z_u1dly_argV+?Ao<9cS!BR*|@VVbg;-nH6$1($;*`!c%ydA45t#eC@k)^oa++rgFb z1C<<*5(LPpCCoUQ^YER?AG)@;29RJQMq1AKyd}c@PDx|xY(#dKu>~Kg&(UiOOpYg) zzcih#tu=J)GTn*RDU6jjZ$GD$?akk%+id#*XCybZZ9nrYZ@e7V!trzegIep~X`XuX z*lQ^>dGSduHLd-a552_;Ka}Y;-X0po9azVnpB2+&=6|*1%31U(+WB_aHsyZzQ|RQ( zSFiWm0`1P$rzAbfV|T6e>3#oA32A1IZI7L&3f_Vql&Axq@XQ|V^_juLgkN*Cwvu{ zZ|^)`an-T$uLY}C~Xdb(XrQfWx7FSMz!mHRaPX#VT`XbX!g(b zFaL?{R54tz5e}n7TzV3)wsJrpMkVIPz-7?r1*j(-WXYv~=j}gb9g0Zrj}kJ`zhHO)VE4Q6N1o=lLQpIEfa?FE< zUlf*9!~j?(C38FE&d&-ULr40H&9zbyX#xM^mBI`lHi+wy1rHQ`1nd#J1qQulMHw67 z`?_sHYL@@J%bo%p4()d{O)7)uPzj%ENj-sHeZ&CI=HICWP1n{y&aB?vAp8!4pIL1=sf_vfB*MZc*DP35pz#aqmzyRkA7+}`08QIl5iC<->>b1nK;w5vIb7=nGJ*&$COB+a7QClLb6EVd3bo&Uo>` zaRU;*X9UI_VQ^nZ(ctd#kbu<)9#U@!RlA!PNnaWGzyeCA4`9MKKCDk_TSKBgF9&g% zHE3Y=F+NlozyNyk#|X?!$~#CZg8${=!Un(?d%$?a{(O|nTsf{#`WfKnR$O6%O7DI$ zNo{vSxohAm#cI&?EaNB)7#CfSu1bm}U7Z7fU-EQM(s3^&2Y63>E^(j#X3sIE156{T zP@psznDuCsO9PH*Bcg!(VPrE@ z@0Qf@s@#T{kV4!Nu6O|fCU4$>Xc#in`FP8)Vp9#{2;!6D>i!om){jIGTo~i8aB$%D z_4P>>QV%7{gkf!k!S21J8AlTX^c|cy3Wvy=WzJx3G1Ps64e=ZG_t{KI@Qz6{{tGuc zi0b)Cpe7);-wqS!X=BBF&;SViaPd|);OEyllDZbZ-6pddJIOQjz%81HI&!1Fm?i#f z#DVn|8)`d}1(j^Mh1ljKwWYgo;os=#iYz&Mhgn31g~3Xc?r(Rg3#c}=gz?`|`}1SP zXd460#oYv?C21P+b@j8hPu+>0os@vl^7N@CA_=krHVT>W@}hnzaPsdk*?XqoZ?BHC zDD=Jn{K-=Lqs|dBxP3BHT6IO%_H5aPPMfIxnY0`JjKEOnQEw&mHb5OpOyt2CPvf$C z-e=VxQRWt(I~`VQ=9@;Rb2t{y^c5I50ivcm05R{=*3gGWTM}O9J(Jl>;OEpt2?)Kb zRP`{8tJJakh~V!WLVw+ukhzn=DdQ%1o91c-8|;?AhxfM);#xunDd%&Is!IPpjj1+A zgZ$gS3t(KJ)IcMi3pB^ZIHl*aR&Ilwdkc?Sj=DR3oBkgVhdde@djn*?|8RM8j335q zZ4vO&!kF1^_1g=6xznlV+ZLsO%1}r?i~}xN-$sea^)!K1pZ`5}w%e$Xn0W(vE(b4f zsyieDd5OYXxd~#DYf*6bkHyI{j`~#(y#m8Rh3!PS&bt%&JnexO{7{26LdivYhYkXFIowy8=CGb(~eSj3hdw>-LG4RFS(FiS?5UmXYg>33g^w+7mUCiJSo zX$4*xA(}TeKM=H|jMI}6=^*E#qvsVl_7`nDJ0S|o>=)I znuBp8>k!zQ;;`Pjcl=%T=LSK`bUNj43y4-WIyWw9fYK38XJ;te-Ws)O`xkN?9yel{VYid z?j^ZDBQkhX=zuLP9po^0f|4lL{^G~So$L%5VWSU)3Te;XO}3wze>K;qitz-{akF`1 z8p9iiqgc2z8?d2D#l$701dmR@9{=b1!kj}7WQ16`jyJfpuwHg;?<3pKQ{EtG>>;r6 zy#|jv5F#v)Fpayjzne2)pWQEcT!4#In|rOID|HH-M%jmM06pi7NXm4bf*?5T`5&U zO`2lqfPoi#3USyzl6=a$pcFWrFIfU{em%$p>(1a=3F4QOOTy|h;sd1}l-uu*)Qknz zMWBgmkTz>IFIo^5q$YzkY%F^|aKd=fROT?>8$k$3W7zb{62Px2nm799sbWOQK_dZZ z`v4)?#04YPX1mgnkvWT601>y+(IJ+wdT#CZLlA8LuqU?MjR0VICsoKhVEOc55WZR= zGN+bs5(4=a3Zns)#9{!%W7U}}A;L}uwCYdd_w|$w^ttQdlsYU2URwHDhYUbPan!Ix5luR(LbX5h0vt9R64GwE&G9|rWG`ylc4 z@OO}~sOV?ic?WSy@T3E)ec?+vb#nqFk5Q7G;nd@7Qzkr;=4x97WDnqm}Sm5xw&;A!~bz`hBa}C<%3;!_CD!Gj7?51lCXMud~g^O zxZ13U^_nr;_Va?>+@e~|frm6$hk_Nl;9y{2xP9)q0S>*+xdsb0H&smM&YAR6FZ-Ze z2p=9AdipD6yGj@8#-+YFXO{?1qnMt&z7~J2LPnOwxW90|2`FPw-(A8FHAr{CA%*y* zaaiF2)b=xIA(8jgrlg=Cca2%3)-eF2x(5w4qi8>!_6qO0T`gi;BzG?{2JrnsBd-p# zLh?H8!^gmM|MTm02>uMvGCr7MCiYIruKYs?W3<6Ul<@a(JQlqOn}voP(219bj(`1k zQVH5g30+Fk0^A<9u~mY|zav4eUV6V>p2Mu$B5XvbB~9K4U!mI`42yuWdHHUJ7G@Ir zn3R&MGun0>_3iA;j^M^gsdX44 z#H@&GiI7^lhA8evXP77$jNN!!Xw|#VB*@!eZl^T9W`2h!A?uOW^(qzgq)?K(3V`;i za9T-psexRCUwsUu`J$iwtY!~pk+sjbVUsFpx`oD37IG^MxBE#5SUqq$uGi(FH&)T$ zm=DkbRtcaDDrMx5Lk^7wF<3K7^31AlKje_y*3qW8JMdRq$Z8d0O%Lg`ik>~klkFA0 z(0%HL89n`xctP+hA6w(ruKFOEf7+h;=h;-sPSZcyRs3H=P&&T{eE401JYG3XO=Lk6 zzJJ)V-gUXUx;SD+-@=%@b7i-Cmjy_U248u^p{4v=K7UPpZ(Opac|>lPUk3#ff?Tgp zv`SyOZS^SGYnl2QR71N?_eXG<{<(HFQjs$pGpN0Z?!b>~+Pda` z7lppn-U_y+-Ld`5bjc%vIGqQ1ibV^jFq(v|(Bc2zWFeMxh@++JK?J-C5{`l30S+FX z8)ymfm&px!?&;Vs?%QGiX9D=zZ;namLDOfs}80)~Hd!MD-8mr{9CDmGV6wmmep8laK~M z`oh;U()PabWhKGMjRmhqPv#&+EA_H0rFON(ID5;wuVvwzwe|mx$>NU3<|A#x>W3c= zIP}T)Umo`AON&%1I9q0qQ}-LrJVeyI*tc!(2&F`$a+)+*r;6Vr!o9la3lsC*SXPXy zIzO-_ZnA$fyv=~a5)Bi1XQuUq76jOcNValuZc{K7>g64Nez@D(RY$pUnY{>=j@jo!lqPDU3EP}_U^wY5q~ zvIDPFbOu9wuh^*NM&hfj+Hw}Xxe8oah8%agg%+N2^5wO)>EVqY-2q3BP`drCgwDL@ zwe)|U4%-n8bTm^q?@O-qmmHMGn0R z23F=xaXdS1Z>TCU-;sKQwU(~Nar=n$(|J0dRL83^X-cgI>btfEvm=44sh!0X_ao2c zD*+po`Dv?={TtY83h2KQus$@LX#|)o_i`Z(9gFzieiFG7AeN%b>1sa2W6V<42Yo8L zZMx4&>!iu??=Crpg9u#gvc$^4=Kp;IUBc=zi7du?sI z)CaY7lanjGg=XEk=j*Asgr95^k|!q7V8nC^m*RCGB)q7iDZ~lv)Bz>O)@m1jQ7>iO zP6#81IBwE5Ee@+$7qS~%EZn4Hr#ZAr^Q%i`RPt2&8I74=!K1=X;`?ig$i7(c+zdaT z*b0~%#3CUkXPK>gVZ)dwA}+*Zo`{P2a`0GoKOa-BUz4WqhxPC8kIgMIu}iJyw6c;- z5|%!yDzuT_0W}nYDi$w<>Gt>Hq~13mnATD7V!Q=IaD4I z{mR)JMgB$61(BatkB|wih$@l59`9XiYrfdKUzJmD_YJ+&p0m(?S!mhX9ik~dCc&`G zJ|2|)8r%WnPUO-8j09!xyx=D8(#6gcRS_>;uHpXxyQ#5@l&a%dbd}!m5&IH9~Q@HWnPQ8S%fBas&mPa^jGd=I~%Lk<Zck3~-tytaY;qeyZun$o~t1c=r@r5cK=U2x=gTbnlnt5V*o0(g# z(dnMN+|L$fccN{TlPX46xJA9@m=~V5_&O7lqC4pq!dA}I7QYcfFX=NFL?jgsYrJq? z51iN2x{% z81<7Dsy<>5|D}45Ge5R`+pjF~jT9n)7|4#b1y*}|n0Pkfw!%kCMR?f%9C4^*$3=!t zIwO(gh$YsJv^b`cF`4O!n^yg3-bGvwJ~sdF=l9De^OzVLymhfuj=A4A^)D0tCV03YV&x1l$k-kqL+&B*#*+^DO>(l+1&KXug!H|Wuga$ zBdv9i(&rOzEB$JnNU!!N!{(PXFIIR}=+g9sNJ^5sdcc^~IPeWwA;#V{y@GE(6?K;}&<2c(D zD5QGXyCR54b~y501VsIyorhziAOBpQ5`bIC%L@&gjR_ICE3*1DK@IoxgsiC z3>{EkMmb%NeaVc|Z#%ED&U9_}diE`@_fn|wQMickf=25!<x+n9|PJ*NGW4I)9CXvzyOOT1c(xOr8+l{DM5~_3XZUoir?leO=B5Ui9H7 zFXzmOb?;a6x{D;Zr24OH}$)wxKOo zqt?NVap{|x#_Zc-R;-hSRe0d!-PcJFOQGWT@~#bL{$w;cuX7f19m%2iD7uL9^4#<7 ztml5}mQ+`As@fg#2yE=bd&QbSk#j{R-e`9_f@XG%w*|0C{rrlp%DrC(204CN3q~P5 zbNu>gy86gO%dz?1*iVI)cDtglsnY&w?N4fxo-T;u&Ui+eselFAwBDOcF& z;o*MkQljxkrHdeM37Vr3^pcXnPgG4k?AIfdh(#NH>>ymrTMUxhI>HL9Qy|asAml}Z zsJ!6Fg1}bCSD5OG##fb(6gCX^M^H)VEZG=1ZsbxgnOb&v6faF)*!Uwq+rmg(Q*!!S zV$8g6S6(xcky&o-m&j^3HQ@y-z3L`?-hP1+upJ5QVAg|5)9oA1|M* zz<;j@qDoFx31itX;WHlfub6Gq_dtqxl4<_iZ!PH*i|IC-KTVZ0ExP_8=AG1~+963? z)4P>&6gw5)05P;V?P?{T>v^RuO+M0C{l#-*-3*UH>^Xc+A4!rEzb;WNiVMG2 z4zJt2P8!M7m60_@^{BJj_ptNMlCcB)L<~GA?44yJruRAXMDq}w1XPUNkh+N(LG#q4r%g9Oul}4`J*QB;;r(vKFYhP31J1`s8lO7VhQLnT4Xj%xIFvmu zSf%2Vm=IeIiZax)^@mHc9)>?U=*-DdZr z`tm)KKw?Hx)`F(xd*Q07hOGAcLPUi-)%iY42aft$M3W3!7(w=GS`YI}^izuM0mto&)U{1W!L|8CGTF_UsvN0-zR z(hj{98YNARLDwqOC(gtP7`Vh%7fJRyLjA;|3PD%qi~2&`+<# zhnO?z>IA<0Q2(G($lV>|oZ)58N5Wh)vY!>Sv@9MCz*xFP02?KS$pQWp$*tuAO#W49D_o~o{7PJ5z%m>U!b+Z=M)-L0JA z$)j6jS@n_|U?l4y{pBY9-g){BKT0DYgvJ2h<7pB`4h3#Y_0NZI`G|@-9>uA%zD{kj zb35m}J=WBeGkduo7UXn5C24(R{R{eTu2exrH`;J6Tcm6 zOGYZ*ukRQmq^97Qy0e2nCrTkiB2e{DVO5$eNqRtAN!pPX*A~&BLp8@GBW199rzr~& zeZ>jqrhcS#spnkYR4vY<>wVR}qP(-LmpOl4RyL+byUUu$27i9YxWZKWNy9+w^J=V7 z$KUF)aav!#`_Epx^DIT5Ze$#(XsM+>!!h19#tx16gVNM~9K#^qZ(!)vw^50+3irdR z7Xg2#63)l=u5~k_W#uk2x0jd@K zOa^^FSAH8MLig;KE{#)Dwn4?+v@utihXd2WNOx<*m%mbZO2$xErlY*b7TsYl29U($ zrzz91xArvAhbvdbCBM8?I=f@o9uhjFi`8z#9m4%}P_yg{iz$d?UN-i*YvSWU1M&V{ zb-(%!Z6g^%@WmNo{7aN7d1AdFN;Ls#60t7={`ShzHjrEj)T+_sY~jiWUxcm42jSwq zSIFn}#SUk+>h619xz|sWxmdr(!}*dPri18qt_VlU8y&t*-SQ&p`)&M|TH&c6mCiMo z?@sQNG!umNIawDx|G{%Uz_uY-<@4Zi^Xt|(CLLy9#+4U+wKIuFmS%p> z;<8}~!?BiA?X2x_`67$Z;w76_QtNXx*vGRu2BG4!8JDTy9zGvR2mAFG{-Irl*39-R zVV)*pN4%BJ?MRV_%OwvJ9Dd&4H_UqG=#_@F>AzJSHF-VsKPlPp3v18JD%!@aLl|^4 z^!A8DgU+Wr_5TlZUl~?Mvu+tAxV!5n1ef6MZo%E%o!}54XpoI;+}+(RXmEFTclX)f znS1ZdJv0Aj{&w|K)%{kz)m^n#t@Tk*cWj1Nj(JB-1;^_+COZrg4deDc$bE=f3az7U zk%DjHa|PW}mdiUPOp)p4(=d-IkV6`PE%L+_h!Z~|Qb@{(XLYT;M-b4=2ekISe2$M( zpx;u79A2y8gBP_Ni-!Atmho#wO+<>Uu@TwzTRvD8p&>qT(Ulw);=W;KX{lvgA!ncS z3EoF>yiO+J3f5-!Yt^iV@2es?I zv1sIr9RUS+P7%*cND3b3OET@PZ~6QBkDhHZWl(z6JP zKpkPQyoo>9l=j+=gpFb+#K)jv(>m-kaCW`UKgO@xIM`_=uRMTKQ#t4tF>FO&hz#is zv1~#?v8&7Uk=5_8O4Pk*^gR!;^lxo23OjL4i@SRHc4hE#b;8Dt1eu3MHdTM&YSx8E zfJO0oBYr4X5MosC`jsJrW6A@lj$qPyJrM>@FGIBE1oUjJKOZiipQw#?dbmB-S&h+> zrQz_ldE(7SZs)*vFvH=2|K#Jv0(92jc}1Lki4`S4e+;xfJCv%rm-Ld$WLr|@|fUi_=%pVt1Vi;)ui=h_kVWd-Sf)egR)TTY@I zZQ=j*?Q)XP;A05|Id$GV?qjz{7?RmJ4ob#B@=`A;Y-7Kg$Mct1ecAsN5D*z8*dZk@ zD6s*w`FAw7kQWkGyVZ0ob#A63t%Z!N%y(ee4}Ec{g!YP z-A(P01zb5Wl^JN2pXn(3KO5R6yUZhflNyE*@6?<2apE!5?kGolT(7ZSLjD#QO8TBC zWw~g^exD+s3GTUPq||r9e|j!Z7CLa_?MT|Q!CSM^^R4)`vwaH#KM{YxRb=yMSfb_X zj`G^pEuQcB!XP%FL1Kj`V}B6T`G_p)Gh}(4P6`cH=4X;GBBc7k_4vwWXx#xo=Ti68 zedc`zaMA7piL&r#_kMQ=89deU-94HpD=&OSj5nqNbRTLmUhrb0Zo3}0f|W& zJeU|`FZiyTrQH9tXxLgeZ-R%S*B^4ywapQxE_yxoj6BS-6ya`IXnX&UvtP~60r5ku zF6f8nmLTGZH}Q1-eO~AN@$K8gmM;H*a)T!ITm+nO=rq{(BNn>&lMwRC_};Mu)_-;2 ze+|m!Zn)$-5F%2u)%br6_kR@{I{%Xg+^cdl;4b{n;rmat5TgRfD{=80XVU-o%iuA* z3x@vOoSRdW|Eu!Ys*~_60 zO`|dUkJSH&q0fkd#_`ip56Ot1Aa-U&V*)0BhP0ZZ? ztX+xh3lsh=e4qcB0{%bli0-|>?#;>A*#}pdEEh-9?hGhj%0U&U?$#4;o?tsGtK+lz zP|%&XUX{u)?}VX$>mc_R616o2BYXy5(cWopu5v}D6OW>qO-HSbPcyccfipRRU_rgo z)72K2(aQXyqLD$tcc;VyFqs1b6O$4wJU~)6>e}`L45|(N{Tq9Ed3iM7Fmz{;t&)Eg zEam!3O7(d6n7KfLQt3M4uVIyxQnJ+g@4Tvvn7Zi-3Mf5__dgV$o6=)&%xyC!ut}40 zwrc#V!OOhe`&X*G0nNFkFE7Wm`FUL$KcOInX)`d;u_z*e)20q~1-fyWd2n`W-Y3P* z&j4D0s=2tBtguLJFbwq3?)QsOLdB8reXk=oBq_LSBqVMC3%yKj?v9zFUdw8e;cw_C zgG?`Y;GAIkanigW_2mPu@^gBz5ceDh*@h-Q(|2J%N==Si z(by8lMum|LMIpsj6nRqx6fke}Hd7NdGuUSkGcu{)Ma`NC!buVyZ3Q~3NF?{cyvVL} z$MUHz<~$ou4DJ~E3rZv0faEQ|F)iQtw1J!iH?Gt}>e;rud*vaa-l&?$^qsG_sluRK zqy?)z?PP19S&>b8vin%L01LL`@zs}gFP9w-T{IVO-4ry*pyDz-PeG^l@52!~{eqJ{ zZg|1|c-b4S-hZS>UWBJ=Bs|~tRA^}^XvO9Y5^1QYQ_a?X$%V`IJ*-3bFRjt)iC@TA zKI_6uk7XDhg_~W_EKz5Ox0&K)wH=CUNdH(L4p%dDR)mL~`Pqzvh4mMW4Tf{gHJ&1W zLg1yG!XqdsNLf#hq3e0BDmXu;uc4?&5{$IbgK?h*u-cZLj1X5=A{#sV(0YgWUX1^1 z-aj%{wZ1RzOW+U8ldZF}TRi<7F-A(q_`|hbA_cJ;g(%doW{igM846!w3h5i)!Absu za%_qQSo-T@zcZioqAK)~0nr`9kEi0d?OE;=?+X{_77d`E0=s-AR zp3wZdF2!ih1?d-|s%dv?U^8xTLv@LI?oZLZ)N)T&URw3J(c6q7**O-!d9$-}+t?Yr zCG~Jv!M|>Q3DL3W!5+=6d3q>ps?tt2L909#Q(C)sd$lIY8*4ek8?oOEj757A)E_EJ zRU+z35tw#BgAr|?edOQmtjhC$eU|$ElFg^T`gvA7{aCncDH2O9>O6Z%_5X$ zTWay1?VgTRmWnd`NR_?l2L`mRPl8rfr-^TE3M0LOOH^4;AUB{!Z1(Y& zaJ^0*y>-tzk#(Q51YKXzPF)B{d<-uRs^U^x(2Sa0t+!;NUq!*$~+D?<<(_1*)%SYXO^1nSu{v z);jG-v|Mz)n1b0V80hHafZADgznip;1V!It72K{Jf!plTQW+EyzOQ8(Rrll%;-k|u zaMOEQP`eEpH7R7}lZ^xBo@35hdWWUBN3c%(xbTWU5xzX@S|@8Rq19s(vDdK9@IRQI zo1vP>z{75ZMXPA^2-AKO5=j|&cnU9Z9!U=&xWWFsP&fnJJ@L#(jPblnR5AuGN0_1h;5L*V>ktmD!tMMXI< zb!lwrhKlFMw%}_c2^APTje10`-^(H zxuO7GPW5ffo>>EXzn}T}63rGW2RcpZz&j1|lE!2-l(X|tfld*M9M6pMU<-!eMs|aq zOTW&3(kxzU_usm@(1o9!?qDDqssUwSYJv01Srei(+1?*m(gakWO{nz<`LbIu2XREB z^E&aU#H@jGPTx-O<}5jw+JybDO{>Eb)1Izqb-{^sbAlw7%K%4;wd%`Bqa`aPbjr(u$0Eu6-eoA0f-@QGM0nT#s^<#Y2>#o!TAtB+OqW^Qeqt$zoj(yzpwDQwU)<-#8ot~ba zGg!ATJR$-JE}xlRU8PqPe5)UpjSr%$YFA$Od$9v6*Tzr`fz1S(_`p`1wXR3XEj!V? zs)Xeq9$haPSlHM!DxXsssyjs<7pX%4wn?JcACg*EQ|NBL~xUx9g2^rL>>dJT*edSES{?A4F!gCAC@vGs5}y?CWl zp??(0CYChvVET1RoSkGZjL_Ec;07zk(xL9@R3lNvW^T%+MDahyWbO}IN4_F zD}IZ1v=YgB%!jk#YCV{Ks&qjC=E#=_+Ky3^Vr(Opv!L_{fRYQg-#6P8_Vw?h0v2Y0 z^IxQ+!!WGnaD%=o>&Q^koc%~7jAZdbvlcI*35lNTovVwlHNi;O!Ir)ro|P*(v)8y! zjH?C}xX;H|kG8QtroWKcX$ffiYvE2-hI>Ws#0V|mNj6DmdFkr-E0bN~DmOecos^2%!&zlg0i6%sNy&M7JG7^bPD$TD z=+MB4vyri(QJs-WV;`kmF6-j@DQHJ;_V#HM$&p7c;o5 zw7?|otDv(Vs(*MI`Zf@6ASE3_q5A(65fJ{Kp>@bOd58epr`*o<`t1yYz zA-SHIlvvmMT@_vJu$Z;GfzOrrQ`N^a)6;?fyM?&@Mf;7y-FRp9+wQLE(~+WosT9i? zexb+;b#ZVn0(v_%7UvE?>Iy#btJ&iWD1XbaB;5_%w(&1t%``Z=RJ;Fj+@D;_ zp&49PArdS`3;jkdd@SkRL3WrRbsH#F*o3_UdPdt9#~+B8^`+ zN24~x2xG#=Sh0Vtc^@z#tEZuO;@r{x$_%w+;!h!zPVB2rDF0RRT4q`TxB^WSG-Yla zI7bT9G3LUdsdv_{LYAy>Z|;-zR!Q3NSvG6$jEnLJ1U_OIV3;oY(Ef~O@-E?rBMKz%$24aP zG8B|~w~uoX;6gSzKNgbW4o$H4xr&c+S-*Cum7=?tQ;)4-=KFft2}r3F)j5TIk%AnB z%@|rAQ6Dksix&y z{O^Fpq&z_my7{_EwZ|IdO~3)2S3<>EhUJY9{18RKJzFQPPl9iCj9$wZJ!4~0Dt~=o zt@znRUG+^(#P_AcYH{zygG`15|CiI7>$i(`s;VAodXz#G;mXj6@1F;_uMm;qHv3$i z+#AE{5W65^yNjV;>3>CH1~ zT>|sPf#8RyR<&WE&tosCI!EKi+qnn!R>A&*E)^A(c;{V(15GX8R;$QWZ$`2M_D&4U z^Q-%(Ph11iV#2Ck&$PHDJuD46sYqyPg8 z7W>!~R&;TVvjm*sG$AJG9e;)ihJ`hOP74fwHl$%Rs78n zLLi923-m`AY8aT5xq6pTUX5}Wr4TpIiRp_;70vS)f3^ z5d}_!qD%N^drA^91JJ3r+(4Nyv70=6X!#~Y-AOlca#G{moPD4JtaX=DPH9^94nWly zb@_`u{pe$C->WXwrq5Sa1R|P<$J~p$fVx!J%|9$kh$HVV1pydY4i$LCp^T%14YQ|o zHCcxp<+MaEA!78%_eIDjAqhqcqJWInuR>4hALMabgCjY~k~8POOfhb7qPpMN zeyKb`H6x1MwG8Wf)>Wbg`%6{Iqk=v<_|z>|ju{Kc=0;y0W_*=lAlr!c5k4B(6u5m` z4l+t_Hpxyp&)mM{U8;YeRns!r#FZ2IT!^Sw^dF!)-x^kUWsM@9*>}Anmpj`J2lCrC zt0px5b(z2#cJS-;7WsnIYG=)sx9<>OelvQZrleaMj9ad*^@FHx`wM&qzroL)8CC7w z_6AylPa{{nBjJwD3Cz3j(nIQlPAB#nW<9L9J?`#tNOx}12jL!b><=Ku*d3$Qm$)Z7 zDl#7QFNmHPz;v27c`y(+_Jl=_#wC6yr^W6%{(M*EUvZpgo;dP+E|R9vcA37c&cm(5 z!^<0zD%aKwsxg=u6c&Em3Yv5!V`NlV2TQxtiO3yP+X&lQ31o-lgN|lzGs}E-2se^_ zuyxmqlwR^f1aGn4H^1k>$%h8(7zoUNZU?$5y@=bHJmNzNI{Vhb?%Y|ld$FfPxT4Eh zbClc>#!H=Kkv9!v2Z|Dex8ke`MyHt4bHU;TmZQ@Fxfz}4YWz0GV|g_Q(JM(!fw~Bp zlhh39gg|uOD|0iSzuD?>J+P5i&p06D#0{s|(-|D}B5AmAj*n!C9PS{JwE5WhNox`N zy`Hzt=k}16HKso2YywU8%%dvr;zFfWm09z6Q)Rk}qd;Es*;kt>RDpI$n%Pa2Q#zP& zi2Npuh~jaGC2dz^qxhns@y+*CrXAcIk*0GjG-%qW*^YLMVF`e z>h(GsQOie3Z7Cg7zh9rT!tVe4^Gt8SLRL8x>7XFwJ04`Vokz@kc2&%Sg7Gp^+BDr( z%7ozX_he)~6&Mw0E<55tj)ZZIK(cy;#rVcsROw8ROiUB946lgT)cPE*Co0?xkVOvl z)Qw&q?mLR^ef^E^>blWdQ%N0#uXX987$!48Qdidx<-LpJ9#J+^riSVhdW#L zjcNiug}-h+Vq_Z5l|N#?WlhBH+8)txlR=HKQ|_I@2Aw^=1-NFc(!Dj99skJgAc>TQ zN}6|5ef#mQ?{g+)plW`@Y6Pv@1&PV%A&j8wdp9Qo#iMUR8rpHx)b(cf^fcjL+5XV7 z`!QGL=hkklU-iwr90zEKz+M^Un!N|!&4|^i(_JRb_`$N#Qh&?{Q0qmZyDK+9)_|VPr7e$4N<=nkLgphc~)zM9kq<)Nm9x?5@V%_DeQ>ud~X zRxZu-WV2ilxXO@LBh|*4c#}8`0t#PJpZ8|q_zObZ%g>;CuR*MI`X|=(le^-u6tA-Z zM3RfN7+Q+8sBZo8#!t&CdNbq|odeo)f5&TRxCajL#-QyxeJh!b1Cy4I-MPJ*)wwl8 zB%84S&$%EEYx{>DjwE!H_gB*#umpSvCHD?m_~v^IW4!2#b$))c=XIZ3{=gKhA4#Xq zgaYBqcs?`*p4~{TZdSz<5Zrg8S?2ElW98L!g&d zmN{zu=e^qc`R%2XBHxo1I8bXzc!3=a=(%Ciy%s8UINy_}Kh$FZL`K{!)jUX1>57&& z={d^i&QMw9y94A7w^@TI#VMz9na2-&f(5C%Pu$nA3rC7vp9>5ER_Yz*9zxF-0%Uir z{P8CHbPkRj>Y`7V{e8z4Jgft_AJS2O6CZS)hnXSl*zg}*={NF9O#S(aa_@@8=yCCL zU=OXnRIBDnhR*7O@r=Ve5PzpBni>Gyt4_(&>OlZ;fUdOb&>73x^R(701*kap>AA{M zO&oKl#U9Ed?crw+$NQD7KZ%f01<6f>KXh&dc39G%)>vK0gtAq%*tsr!#38BN?nGq5 z6i#){#ow-wxN2IUj{fAb_vkq-DKBDMPM!X?alVU%X?rK-Am9jUireUR<7Xt1=>bexy6&2C%r7(0_vTCoz$bn*7g>6?^ha&tGQS zwgx0N)#6?eWc*k>;W>i3ndudJ*MaGMK8&v2Z$n;So- zS0b}F91Ev=TzE|43@i28)DJi3mU&V~CzECsR7(;IV!N%6&S9JaxwfQja3$CuS2o@` zFrp(DinkbOXlg77Z!{MwHr&iJRnmB^kXiB|l$z^DNk6?$amyBL5i~e`*Bk)Q)E45YNmf)nhj8C@o?P*98H$_@#Xyfl3`=%FXJqXUnQu{cu`ef2AC=o1-=XC z&BW;Q6#b?W$8@rOyk!B^&=8UKZ}s%?)E&=^?6yXAO1MB$Cruw_i<6_cg*tws7_MP4 z{uINWGh>=^)S9KcZPCF;jpDAOHh;tjuwq-vE)n5;|GL^2V3NC^Lh=gcK$-7VpPC#c zNED*QSE>l>UiD%koMDo}GzTjhd9yKWVpI?i4vgI|dSGZ_Xs6;G-XGzKJIlW)r)pS| z*1mNJTl*jejMgau4FAd zat3?HvQC=&F7U)z;GqIVzYdHfGkzUmzo zS}1{{MA2Pyd^|2=3hlCO?$CDO6Nh`;l6ASw;1I`F%qLi`6}VqCsdd;8P>VuQmzI_m zIr!?4lZQ1UE#lB|r|!8Gf}5}(Z)SNQGXYHR2^h0M;)Nh>HsJ8wE)WW8_`H{%T^ zZnYdfimzz;q0f-pfDiJCY!BafUvy~rT?UahgL%Sh$2My+Abr=XdQD5&^^?&QlIcdK?b;N}xxW`} zxm=%=+Z939Qf=g4S80rxG^wJu66`#bF2(?5pf2v;@0R2Q4_eh3nb=gwP3nL*MRUms zW4a^@ABFg}13!c(>g6Pz);N?-T4*(-d66o}^Y`FyQ)1@x4%#KxDP}5HhM+h{!qsv!RBkDH~%VF z`k5%!rm{tX+vEDhoX@k=l2oeLjKkHdrAwnXM1Qur` z|KN&4cyxH-=rn%``7Fbq9;=LY*(DjLP_&wgO-@)-QL2X`TJ{Xd_pGx?lNHch5jgqI zp0moAjQa{zUy;MLfdE_DrQuPgq)*R_luH65S+rs_^IW8okvE-}&HhSDRJ$5Nuz5jh zuc6bw+$p_lfIIlZb=}zWS(EQ(^c!u0iN1cOcVgbS!J`X$(&fo4ipK5o!7s?$r{b~` zu|)}6&UQBO$}Vny(;$ce<+t7o6lvt7cixp3D%vk%)2zn3Zx2jKA5T||R^=DUPC8Xf z>4LTO2B?s)j}G@Vp|l*CDa1^itH78l%%b);^Ol74`oG)lpNBB;@B%iB`1rDYhb09& zwAITMKvlj+*jg^)h3Q=LU(1ZBM;I(xym3X{^n9TNK4Qa1xx=ky8U+oN4J(kRiJ;Fc z97R%>mxfUv2xq`pN-4930w311kw6L#{|CYmUyJOUz}FibzSI2mE2e3!M-B>v%{Eky z$#-T<%9^{NBZRR}%~wJeQz%CQEwFc*ZlOjyLCJ}nz3+GRu0*~nM4EwD2SM}#vMhA= zSZ(|j1nHhB82{i{{Gd3c8yR^z_UtII59+{Yz#l+q)R`Ap5hI38czOtxjlKGnVmDeh ziYS}7PJ-2oJKOUU@n1ISd!XI(7nL#z5gF$#Y1MR%*vca^z@V$tmHAHTi%bP!Lw~*G z9Y_1k*7`!@-47`?E5&{@jR%3K1y18a_|77CH^R(HPnnyftiD7P z*Se}t-=X#6@vDmNPpx*wy7;S#OiZc@-;l@$96X)8$9hXG-LTpF^*b)v>E zD}K z;Nzy_&o9IZi6BiSMEUJjRw~&g==fEevj)bp4bmT*5CyA~2kYx45O$|7#l#g-V5s1d z3~or0Uc>IoW{K@*<4TTOE-WWA`o)@SFIxZ3>Eg}J3vK&K@#Ap~Ufk04)@6l;yoFC} zJ`eQ?)CzPpxN2kxS6X=V@cL!6lZ5=TZ?a(la=8& zvu`)TLVOK7vdfy#QG0F%(zYFOItI&56L}|YiVu6Qy3e^QR9eXQ*M>PaKTaa6l0Cr+ zx#uPP73lNPx|!F_3*S<1t_E23Gf1BAre&G*s?R|9))D>(vX%y=JAW1Gk*ztOaCakA zyRXZc;L31kx1VV{7jvvDSWXr=YRdg%%MdHUKROQi4LD^cowPVoTzR@vMwUCR6jKzW zr8zA6wT3v|T}J@b?$eDfgAn_4g*P5hCo>UF2uI~oHJAa8nd0K$Cif*HKfdQu9fGl_ zJbX4!4s0aXEk>56x{hDvv{y{(_h{5W4q?(hO%s_qdywU?e`eb+k7Qo%?y9U}jerk5 zP6_F%$@{y2%{r;YR1vmr6#1a$Pf1y-Irpk+@CI^DP97+#s4_9-J7z=niecUg6te42 zIyMbhTl);nZCw->7~W$_i3P)Ev zWNaiSI0e{P4g=qq8`@S416iy)taS0?(+c%KX}Xm=`+NpbH7;!$>yc3qyiu^FLgh)1 zp-=XHE-h?Hh_@zI%#^KXncBuITQ6K#%;ZU5)L z@yA?6!2GczHo`(CTx19sAc)b%Wlg-0Y({cKe0?x`h_o4w#s1)qR& z6aN~~zZj7Jur#~LHhICYu;B;9hBvRg%YDKbSTwS2R)2s+whB~T9-{9(tS8VK;a*}m zKwk2Vt{&(2g3kIIojc+YsJb2!TEBDXgll3jWT2v}e;q*&BVhAXdcl9z1~WtaLTcm4 zm>x9C^|uS{nN;IXBy~mIt=&1M@6tHc2oEu-j9z+fo<8X{GXDLYfdK zl|!+|C8>DG`LN<^hn$|Gc+PF606G6&X!~w6*N|^RzqLkejW%k8d|Yz>LNR@`fQ@8A z&GkyvlX4e-SCNST7Bt!4);fPzTm6Ujt&~WB^|_amdqZiLlHaoc z7swmdiwL zq;-N4=3gMkD%U~_OCJWac<3vzD}cUw0`B7#J<&aj0lO54gUts;7sVN!g{^e&VeWDG zbgybZYp|j#m~_kos2pPZ+I*3Uhldw!?p_Gfg4C8x>-wIxq6gaRVjAn&Z4D? z@5)7F`fH{AH}%%!0MIb1Ad$E#`o^iFs}jky!fmxwbOy0H?{A-p;j{LKCNZ2sDuZy` zRkvrk?z{qZ4u0iRcY3fHBO1_Naau?Y*jJ;!6hlIl=78UIe zF*+9%uDwl%haWG+a`mls*Dpqq%e$AleTDM9>`bJ8>ty5^$?42X=;fBf7(6Cyt-rOT z{)J{Vsp<1ti1YP7Zv^vqq9W0m=UVbL*u&H)ecCtqpB@7(!b+2P=d zxW~6m^xOW{1Hp^CiNfzudT1~!x!~>g5j6%AKe5hVNo1$u$9xX_;_(y3NC|EPz*lYG z^P+z#E4IXe6(`tjuDlH`=ze>L=PvIXI{|bd6XcBX$U=0U!?2JGd!yC~D2$sfQGGbj zY`r^7GhDy^yOo%?jbU9F!)38v#K3ehj&(JUVkg~rw7KOU5hF=1gIpZl@PZYQ253IE z$+CJa(X}(p>CaCA?rg^62;xL5?)dm{O&or}{ZX_#}aLBi=78%x`CdBAhN$ya1@v&M@N?SF zlJ1fhxpTW)fi#6#N3(E%S($S=Qri)GAf4LI+W`n$hLODd@~UT8D%`d>90`1YdPY}yH8Ns*ar7Kb!dl<^78JMT0> zy$%9VY@)R6v)#ANPuyuT@DZ{&k`{IK?Ke@u7ECaH+%l0vCd)I*VGJQI`|={n&S3O! zt9+zmPJD8B9C*39*IkXU54mTtmjcGZNrCAFZ2k2PU}}X%iJ&H(uiayMU$C^QS3Mgx zyOfY)0DwU*ZnGFoOMULQ!y-c|}=<%jVE&fSq)PFvq+sLM{3Q+iIN z)V7l9QrI)u2xsKTKxsDQiNNr_P+7QivSnEU|D5}!_^LI|5*LRcUsaE;#}=Hjz4g15iF*;s7Ku*U0+$E04cQjizYayl7qpc~w%YIjh)V%6O5*VvqrP6!0wAOu@=)TRF^bk3k_?HyF zLy)I`aw9~*DZ5N~&h}ipb(a;@NN(R?0-OZ)iBFY&!*Vg(EppRWP%&Vz&?%2GU68k- ziFOJ3hbyG%Dm|PBnscxDiog)90SVh=w4a&%eH(y8an#{k%^KNuZlLu)C?%&{Q8s?V z`|R|GQ3EM|_D7Z9{1VnqNPkJ)p@@W6cU~@G4NgQOBkU)2OGC-@ReLJ4$L<~4bxMQ)9cM$tV_Ds zgL&n0NP%^NN0@YiR4pyH+^3F0CbQ(kWZwgwXfWaiNuOn`RwdecFfjs2-@C)`j%ax4 z>p}UG=GATwH`XZ}N55j|l$Z3{aXnFfN_{bWd3F@%X;VBM0MXx9XSR)7?`)ow`oM+h zO67_xORGyazv_B>c#wD#qG()5Ny#(0=x1$)y1{b^!;@kNW*#ZYWQT8 zSu-C${|fj~ME4@SY@qCO1ZF64bX+wfv8jnhdPf5E9*mrf=?2xgl)j7N;bjyvySO*d z43f`_UW?bBSlTIxrM3l$?N_PFKRPiv85$a>JNUgwlNTD*A*d3r_TrZttAdt=2DsmM z$FgFT(3&%WP}5GgV2iEt=fPA#oO5~~#N;m8`n}ypNE0LPh|sVStZ`CO^ItWQiVr1k z)3rNVCH_|j!&0^n`xL>!J09)ya z0i_!2kMkQ;E+)}P+3k4~L2J${n}Togh|YjMcBf7pgH%SYC2kc&?c$C)S+$8{!=(D?1;C)hDSToC~^Q&B)byZ{HzIA2? zsO##epXs|jSKmJxr5#PHM}03KkGR$tVf}tzsQ#VI#teQ5JAEGFoz`B?h4Uo*dy=-X z91%($*{AjNOZCmgK6A07gutl-$w(;U4Y{`JUVKUUTIF<<`>WOWn?0pox@+yHUc4;! zrdByUjT)k2e~5ezmDHEgy}jwITo^>?ecyu9qKNuRxenryH=1jotA_%Q$yQjN7adhz z*kgl)nGA6t0lP*->PFHUyT6l~ApIItMQzjl=OY4Jbt@FxAfHmRqIUIrx)Nc08LJgP zxAj?_=rdQz2Sqbxrhon2Vy1T?KNZ0(bJ^nx z${Wumesm^DTHITpyB3x&BPG(XRFobw{PD*+F>4UKK>$>;lnvUM$!6)p2_BCI#YN%omW|H^peQB%3g=UGzC%RasOPK0;a< z4-HmEafuw?wxl2G4`0p=#z~}(Kd6=C_;V_*f0XoZ)Kon_=>Rz$F~5IhqX3y}&?l7{ z;<{}K_N;V+OvQr9PBx>Rj$!x2RC5GrvWBxYY7Z^>IB2CW&F@n3XG#W~mpa7}@$Jgb zmt-8+7(`hpn9z&uUnVmhu32r=eod!K6!j1Hyntv#IBf20hcv(G+`pa@7c@kT$SS2O z0LQiHrU4IyK3Y#PCxEYfQW2T_wCF?MSzm^2(e(Yp$D*O)Xk$CN7E9_K0VzfLj6k+} z(K7&h-Y%-%BnRV1pY@#nx;z=16rzOhY-{D?qqgN0hFP_Tl9$kBGA9{8Xl`X~wm=FX zDTjP?N>`_I8rKFpZ_{nVdXDxDfS9VYm&QHS*m#v#Y=G$0jL6sC}TB8Eit})Tg ztx^oKZ2Vb4d{x8WtHMh$IW3vuP;1ljkV^fDW5NtiCEx^!t~ipYuu1?MKM5 z^8{KcUPaT{tj|J3r^aQi2#MxyTEkv{ zAW4G|hnXLesTU(#MUhbcl~U*-OcS<`bW+i3VSR0>^36Yj4e6MWGd-#Sl$WUWovJ~Y zi6jN<+pW6#2&xNFLq>XPI__ZO9VbRJU8 zZjr-Qdg&_=+m;2b?VwQ*yUCvLPL$UAEEgd)nuJ$3=o8!2L1fB>DG9&WfQIf}KIH>3 zAnj^!fEwpjrzd=II7A&s* z5dR6>y6vU%^(vca>!mnor61ZB!nNfv($6OT2`N%uMR<@cYIYKu(jC?;!bUZ!s4P9J zt}OMp@_pZ&j}Hb8H0hnjz(B&@q5UdVNaiJ-IVg-L3&~amC^%jfUrwJyQ)ZpF5T$FA zE%i{7K}cFu(I^N?9#dav?`#&}6yCDv-+izN6};Q)fOqUHSd*3@KHoG zq&Qd=={2S3(uP{Y43tN86o{9^4!#T4yt6q*C+lVB^a+Nq&8qdJctaCGt>f{^XdeAi zEwg{QDHnojUIm#=*Zk#fO;a~RC#D8QVRu{NKdAh9pY*i2PxvElqg9Pj{#hEkNEj~n zIZ-jF&Q1yn@$b2qT4V(MJ@lD{zD(XtN>yY)=ei6vSJCs>Gm4VWRFN3&B~e|-nRZ-* zKhxYeqKKUoOJssuUJ47*C~m_M}Ad%;Y+vIW2nn=qH-kXDUqs`QeD)p$=m%qopQs6nN#`TQo8G8m4G6!4^k{v*Dl3je6Kx-{HHNQYq4%anEKw51+jGCDq7G9m{>Tp2 zAY@>D`tUl{z%pNqX@%bR_gwX7dlk>0M!d@lN)SEpytcUdqc$Lc#p_!Am%H_4 z89vdlu>c3I0R#0Rd#pd+z4wit__6m9#MS+EO%~p+gE$lydTZbRSez4Q2n4h8eL2#< zI&zzavoj$M)7cw$wqydy4q)ezCTPastVLHw9?f!vc5@`}n0jgG6%l!vRsjNH3wtFN zAbvX$*0HX&nJ-I6lR`RMtouGyXYunwH*K=Dz(<}2hLnzjATr^zLLEfy8p|bQ*D5Jp7Yz!L1ugcam%6EOBKb zT5Qdo!Fqmu$?-NG6MI`({_Y%R?eh&`f&?2qy8DUdwWP(Ct{Jz}k++j8dqm}Lh1v{a zKcBAYSjJ$QKkOe)dAs(E-Ok+gub5aox>G(Bz-YNoP8%Pm%7J&dt4*rt?ihRsH&zTZ z6H9G%eMw@@`SM8P!=b^+wz4xazYFzR)7mOJ7cmF9qsM*+<>7@jrJ>WCbV~F>#7wWO zek;XUu3Bzb_Y#+}ICfg&jUzimJlCU`otE4Z4)b5OqJtbXVzx?7U+fn7eD-$dI6W+& zg3TB+?zE2)b2vFS6LS>;h2#dFBz3QD0V7G zJ~>c{P^l^rm66a1PfwB;t`Qv6aIQHb2oFdx8MCNnG+oHwDyNQVM|wsu->!O2#?V!a!dP3hfK8w32?)6OtoCh(Sso{ zposrOs(&;W!_!}dXJ25NH1_(oCqWV*o=L>k8jz^TEBrzWrS_U8FoD@Jh~KC-=W@n5 zHbmZ*oe$lE+sW~JbhFG!6^PFE+gx4oM}oZH3o2X)4Y@DfV?c$_f=zSJ*7zq&bYrbq zC|_&5KtJMcr~Gc`YrUk1AP6Y-evC8t&zsPBNydambzXYw8CB=IjrmvHV3Uyr$k%g+N})n=OkO@~Z1>KyBuX$)%y&Nx=RfD?gg%-#Bz6q5j6!7;~-zhV(!F__w?;Df%}QI zlSn`IS8|#LeD2W5PDrCAmBd?SkwWRU>MTzY?e4-z-fIUWnKF8B*S>Ss)YSYUtghwl zh*|W*t>2AmpaWC&?U#z9+91*qSu)9_{rO`H?Aca!bQeVw(=zUkR-0p(TO}Tx>R0$& zm9lhLOLd{}jwtM?4TiMWQtNd?(P60mBNVDuhJO;F5qVcUJboZas~W>XfC-H% zqc$t`yI(dD4K_p=@vD0W{(4vZmW(q}F(7m78V{3u5_C2dF@N~nF1!QfHlgcL?>KSHcN{ME=3V~w5i&3@ z{3Z!-N(eq_-YrJ>;3KaEYO}OHxW-Ch_*~v?J2lh{#$Euu?G=KG3bS<1fTz6LVDGJ`v zO^vCqmoI@vD7`jtpiM=8N+uIp?0{&xlWJ(D3k6h#ah7#kh_vZ%^3U@z#3k`Dt%}E1 zpo^o}HYVC71n!b+;sH<)8yEEyt?k1&1-t89*JBzU7ozcb8Ch<@u)Um-7IK`0&got} z&Zt_28x)S6y#Z#vYv5U1whNz&q!^%xC;J+5zf(xQ^tPJ7-L*ojJx5ZRK9#?AxRsWA zvO~pHn6s*e$RjS2&BaMD=rPqbZY@TYdwy$B3kiz1HF#q-THm`)d|YG!k2E0_LNz`$ zRMl82ulIRM26s+;&Wf{2di^C#0|s&8H%2fgLP#k_tK9%EFdQociH6&y&|eo`BWr$k zMJ@0VF$NU#pT`AxpB{%RFml$R;VqL3Kn8+1{d|Iq$-LQ-?AveZQNMHB8Sv9^cQKQW^w&n@Nlv4q@>d1tup{#~ zGa?AGHpi+$+G1F2?M!8*At)m;tdA6O}j5j&Z*HLEEPAvF8cZcw~5l>(sKrw z?|xs|=r*cMN>gTp73`*5T(=?X+2=jQ9jSQCSK)mlFvAP8Rp90;5Or8;LKlQR8~g?% z5H5`K^7E=oc_V^senuCRlr&*Y!lt3T+zuIQ9px}Hiua&n}BvxOtTSdzK`0B z$`b>HFuqc$b=csTMd%W7pZ_hR`z%R()O6CEr-s$Zc2mK6ChI7=1a4~jlGw~}GX^W0 zRJuJ&f>8Yw>gK z)Ku1>V(JaXg^*0K9_N7DJL1Vr8p8{zJI*FJ{Gy|DcmXsnd(=r=F*)Sn+4nL zE3Yq?X0ne(1wFQYA-nz|p~KvO$xd_$QGzSs=;DbpH8^og|G2z!&YUs6xp!J)X$-!;#%nzGqIk_{fL#9X%;B$_Jmm)R4lEyPf0Tgn47c!C=0$ zlZnN-pGfw7)0o!1E^CYEq+g1(IXNE74+`-?KVF}l4#+o6ivW8~g?9u>ijt@=fBbnU zbGXTRybw6HbRLj7N|gNh%iy0oTsb?V9x);x>tHq{*CHFBHbmnGXUdx~ z%7!!JYTHVm_6K7rxD{FX*!jxs(jnLZG`UvybG_qWcm2c!$@#)8qqRlH>gK1|XX#fx z)JGeA8b9n6Sg=;8jaAFy2g~_p#B3JCguTX6@*0aCj(U%E0U=D>0Le6*WTv%2p76)4 z<~(oJUIeb3jv!=J7Ab~0x`zDI?yFxoM0bF>B_(k&F)?CtI-BkxN|Fx4wXX_vPxQCq z5qI1^uJ?~8S2c(wY-i$7^tlU|mskb9| zk$b3Abrl0@Xj@|64X&6!6K9HY3A4@m?6 zv&Jr?BJpJatt1+NjgWu#ETP%IohA?&@n)baD9dx`H+A-#V*1@knk=jxFP*F<{KpmK zexYH$1D~GHQHp;3S1RPvEjtV(R2&CJcm1s*eY{kWxO9E(r~cHBnh0Iw`MQD=p8Y*6 zCgM!@hE96RMCm^rrL9X%4c5{%_qR{gd5JHE$`faM{yAFWX`m>x(X^X*&0h%_VR3@U e^#AXo>4J9Q22WS;fJ-wGp+MAhREt$C{r?M}E*bFv literal 0 HcmV?d00001 From ddff7cdaaa74e8faf10f52f1199624426f940d20 Mon Sep 17 00:00:00 2001 From: Justin-Bergmann <48309261+Justin-Bergmann@users.noreply.github.com> Date: Tue, 23 Jan 2024 09:40:10 +0100 Subject: [PATCH 050/403] Create work_flow_design.md first steps of workflow documentation --- .../essnmx/docs/about/work_flow_design.md | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 packages/essnmx/docs/about/work_flow_design.md diff --git a/packages/essnmx/docs/about/work_flow_design.md b/packages/essnmx/docs/about/work_flow_design.md new file mode 100644 index 00000000..903666fd --- /dev/null +++ b/packages/essnmx/docs/about/work_flow_design.md @@ -0,0 +1,20 @@ +# Design document data workflow for the NMX instrument at ESS + +This is a description of the data workflow for the NMX instrument at ESS. + +The [NMX](https://europeanspallationsource.se/instruments/nmx) Macromolecular Diffractometer is a time-of-flight (TOF) quasi-Laue diffractometer optimised for small samples and large unit cells dedicated to the structure determination of biological macromolecules by crystallography. +The main scientific driver is to locate the hydrogen atoms relevant for the function of the macromolecule. + +## Data reduction + +![work_flow](https://github.com/scipp/essnmx/new/main/docs/about/NMX_work_flow.png) + + +### From single event data to binned image like data +From single event data to binned image like data +The first step in the data reduction is to reduce the data from single event data to image like data. +Therefore the [essNMX](https://github.com/scipp/essnmx) package is used. +Tirst the time of arrival (TOA) is converted into time of flight (TOF). + +Then the single events get binned into pixels and then histogramed in the TOF dimension. +These data will be written be added with some meta and instrument data to an HDF5 file. From f25f021a060ae8ae0981900c64928d06a0cab7c8 Mon Sep 17 00:00:00 2001 From: YooSunYoung Date: Tue, 6 Feb 2024 09:25:46 +0000 Subject: [PATCH 051/403] Apply automatic formatting --- packages/essnmx/docs/examples/workflow.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index ed652500..774ab94e 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -150,7 +150,7 @@ "\n", "da = dg['weights']\n", "da.coords['position'] = dg['position']\n", - "# Plot one out of 100 pixels to reduce size of docs output\n", + "# Plot one out of 100 pixels to reduce size of docs output\n", "view = scn.instrument_view(da.flatten(['panel', 'id'], 'id')['id', ::100].hist(), pixel_size=0.0075)\n", "view.children[0].toolbar.cameraz()\n", "view\n", From fbcc0222e0ffedf0f2be35999cd4da126c15b176 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 6 Feb 2024 11:50:40 +0100 Subject: [PATCH 052/403] Fix unit of weights to be counts. --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 59 +++++++++++--------- packages/essnmx/tests/loader_test.py | 4 +- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 60aeefc7..c74e822a 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -34,7 +34,7 @@ def _copy_partial_var( ) -> sc.Variable: """Retrieve a property from a variable.""" var = var['dim_1', idx].astype(dtype or var.dtype, copy=True) - if unit: + if unit is not None: var.unit = sc.Unit(unit) return var @@ -67,43 +67,48 @@ def load_mcstas_nexus( from .mcstas_xml import read_mcstas_geometry_xml geometry = read_mcstas_geometry_xml(file_path) - probability = max_probability or DefaultMaximumProbability + maximum_probability = sc.scalar( + max_probability or DefaultMaximumProbability, unit='counts' + ) with snx.File(file_path) as file: bank_name = _retrieve_event_list_name(file["entry1/data"].keys()) var: sc.Variable var = file["entry1/data/" + bank_name]["events"][()].rename_dims( {'dim_0': 'event'} - ) + ) # ``dim_0``: event index, ``dim_1``: property index. weights = _copy_partial_var(var, idx=0, unit='counts') # p id_list = _copy_partial_var(var, idx=4, dtype='int64') # id t_list = _copy_partial_var(var, idx=5, unit='s') # t - - weights = (probability / weights.max()) * weights - - loaded = sc.DataArray(data=weights, coords={'t': t_list, 'id': id_list}) - coords = geometry.to_coords() - grouped: sc.DataArray = loaded.group(coords.pop('pixel_id')) - da: sc.DataArray = grouped.fold( - dim='id', sizes={'panel': len(geometry.detectors), 'id': -1} - ) crystal_rotation = _retrieve_crystal_rotation( file, geometry.simulation_settings.angle_unit ) - # Proton charge is proportional to the number of neutrons, - # which is proportional to the number of events. - # The scale factor is chosen by previous results - # to be convenient for data manipulation in the next steps. - # It is derived this way since - # the protons are not part of McStas simulation, - # and the number of neutrons is not included in the result. - proton_charge = _PROTON_CHARGE_SCALE_FACTOR * da.bins.size().sum().value - return NMXData( - sc.DataGroup( - weights=da, - proton_charge=proton_charge, - crystal_rotation=crystal_rotation, - **coords, - ) + + coords = geometry.to_coords() + loaded = sc.DataArray( + # Scale the weights so that the weights are + # within the range of [0,``max_probability``]. + data=(maximum_probability / weights.max()) * weights, + coords={'t': t_list, 'id': id_list}, + ) + grouped: sc.DataArray = loaded.group(coords.pop('pixel_id')) + da: sc.DataArray = grouped.fold( + dim='id', sizes={'panel': len(geometry.detectors), 'id': -1} + ) + # Proton charge is proportional to the number of neutrons, + # which is proportional to the number of events. + # The scale factor is chosen by previous results + # to be convenient for data manipulation in the next steps. + # It is derived this way since + # the protons are not part of McStas simulation, + # and the number of neutrons is not included in the result. + proton_charge = _PROTON_CHARGE_SCALE_FACTOR * da.bins.size().sum().value + return NMXData( + sc.DataGroup( + weights=da, + proton_charge=proton_charge, + crystal_rotation=crystal_rotation, + **coords, ) + ) diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 16fd5a13..adf2de41 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -27,7 +27,9 @@ def test_file_reader_mcstas() -> None: raw_data = file[entry_path]["events"][()] data_length = raw_data.sizes['dim_0'] - expected_weight_max = sc.scalar(DefaultMaximumProbability, unit='1', dtype=float) + expected_weight_max = sc.scalar( + DefaultMaximumProbability, unit='counts', dtype=float + ) assert isinstance(dg, sc.DataGroup) assert dg.shape == (3, 1280 * 1280) From b3f40ac03760e702472a4641c9658d1cdcbaec8b Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 6 Feb 2024 12:00:54 +0100 Subject: [PATCH 053/403] Refactorize functions. --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 28 +++++++++++--------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index c74e822a..e9c4fe02 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -29,6 +29,15 @@ def _retrieve_event_list_name(keys: Iterable[str]) -> str: raise ValueError("Can not find event list name.") +def retrieve_events_data(file: snx.File) -> sc.Variable: + """Retrieve events from the nexus file.""" + bank_name = _retrieve_event_list_name(file["entry1/data"].keys()) + # ``dim_0``: event index, ``dim_1``: property index. + return file["entry1/data/" + bank_name]["events"][()].rename_dims( + {'dim_0': 'event'} + ) + + def _copy_partial_var( var: sc.Variable, idx: int, unit: Optional[str] = None, dtype: Optional[str] = None ) -> sc.Variable: @@ -72,15 +81,10 @@ def load_mcstas_nexus( ) with snx.File(file_path) as file: - bank_name = _retrieve_event_list_name(file["entry1/data"].keys()) - var: sc.Variable - var = file["entry1/data/" + bank_name]["events"][()].rename_dims( - {'dim_0': 'event'} - ) # ``dim_0``: event index, ``dim_1``: property index. - - weights = _copy_partial_var(var, idx=0, unit='counts') # p - id_list = _copy_partial_var(var, idx=4, dtype='int64') # id - t_list = _copy_partial_var(var, idx=5, unit='s') # t + raw_data = retrieve_events_data(file) + weights = _copy_partial_var(raw_data, idx=0, unit='counts') # p + id_list = _copy_partial_var(raw_data, idx=4, dtype='int64') # id + t_list = _copy_partial_var(raw_data, idx=5, unit='s') # t crystal_rotation = _retrieve_crystal_rotation( file, geometry.simulation_settings.angle_unit ) @@ -93,7 +97,7 @@ def load_mcstas_nexus( coords={'t': t_list, 'id': id_list}, ) grouped: sc.DataArray = loaded.group(coords.pop('pixel_id')) - da: sc.DataArray = grouped.fold( + folded: sc.DataArray = grouped.fold( dim='id', sizes={'panel': len(geometry.detectors), 'id': -1} ) # Proton charge is proportional to the number of neutrons, @@ -103,10 +107,10 @@ def load_mcstas_nexus( # It is derived this way since # the protons are not part of McStas simulation, # and the number of neutrons is not included in the result. - proton_charge = _PROTON_CHARGE_SCALE_FACTOR * da.bins.size().sum().value + proton_charge = _PROTON_CHARGE_SCALE_FACTOR * weights.sum().value return NMXData( sc.DataGroup( - weights=da, + weights=folded, proton_charge=proton_charge, crystal_rotation=crystal_rotation, **coords, From a958871d2d42ae4bbae3196cdaecb3ebe250de78 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 6 Feb 2024 12:03:44 +0100 Subject: [PATCH 054/403] Rename functions. --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index e9c4fe02..95499e86 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -29,7 +29,7 @@ def _retrieve_event_list_name(keys: Iterable[str]) -> str: raise ValueError("Can not find event list name.") -def retrieve_events_data(file: snx.File) -> sc.Variable: +def _retrieve_raw_event_data(file: snx.File) -> sc.Variable: """Retrieve events from the nexus file.""" bank_name = _retrieve_event_list_name(file["entry1/data"].keys()) # ``dim_0``: event index, ``dim_1``: property index. @@ -81,7 +81,7 @@ def load_mcstas_nexus( ) with snx.File(file_path) as file: - raw_data = retrieve_events_data(file) + raw_data = _retrieve_raw_event_data(file) weights = _copy_partial_var(raw_data, idx=0, unit='counts') # p id_list = _copy_partial_var(raw_data, idx=4, dtype='int64') # id t_list = _copy_partial_var(raw_data, idx=5, unit='s') # t From f8b1243ec2afe97479fee8439a9983aac96ec149 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 6 Feb 2024 15:51:08 +0100 Subject: [PATCH 055/403] Extract functions for better documentation. --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 117 ++++++++++++++----- 1 file changed, 88 insertions(+), 29 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 95499e86..5cd39901 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -7,7 +7,6 @@ from .reduction import NMXData -_PROTON_CHARGE_SCALE_FACTOR = 1 / 10_000 # Arbitrary number to scale the proton charge PixelIDs = NewType("PixelIDs", sc.Variable) InputFilepath = NewType("InputFilepath", str) @@ -57,12 +56,85 @@ def _retrieve_crystal_rotation(file: snx.File, unit: str) -> sc.Variable: ) +def event_weights_from_probability( + *, + probabilities: sc.Variable, + id_list: sc.Variable, + t_list: sc.Variable, + pixel_ids: sc.Variable, + max_probability: sc.Variable, + num_panels: int, +) -> sc.DataArray: + """Create event weights by scaling probability data. + + event_weights = max_probability * (probabilities / max(probabilities)) + + Parameters + ---------- + probabilities: + The probabilities of the events. + + id_list: + The pixel IDs of the events. + + t_list: + The time of arrival of the events. + + pixel_ids: + All possible pixel IDs of the detector. + + max_probability: + The maximum probability to scale the weights. + + num_panels: + The number of (detector) panels used in the experiment. + + """ + if max_probability.unit != sc.units.counts: + raise ValueError("max_probability must have unit counts") + + weights = sc.DataArray( + # Scale the weights so that the weights are + # within the range of [0,``max_probability``]. + data=max_probability * (probabilities / probabilities.max()), + coords={'t': t_list, 'id': id_list}, + ) + grouped: sc.DataArray = weights.group(pixel_ids) + return grouped.fold(dim='id', sizes={'panel': num_panels, 'id': -1}) + + +def proton_charge_from_weights(weights: sc.DataArray) -> float: + """Make up the proton charge from the weights. + + Proton charge is proportional to the number of neutrons, + which is proportional to the number of events. + The scale factor is manually chosen based on previous results + to be convenient for data manipulation in the next steps. + It is derived this way since + the protons are not part of McStas simulation, + and the number of neutrons is not included in the result. + + Parameters + ---------- + weights: + The event weights binned in detector panel and pixel id dimensions. + + """ + # Arbitrary number to scale the proton charge + _proton_charge_scale_factor = 1 / 10_000 + + return _proton_charge_scale_factor * weights.bins.size().sum().value + + def load_mcstas_nexus( file_path: InputFilepath, max_probability: Optional[MaximumProbability] = None, ) -> NMXData: """Load McStas simulation result from h5(nexus) file. + See :func:`~event_weights_from_probability` and + :func:`~proton_charge_from_weights` for details. + Parameters ---------- file_path: @@ -70,49 +142,36 @@ def load_mcstas_nexus( max_probability: The maximum probability to scale the weights. + If not provided, ``DefaultMaximumProbability`` is used. """ from .mcstas_xml import read_mcstas_geometry_xml geometry = read_mcstas_geometry_xml(file_path) + coords = geometry.to_coords() maximum_probability = sc.scalar( max_probability or DefaultMaximumProbability, unit='counts' ) with snx.File(file_path) as file: raw_data = _retrieve_raw_event_data(file) - weights = _copy_partial_var(raw_data, idx=0, unit='counts') # p - id_list = _copy_partial_var(raw_data, idx=4, dtype='int64') # id - t_list = _copy_partial_var(raw_data, idx=5, unit='s') # t + weights = event_weights_from_probability( + probabilities=_copy_partial_var(raw_data, idx=0, unit='counts'), # p + max_probability=maximum_probability, + id_list=_copy_partial_var(raw_data, idx=4, dtype='int64'), # id + t_list=_copy_partial_var(raw_data, idx=5, unit='s'), # t + pixel_ids=coords.pop('pixel_id'), + num_panels=len(geometry.detectors), + ) + proton_charge = proton_charge_from_weights(weights) crystal_rotation = _retrieve_crystal_rotation( file, geometry.simulation_settings.angle_unit ) - coords = geometry.to_coords() - loaded = sc.DataArray( - # Scale the weights so that the weights are - # within the range of [0,``max_probability``]. - data=(maximum_probability / weights.max()) * weights, - coords={'t': t_list, 'id': id_list}, - ) - grouped: sc.DataArray = loaded.group(coords.pop('pixel_id')) - folded: sc.DataArray = grouped.fold( - dim='id', sizes={'panel': len(geometry.detectors), 'id': -1} - ) - # Proton charge is proportional to the number of neutrons, - # which is proportional to the number of events. - # The scale factor is chosen by previous results - # to be convenient for data manipulation in the next steps. - # It is derived this way since - # the protons are not part of McStas simulation, - # and the number of neutrons is not included in the result. - proton_charge = _PROTON_CHARGE_SCALE_FACTOR * weights.sum().value return NMXData( - sc.DataGroup( - weights=folded, - proton_charge=proton_charge, - crystal_rotation=crystal_rotation, - **coords, - ) + weights=weights, + proton_charge=proton_charge, + crystal_rotation=crystal_rotation, + **coords, ) From 1e6e52efede91448212220841af59f73414b9301 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 6 Feb 2024 17:55:22 +0100 Subject: [PATCH 056/403] Reorganize tests and update type hints of properties. --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 6 ++-- packages/essnmx/src/ess/nmx/reduction.py | 2 +- packages/essnmx/tests/loader_test.py | 34 ++++++++++++++------ 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 5cd39901..1ba836c4 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -103,7 +103,7 @@ def event_weights_from_probability( return grouped.fold(dim='id', sizes={'panel': num_panels, 'id': -1}) -def proton_charge_from_weights(weights: sc.DataArray) -> float: +def proton_charge_from_weights(weights: sc.DataArray) -> sc.Variable: """Make up the proton charge from the weights. Proton charge is proportional to the number of neutrons, @@ -121,9 +121,9 @@ def proton_charge_from_weights(weights: sc.DataArray) -> float: """ # Arbitrary number to scale the proton charge - _proton_charge_scale_factor = 1 / 10_000 + _proton_charge_scale_factor = sc.scalar(1 / 10_000, unit=None) - return _proton_charge_scale_factor * weights.bins.size().sum().value + return _proton_charge_scale_factor * weights.bins.size().sum().data def load_mcstas_nexus( diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 37ec7127..bfc6cb18 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -45,7 +45,7 @@ def slow_axis(self) -> sc.Variable: return self['slow_axis'] @property - def proton_charge(self) -> float: + def proton_charge(self) -> sc.Variable: """Accumulated number of protons during the measurement.""" return self['proton_charge'] diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index adf2de41..79064334 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -7,6 +7,22 @@ import scipp as sc from ess.nmx.data import small_mcstas_sample +from ess.nmx.reduction import NMXData + + +def check_scalar_properties(dg: NMXData): + """Test helper for NMXData. + + Expected numbers are hard-coded based on the sample file. + """ + + assert dg.proton_charge == sc.scalar(0.15430000000000002, unit=None) + assert sc.identical(dg.crystal_rotation, sc.vector([20, 0, 90], unit='deg')) + assert sc.identical(dg.sample_position, sc.vector(value=[0, 0, 0], unit='m')) + assert sc.identical( + dg.source_position, sc.vector(value=[-0.53123, 0.0, -157.405], unit='m') + ) + assert dg.sample_name == sc.scalar("sampleMantid") def test_file_reader_mcstas() -> None: @@ -20,25 +36,23 @@ def test_file_reader_mcstas() -> None: ) file_path = InputFilepath(small_mcstas_sample()) - dg = load_mcstas_nexus(file_path) - entry_path = "entry1/data/bank01_events_dat_list_p_x_y_n_id_t" with snx.File(file_path) as file: raw_data = file[entry_path]["events"][()] data_length = raw_data.sizes['dim_0'] - expected_weight_max = sc.scalar( - DefaultMaximumProbability, unit='counts', dtype=float - ) - + dg = load_mcstas_nexus(file_path) assert isinstance(dg, sc.DataGroup) assert dg.shape == (3, 1280 * 1280) - assert sc.identical(dg.sample_position, sc.vector(value=[0, 0, 0], unit='m')) + check_scalar_properties(dg) + # Check size and maximum value of weights. assert dg.weights.bins.size().sum().value == data_length - assert sc.identical(dg.weights.max().data, expected_weight_max) - # Expected coordinate values are provided by the IDS + assert sc.identical( + dg.weights.max().data, + sc.scalar(DefaultMaximumProbability, unit='counts', dtype=float), + ) + # Expected values are provided by the IDS # based on the simulation settings of the sample file. - # The expected values are rounded to 2 decimal places. assert np.all( np.round(dg.fast_axis.values, 2) == sc.vectors( From ffd7c3cbee1f72f3bbfacf18a58ddba826394e2c Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 6 Feb 2024 18:27:21 +0100 Subject: [PATCH 057/403] Update copyright year. --- packages/essnmx/tests/loader_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 79064334..c748eeb2 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) import pathlib from typing import Generator From a1a2a15ec89ed3d09ac48390482df3eb3aeb872a Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 6 Feb 2024 18:27:37 +0100 Subject: [PATCH 058/403] Export method tests draft. --- packages/essnmx/tests/exporter_test.py | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 packages/essnmx/tests/exporter_test.py diff --git a/packages/essnmx/tests/exporter_test.py b/packages/essnmx/tests/exporter_test.py new file mode 100644 index 00000000..a05fa964 --- /dev/null +++ b/packages/essnmx/tests/exporter_test.py @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +from unittest import mock + +import pytest + + +class FakeFile(dict): + def create_group(self, name): + self[name] = dict() + return self[name] + + +@pytest.fixture +def mock_h5py(): + with mock.patch('ess.nmx.reduction.h5py') as mock_h5py: + yield mock_h5py + + +def test_exports(mock_h5py): + from ess.nmx.reduction import h5py + + fake_file = FakeFile() + mock_h5py.File.return_value.__enter__.return_value = fake_file + + with h5py.File('fake_file', 'w') as f: + f.create_group('entry1') + assert f == {'entry1': {}} From 4038e5e57e05601823376e09a7ae5df1987d7966 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 6 Feb 2024 18:32:00 +0100 Subject: [PATCH 059/403] Add fake group and fake create_dataset method. --- packages/essnmx/tests/exporter_test.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/essnmx/tests/exporter_test.py b/packages/essnmx/tests/exporter_test.py index a05fa964..0f665a94 100644 --- a/packages/essnmx/tests/exporter_test.py +++ b/packages/essnmx/tests/exporter_test.py @@ -5,9 +5,15 @@ import pytest +class FakeGroup(dict): + def create_dataset(self, name, data, **_): + self[name] = data + return self[name] + + class FakeFile(dict): def create_group(self, name): - self[name] = dict() + self[name] = FakeGroup() return self[name] From 177727d64d46e40c57013d282fa4a3e188b220fd Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 7 Feb 2024 13:29:47 +0100 Subject: [PATCH 060/403] Add tests for exporting method and add in-memory byte stream IO option for testing. --- packages/essnmx/src/ess/nmx/reduction.py | 40 ++++++-- packages/essnmx/tests/exporter_test.py | 118 +++++++++++++++++++---- 2 files changed, 128 insertions(+), 30 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index bfc6cb18..1f9b2c9f 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -1,7 +1,8 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +import io import pathlib -from typing import NewType, Optional, Union +from typing import NewType, Optional, Union, overload import h5py import scipp as sc @@ -140,7 +141,7 @@ def _create_sample_group(self, nx_entry: h5py.Group) -> h5py.Group: def _create_instrument_group(self, nx_entry: h5py.Group) -> h5py.Group: nx_instrument = nx_entry.create_group("NXinstrument") nx_instrument.attrs["nr_detector"] = self.origin_position.sizes['panel'] - nx_instrument.create_dataset("proton_charge", data=self.proton_charge) + nx_instrument.create_dataset("proton_charge", data=self.proton_charge.value) nx_detector_1 = nx_instrument.create_group("detector_1") # Detector counts @@ -196,14 +197,33 @@ def _create_source_group(self, nx_entry: h5py.Group) -> h5py.Group: nx_source["target_material"] = "W" return nx_source - def export_as_nexus(self, output_file_name: Union[str, pathlib.Path]) -> None: - import h5py - - file_name = pathlib.Path(output_file_name) - if file_name.suffix not in (".h5", ".nxs"): - raise ValueError("Output file name must end with .h5 or .nxs") - - with h5py.File(file_name, "w") as out_file: + @overload + def export_as_nexus(self, output_file_base: str) -> None: + ... + + @overload + def export_as_nexus(self, output_file_base: pathlib.Path) -> None: + ... + + @overload + def export_as_nexus(self, output_file_base: io.BytesIO) -> None: + ... + + def export_as_nexus( + self, output_file_base: Union[str, pathlib.Path, io.BytesIO] + ) -> None: + """Export the reduced data to a NeXus file. + + Currently exporting step is not part of the sciline pipeline. + """ + if isinstance(output_file_base, (str, pathlib.Path)): + file_base = pathlib.Path(output_file_base) + if file_base.suffix not in (".h5", ".nxs"): + raise ValueError("Output file name must end with .h5 or .nxs") + else: + file_base = output_file_base + + with h5py.File(file_base, "w") as out_file: out_file.attrs["default"] = "NMX_data" # Root Data Entry nx_entry = self._create_root_data_entry(out_file) diff --git a/packages/essnmx/tests/exporter_test.py b/packages/essnmx/tests/exporter_test.py index 0f665a94..927a8dad 100644 --- a/packages/essnmx/tests/exporter_test.py +++ b/packages/essnmx/tests/exporter_test.py @@ -1,34 +1,112 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) -from unittest import mock +import io import pytest +from ess.nmx.reduction import NMXReducedData -class FakeGroup(dict): - def create_dataset(self, name, data, **_): - self[name] = data - return self[name] +@pytest.fixture +def reduced_data() -> NMXReducedData: + import numpy as np + import scipp as sc -class FakeFile(dict): - def create_group(self, name): - self[name] = FakeGroup() - return self[name] + rng = np.random.default_rng(42) + id_list = sc.array(dims=['event'], values=rng.integers(0, 12, size=100)) + t_list = sc.array(dims=['event'], values=rng.random(size=100, dtype=float)) + counts = ( + sc.DataArray( + data=sc.ones(dims=['event'], shape=[100]), + coords={'id': id_list, 't': t_list}, + ) + .group('id') + .hist(t=10) + ) + return NMXReducedData( + counts=counts, + proton_charge=sc.scalar(1.0, unit='counts'), + crystal_rotation=sc.vector(value=[0.0, 20.0, 0.0], unit='deg'), + fast_axis=sc.vectors( + dims=['panel'], + values=[[1.0, 0.0, 0.0], [1.0, 0.0, 0.0], [1.0, 0.0, 0.0]], + unit='m', + ), + slow_axis=sc.vectors( + dims=['panel'], + values=[[0.0, 1.0, 0.0], [0.0, 1.0, 0.0], [0.0, 1.0, 0.0]], + unit='m', + ), + origin_position=sc.vectors( + dims=['panel'], + values=[[-0.2, 0.0, 0.0], [0.0, 0.0, 0.0], [0.2, 0.0, 0.0]], + unit='m', + ), + sample_position=sc.vector(value=[0.0, 0.0, 0.0], unit='m'), + source_position=sc.vector(value=[-3, 0.0, -4], unit='m'), + sample_name=sc.scalar('Unit Test Sample'), + position=sc.zeros(dims=['panel', 'id'], shape=[3, 4], unit='m'), + ) -@pytest.fixture -def mock_h5py(): - with mock.patch('ess.nmx.reduction.h5py') as mock_h5py: - yield mock_h5py +def test_mcstas_reduction_export_to_bytestream(reduced_data: NMXReducedData) -> None: + """Test export method.""" + import h5py + import numpy as np + import scipp as sc + + data_fields = [ + 'NXdetector', + 'NXsample', + 'NXsource', + 'NXinstrument', + 'definition', + 'name', + ] + + with io.BytesIO() as bio: + reduced_data.export_as_nexus(bio) + with h5py.File(bio, 'r') as f: + assert 'NMX_data' in f + nmx_data: h5py.Group = f.require_group('NMX_data') + for field in data_fields: + assert field in nmx_data + + nx_detector = nmx_data.require_group('NXdetector') + assert np.all(nx_detector['fast_axis'][()] == reduced_data.fast_axis.values) + assert np.all(nx_detector['slow_axis'][()] == reduced_data.slow_axis.values) + assert np.all( + nx_detector['origin'][()] == reduced_data.origin_position.values + ) + + instrument_data = nmx_data.require_group('NXinstrument') + assert ( + instrument_data['proton_charge'][()] == reduced_data.proton_charge.value + ) + + det1_data = instrument_data.require_group('detector_1') + assert np.all(det1_data['counts'][()] == reduced_data.counts.values) + assert np.all( + det1_data['pixel_id'][()] == reduced_data.counts.coords['id'].values + ) + assert np.all( + det1_data['t_bin'][()] == reduced_data.counts.coords['t'].values + ) + + nx_sample = nmx_data.require_group('NXsample') + sample_name: bytes = nx_sample['name'][()] + assert sample_name.decode() == reduced_data.sample_name.value -def test_exports(mock_h5py): - from ess.nmx.reduction import h5py + nx_source = nmx_data.require_group('NXsource') + assert ( + nx_source['distance'][()] == sc.norm(reduced_data.source_position).value + ) - fake_file = FakeFile() - mock_h5py.File.return_value.__enter__.return_value = fake_file - with h5py.File('fake_file', 'w') as f: - f.create_group('entry1') - assert f == {'entry1': {}} +def test_mcstas_reduction_export_wrong_extension_raises( + reduced_data: NMXReducedData, +) -> None: + """Test export method.""" + with pytest.raises(ValueError, match="Output file name must end with .h5 or .nxs"): + reduced_data.export_as_nexus('wrong_name.txt') From a2d51405ace2047a4189d58fefc4602820a66866 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 7 Feb 2024 14:04:47 +0100 Subject: [PATCH 061/403] Fix workflow example page. --- packages/essnmx/docs/examples/workflow.ipynb | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index 774ab94e..de9ce2be 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -29,7 +29,7 @@ " DefaultMaximumProbability\n", ")\n", "from ess.nmx.data import small_mcstas_sample\n", - "from ess.nmx.reduction import bin_time_of_arrival, NMXReducedData, TimeBinSteps\n", + "from ess.nmx.reduction import bin_time_of_arrival, TimeBinSteps\n", "\n", "providers = (load_mcstas_nexus, bin_time_of_arrival, )\n", "\n", @@ -137,7 +137,7 @@ "All pixel positions are relative to the sample position,\n", "therefore the sample is at (0, 0, 0).\n", "\n", - "The instrument view is shown using" + "**It might be very slow or not work in the ``VS Code`` jupyter notebook editor.**" ] }, { @@ -151,12 +151,9 @@ "da = dg['weights']\n", "da.coords['position'] = dg['position']\n", "# Plot one out of 100 pixels to reduce size of docs output\n", - "view = scn.instrument_view(da.flatten(['panel', 'id'], 'id')['id', ::100].hist(), pixel_size=0.0075)\n", + "view = scn.instrument_view(da['id', ::100].hist(), pixel_size=0.0075)\n", "view.children[0].toolbar.cameraz()\n", - "view\n", - "```\n", - "\n", - "**It might be very slow or not work in the ``VS Code`` jupyter notebook editor.**" + "view" ] } ], @@ -175,7 +172,8 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" + "pygments_lexer": "ipython3", + "version": "3.9.18" } }, "nbformat": 4, From ce9f8af973f7ffb96bec9ed8ab48f48e646beba5 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 7 Feb 2024 14:08:59 +0100 Subject: [PATCH 062/403] Add new types into API references. --- packages/essnmx/docs/api-reference/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/essnmx/docs/api-reference/index.md b/packages/essnmx/docs/api-reference/index.md index cd74eadf..8cf99737 100644 --- a/packages/essnmx/docs/api-reference/index.md +++ b/packages/essnmx/docs/api-reference/index.md @@ -11,7 +11,9 @@ :recursive: NMXData + NMXReducedData InputFilepath + TimeBinSteps ``` From 90bb36a6777f8709b4f0fc2ef5e4092f9d48d95d Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 7 Feb 2024 14:09:19 +0100 Subject: [PATCH 063/403] Fix docstring of exporting method. --- packages/essnmx/src/ess/nmx/reduction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 1f9b2c9f..31eaf7ec 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -214,7 +214,7 @@ def export_as_nexus( ) -> None: """Export the reduced data to a NeXus file. - Currently exporting step is not part of the sciline pipeline. + Currently exporting step is not expected to be part of sciline pipelines. """ if isinstance(output_file_base, (str, pathlib.Path)): file_base = pathlib.Path(output_file_base) From 7ab0ab72852907c01dde2ee7a508b215c91f1928 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 8 Feb 2024 14:26:15 +0100 Subject: [PATCH 064/403] Add McStas specific field manipulation functions as parameters. --- packages/essnmx/docs/examples/workflow.ipynb | 22 +++- packages/essnmx/src/ess/nmx/mcstas_loader.py | 101 ++++++++++++------- packages/essnmx/tests/loader_test.py | 21 +++- packages/essnmx/tests/workflow_test.py | 6 ++ 4 files changed, 112 insertions(+), 38 deletions(-) diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index de9ce2be..d5312a5b 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -26,7 +26,11 @@ "from ess.nmx.mcstas_loader import (\n", " InputFilepath,\n", " MaximumProbability,\n", - " DefaultMaximumProbability\n", + " DefaultMaximumProbability,\n", + " McStasEventWeightsConverter,\n", + " event_weights_from_probability,\n", + " McStasProtonChargeConverter,\n", + " proton_charge_from_event_data,\n", ")\n", "from ess.nmx.data import small_mcstas_sample\n", "from ess.nmx.reduction import bin_time_of_arrival, TimeBinSteps\n", @@ -39,9 +43,25 @@ " InputFilepath: InputFilepath(file_path),\n", " # Additional parameters for McStas data handling.\n", " MaximumProbability: DefaultMaximumProbability,\n", + " McStasEventWeightsConverter: event_weights_from_probability,\n", + " McStasProtonChargeConverter: proton_charge_from_event_data,\n", "}" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "``event weights converter`` and ``proton_charge_from_event_data`` are\n", + "\n", + "set as parameters for reproducibility of workflow and accessibility to the documentation.\n", + "\n", + "The reason of having them as parameters not as providers is,\n", + "\n", + "1. They are not part of general reduction, which are only for McStas cases.\n", + "2. They are better done while the file is open and read in the loader.\n" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 1ba836c4..6b7d7a8f 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) -from typing import Iterable, NewType, Optional +from typing import Callable, Iterable, NewType, Optional import scipp as sc import scippnexus as snx @@ -14,6 +14,17 @@ MaximumProbability = NewType("MaximumProbability", int) DefaultMaximumProbability = MaximumProbability(100_000) +ConvertedEventWeights = NewType("ConvertedEventWeights", sc.Variable) +McStasEventWeightsConverter = Callable[..., ConvertedEventWeights] +# It should be ``Callable[[MaximumProbability, sc.Variable], ConvertedEventWeights]`` +# but sciline pipeline breaks if there is a list in the type arguments. +McStasEventWeightsConverter.__name__ = "McStasEventWeightsConverter" + +ProtonCharge = NewType("ProtonCharge", sc.Variable) +McStasProtonChargeConverter = Callable[..., ProtonCharge] +# Should be ``Callable[[sc.DataArray], ProtonCharge]`` for the same reason as above. +McStasProtonChargeConverter.__name__ = "McStasProtonChargeConverter" + def _retrieve_event_list_name(keys: Iterable[str]) -> str: prefix = "bank01_events_dat_list" @@ -57,22 +68,42 @@ def _retrieve_crystal_rotation(file: snx.File, unit: str) -> sc.Variable: def event_weights_from_probability( + max_probability: MaximumProbability, probabilities: sc.Variable +) -> ConvertedEventWeights: + """Create event weights by scaling probability data. + + event_weights = max_probability * (probabilities / max(probabilities)) + + Parameters + ---------- + probabilities: + The probabilities of the events. + + max_probability: + The maximum probability to scale the weights. + + """ + maximum_probability = sc.scalar(max_probability, unit='counts') + + return ConvertedEventWeights( + maximum_probability * (probabilities / probabilities.max()) + ) + + +def _compose_event_data_array( *, - probabilities: sc.Variable, + weights: sc.Variable, id_list: sc.Variable, t_list: sc.Variable, pixel_ids: sc.Variable, - max_probability: sc.Variable, num_panels: int, ) -> sc.DataArray: - """Create event weights by scaling probability data. - - event_weights = max_probability * (probabilities / max(probabilities)) + """Combine data with coordinates loaded from the nexus file. Parameters ---------- - probabilities: - The probabilities of the events. + weights: + The weights of the events. id_list: The pixel IDs of the events. @@ -83,28 +114,18 @@ def event_weights_from_probability( pixel_ids: All possible pixel IDs of the detector. - max_probability: - The maximum probability to scale the weights. - num_panels: The number of (detector) panels used in the experiment. """ - if max_probability.unit != sc.units.counts: - raise ValueError("max_probability must have unit counts") - - weights = sc.DataArray( - # Scale the weights so that the weights are - # within the range of [0,``max_probability``]. - data=max_probability * (probabilities / probabilities.max()), - coords={'t': t_list, 'id': id_list}, - ) - grouped: sc.DataArray = weights.group(pixel_ids) + + events = sc.DataArray(data=weights, coords={'t': t_list, 'id': id_list}) + grouped: sc.DataArray = events.group(pixel_ids) return grouped.fold(dim='id', sizes={'panel': num_panels, 'id': -1}) -def proton_charge_from_weights(weights: sc.DataArray) -> sc.Variable: - """Make up the proton charge from the weights. +def proton_charge_from_event_data(event_da: sc.DataArray) -> ProtonCharge: + """Make up the proton charge from the event data array. Proton charge is proportional to the number of neutrons, which is proportional to the number of events. @@ -116,18 +137,20 @@ def proton_charge_from_weights(weights: sc.DataArray) -> sc.Variable: Parameters ---------- - weights: - The event weights binned in detector panel and pixel id dimensions. + event_da: + The event data binned in detector panel and pixel id dimensions. """ # Arbitrary number to scale the proton charge _proton_charge_scale_factor = sc.scalar(1 / 10_000, unit=None) - return _proton_charge_scale_factor * weights.bins.size().sum().data + return ProtonCharge(_proton_charge_scale_factor * event_da.bins.size().sum().data) def load_mcstas_nexus( file_path: InputFilepath, + event_weights_converter: McStasEventWeightsConverter, + proton_charge_converter: McStasProtonChargeConverter, max_probability: Optional[MaximumProbability] = None, ) -> NMXData: """Load McStas simulation result from h5(nexus) file. @@ -140,6 +163,16 @@ def load_mcstas_nexus( file_path: File name to load. + event_weights_converter: + A function to convert probabilities to event weights. + The function should accept the probabilities as the first argument, + and return the converted event weights. + + proton_charge_converter: + A function to convert the event weights to proton charge. + The function should accept the event weights as the first argument, + and return the proton charge. + max_probability: The maximum probability to scale the weights. If not provided, ``DefaultMaximumProbability`` is used. @@ -150,27 +183,27 @@ def load_mcstas_nexus( geometry = read_mcstas_geometry_xml(file_path) coords = geometry.to_coords() - maximum_probability = sc.scalar( - max_probability or DefaultMaximumProbability, unit='counts' - ) with snx.File(file_path) as file: raw_data = _retrieve_raw_event_data(file) - weights = event_weights_from_probability( - probabilities=_copy_partial_var(raw_data, idx=0, unit='counts'), # p - max_probability=maximum_probability, + weights = event_weights_converter( + max_probability or DefaultMaximumProbability, + _copy_partial_var(raw_data, idx=0, unit='counts'), # p + ) + event_da = _compose_event_data_array( + weights=weights, id_list=_copy_partial_var(raw_data, idx=4, dtype='int64'), # id t_list=_copy_partial_var(raw_data, idx=5, unit='s'), # t pixel_ids=coords.pop('pixel_id'), num_panels=len(geometry.detectors), ) - proton_charge = proton_charge_from_weights(weights) + proton_charge = proton_charge_converter(event_da) crystal_rotation = _retrieve_crystal_rotation( file, geometry.simulation_settings.angle_unit ) return NMXData( - weights=weights, + weights=event_da, proton_charge=proton_charge, crystal_rotation=crystal_rotation, **coords, diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index c748eeb2..8d47c254 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -32,7 +32,9 @@ def test_file_reader_mcstas() -> None: from ess.nmx.mcstas_loader import ( DefaultMaximumProbability, InputFilepath, + event_weights_from_probability, load_mcstas_nexus, + proton_charge_from_event_data, ) file_path = InputFilepath(small_mcstas_sample()) @@ -41,7 +43,11 @@ def test_file_reader_mcstas() -> None: raw_data = file[entry_path]["events"][()] data_length = raw_data.sizes['dim_0'] - dg = load_mcstas_nexus(file_path) + dg = load_mcstas_nexus( + file_path=file_path, + event_weights_converter=event_weights_from_probability, + proton_charge_converter=proton_charge_from_event_data, + ) assert isinstance(dg, sc.DataGroup) assert dg.shape == (3, 1280 * 1280) check_scalar_properties(dg) @@ -83,7 +89,12 @@ def test_file_reader_mcstas_additional_fields(tmp_mcstas_file: pathlib.Path) -> """Check if additional fields names do not break the loader.""" import h5py - from ess.nmx.mcstas_loader import InputFilepath, load_mcstas_nexus + from ess.nmx.mcstas_loader import ( + InputFilepath, + event_weights_from_probability, + load_mcstas_nexus, + proton_charge_from_event_data, + ) entry_path = "entry1/data/bank01_events_dat_list_p_x_y_n_id_t" new_entry_path = entry_path + '_L' @@ -93,7 +104,11 @@ def test_file_reader_mcstas_additional_fields(tmp_mcstas_file: pathlib.Path) -> del file[entry_path] file[new_entry_path] = dataset - dg = load_mcstas_nexus(InputFilepath(str(tmp_mcstas_file))) + dg = load_mcstas_nexus( + file_path=InputFilepath(str(tmp_mcstas_file)), + event_weights_converter=event_weights_from_probability, + proton_charge_converter=proton_charge_from_event_data, + ) assert isinstance(dg, sc.DataGroup) assert dg.shape == (3, 1280 * 1280) diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index 3eaa9dc5..65b6e34b 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -12,7 +12,11 @@ def mcstas_workflow() -> sl.Pipeline: DefaultMaximumProbability, InputFilepath, MaximumProbability, + McStasEventWeightsConverter, + McStasProtonChargeConverter, + event_weights_from_probability, load_mcstas_nexus, + proton_charge_from_event_data, ) from ess.nmx.reduction import TimeBinSteps, bin_time_of_arrival @@ -22,6 +26,8 @@ def mcstas_workflow() -> sl.Pipeline: InputFilepath: small_mcstas_sample(), MaximumProbability: DefaultMaximumProbability, TimeBinSteps: TimeBinSteps(50), + McStasEventWeightsConverter: event_weights_from_probability, + McStasProtonChargeConverter: proton_charge_from_event_data, }, ) From 7b4dbbd91ccf8033bece2a3080da62cbe0172f43 Mon Sep 17 00:00:00 2001 From: Justin-Bergmann <48309261+Justin-Bergmann@users.noreply.github.com> Date: Fri, 9 Feb 2024 10:40:35 +0100 Subject: [PATCH 065/403] Update work_flow_design.md documentation of data reduction workflow --- .../essnmx/docs/about/work_flow_design.md | 81 +++++++++++++++++-- 1 file changed, 75 insertions(+), 6 deletions(-) diff --git a/packages/essnmx/docs/about/work_flow_design.md b/packages/essnmx/docs/about/work_flow_design.md index 903666fd..6a4137bf 100644 --- a/packages/essnmx/docs/about/work_flow_design.md +++ b/packages/essnmx/docs/about/work_flow_design.md @@ -3,18 +3,87 @@ This is a description of the data workflow for the NMX instrument at ESS. The [NMX](https://europeanspallationsource.se/instruments/nmx) Macromolecular Diffractometer is a time-of-flight (TOF) quasi-Laue diffractometer optimised for small samples and large unit cells dedicated to the structure determination of biological macromolecules by crystallography. -The main scientific driver is to locate the hydrogen atoms relevant for the function of the macromolecule. +The main scientific driver is to locate the hydrogen atoms relevant to the function of the macromolecule. ## Data reduction ![work_flow](https://github.com/scipp/essnmx/new/main/docs/about/NMX_work_flow.png) -### From single event data to binned image like data -From single event data to binned image like data -The first step in the data reduction is to reduce the data from single event data to image like data. +### From single event data to binned image-like data +From single event data to binned image-like data +The first step in the data reduction is to reduce the data from single event data to image-like data. Therefore the [essNMX](https://github.com/scipp/essnmx) package is used. -Tirst the time of arrival (TOA) is converted into time of flight (TOF). +First, the time of arrival (TOA) is converted into time of flight (TOF). Then the single events get binned into pixels and then histogramed in the TOF dimension. -These data will be written be added with some meta and instrument data to an HDF5 file. +These data will be written be added with some meta and instrument data to an HDF5 file. + +### Spot finding and integration +For the next five steps of the data reduction from spot finding to spot integration, we use the [programme](https://dials.github.io/index.html) [DIALS](https://onlinelibrary.wiley.com/doi/10.1002/pro.4224) + +First, we use [dials.import](https://dials.github.io/documentation/programs/dials_import.html) to convert image data files into a format compatible with dials. It the metadata and filenames of each image to establish relationships between different sets of images. Once all images are processed, the program generates an experiment object file, outlining the connections between the files. The images to be processed are designated as command-line arguments. Occasionally, there may be a restriction on the maximum number of arguments allowed on the command line, and the number of files could surpass this limit. In such cases, image filenames can be entered through stdin, as demonstrated in the examples below. +The Format class for NMX is at modules/dxtbx/src/dxtbx/format/FormatNMX.py where beam-line-specific parameters and file format information are stored. + +```consol +dials.import *.nxs +``` + +In the next step, a [search for strong pixel](https://dials.github.io/documentation/programs/dials_find_spots.html) is performed. Therefore the intensity of a pixel or pixel group is compared with its local surroundings. With the information of strong pixels, strong spots are defined. for these spots, the centroids and intensities will be calculated. the results can be visualised in the image viewer or the [browser](https://toastisme.github.io/dials_browser_experiment_viewer/) + +```consol +dials.find_spots imported.expt find_spots.phil +``` + +In the [indexing](https://dials.github.io/documentation/programs/dials_index.html) step the unit cell is determined. a list of indexed reflexes and an instrument model including a crystal model is returned. One-dimensional and three-dimensional fast Fourier transform-based methods are available. + +As input parameters the imported.exp and strong.refl files are used. more parameters such as unit cell and spacegroup can be given. + +```consol +dials.index imported.expt strong.refl space_group=P1 unit_cell=a,b,c,alpha,beta,gamma +``` + + + +After indexing the instrument geometry is getting [refined](https://dials.github.io/documentation/programs/dials_refine.html). +```consol +dials.refine indexed.refl indexed.expt detector.panels=hierarchical +``` + +The last step in DIALS is to integrate(https://dials.github.io/documentation/programs/dials_integrate.html) each reflex. Currently, in the dimension of the image, a simple summation is used and in the TOF dimension, a profile-fitting approach is used. + +```consol +dev.dials.simple_tof_integrate refined.expt refined.refl +``` + + + + +### Scaling +Currently [LSCALE](https://scripts.iucr.org/cgi-bin/paper?S0021889898015350) can be used in a docker container which makes it indented from the OS. The source code is available on [Zenodo](https://zenodo.org/records/4381992). LSCALE is a program for scaling and normalisation of Laue intensity data. +Since LSCALE is not maintained anymore we currently develop a Python-based [alternative](https://github.com/mlund/pyscale) to LSCALE. + +start docker desktop +```consol +docker run -it -v $HOME:/mnt/host -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=host.docker.internal:0 lscale +``` +```consol +lscale < lscale.com > lscale.out +``` + +### AIMLESS and CTRUNCATE + +[AIMLESS](https://www.ccp4.ac.uk/html/aimless.html) is used to scale multiple observations of reflections together, and merges multiple observations into an average intensity. + + +[CTRUNCATE](https://www.ccp4.ac.uk/html/ctruncate.html) converts measured intensities into structure factors. CTRUNCATE includes corrections for weak reflections to avoid negative intensities due to background corrections. + +```consol +Start CCP4 GUI +go to all programs +select Aimless +select scaled *.mtz file +``` +usually, standard parameters are fine but parameters can be modified. + +This results in a final *mtz file which can be used in a standard protein crystallographic program to solve and refine the structure. From 9e38e4ebbf9faf1b82c487009a19656c45972847 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Fri, 9 Feb 2024 10:54:42 +0100 Subject: [PATCH 066/403] Copier update. --- packages/essnmx/.copier-answers.yml | 2 +- packages/essnmx/.github/workflows/ci.yml | 11 ++++++----- packages/essnmx/.gitignore | 1 + packages/essnmx/docs/conf.py | 16 ++++++++++++++++ packages/essnmx/pyproject.toml | 1 + packages/essnmx/tox.ini | 2 +- 6 files changed, 26 insertions(+), 7 deletions(-) diff --git a/packages/essnmx/.copier-answers.yml b/packages/essnmx/.copier-answers.yml index 4a0801f7..f4cc884b 100644 --- a/packages/essnmx/.copier-answers.yml +++ b/packages/essnmx/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY -_commit: 8cfdb1b +_commit: 1abe96f _src_path: gh:scipp/copier_template description: Data reduction for NMX at the European Spallation Source. max_python: '3.11' diff --git a/packages/essnmx/.github/workflows/ci.yml b/packages/essnmx/.github/workflows/ci.yml index 20c647dc..497ab895 100644 --- a/packages/essnmx/.github/workflows/ci.yml +++ b/packages/essnmx/.github/workflows/ci.yml @@ -27,12 +27,13 @@ jobs: - uses: actions/setup-python@v4 with: python-version-file: '.github/workflows/python-version-ci' - - run: python -m pip install --upgrade pip - - run: python -m pip install -r requirements/ci.txt - - run: tox -e static - - uses: stefanzweifel/git-auto-commit-action@v5 + - uses: pre-commit/action@v3.0.1 with: - commit_message: Apply automatic formatting + extra_args: --all-files + - uses: pre-commit-ci/lite-action@v1.0.1 + if: always() + with: + msg: Apply automatic formatting tests: name: Tests diff --git a/packages/essnmx/.gitignore b/packages/essnmx/.gitignore index e8b1ec78..0f0541bc 100644 --- a/packages/essnmx/.gitignore +++ b/packages/essnmx/.gitignore @@ -37,3 +37,4 @@ docs/generated/ *.raw *.cif *.rcif +*.ort diff --git a/packages/essnmx/docs/conf.py b/packages/essnmx/docs/conf.py index 276bf231..8aae470a 100644 --- a/packages/essnmx/docs/conf.py +++ b/packages/essnmx/docs/conf.py @@ -29,6 +29,11 @@ 'nbsphinx', 'myst_parser', ] +try: + import sciline.sphinxext.domain_types # noqa: F401 + extensions.append('sciline.sphinxext.domain_types') +except ModuleNotFoundError: + pass myst_enable_extensions = [ "amsmath", @@ -54,6 +59,7 @@ intersphinx_mapping = { 'python': ('https://docs.python.org/3', None), 'numpy': ('https://numpy.org/doc/stable/', None), + 'scipp': ('https://scipp.github.io/', None), } # autodocs includes everything, even irrelevant API internals. autosummary @@ -73,6 +79,16 @@ typehints_defaults = 'comma' typehints_use_rtype = False +sciline_domain_types_prefix = 'ess.nmx' +sciline_domain_types_aliases = { + 'scipp._scipp.core.DataArray': 'scipp.DataArray', + 'scipp._scipp.core.Dataset': 'scipp.Dataset', + 'scipp._scipp.core.DType': 'scipp.DType', + 'scipp._scipp.core.Unit': 'scipp.Unit', + 'scipp._scipp.core.Variable': 'scipp.Variable', + 'scipp.core.data_group.DataGroup': 'scipp.DataGroup', +} + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 79092497..8052920f 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -54,6 +54,7 @@ minversion = "7.0" addopts = """ --strict-config --strict-markers +--import-mode=importlib -ra -v """ diff --git a/packages/essnmx/tox.ini b/packages/essnmx/tox.ini index 4eaad04e..df80a9d1 100644 --- a/packages/essnmx/tox.ini +++ b/packages/essnmx/tox.ini @@ -15,8 +15,8 @@ commands = pytest [testenv:unpinned] description = Test with unpinned dependencies, as a user would install now. deps = + -r requirements/basetest.txt essnmx - pytest commands = pytest [testenv:docs] From 330b07c74c38327e0682db4893c3675c063c429a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Fri, 9 Feb 2024 09:57:52 +0000 Subject: [PATCH 067/403] Apply automatic formatting --- packages/essnmx/docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/essnmx/docs/conf.py b/packages/essnmx/docs/conf.py index 8aae470a..9d4285fe 100644 --- a/packages/essnmx/docs/conf.py +++ b/packages/essnmx/docs/conf.py @@ -31,6 +31,7 @@ ] try: import sciline.sphinxext.domain_types # noqa: F401 + extensions.append('sciline.sphinxext.domain_types') except ModuleNotFoundError: pass From 6dbadbe9f8ddd2039449f0fca2f829434d2456e9 Mon Sep 17 00:00:00 2001 From: Justin-Bergmann <48309261+Justin-Bergmann@users.noreply.github.com> Date: Fri, 9 Feb 2024 12:47:41 +0100 Subject: [PATCH 068/403] Update work_flow_design.md --- packages/essnmx/docs/about/work_flow_design.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/essnmx/docs/about/work_flow_design.md b/packages/essnmx/docs/about/work_flow_design.md index 6a4137bf..28b9c111 100644 --- a/packages/essnmx/docs/about/work_flow_design.md +++ b/packages/essnmx/docs/about/work_flow_design.md @@ -25,13 +25,13 @@ For the next five steps of the data reduction from spot finding to spot integrat First, we use [dials.import](https://dials.github.io/documentation/programs/dials_import.html) to convert image data files into a format compatible with dials. It the metadata and filenames of each image to establish relationships between different sets of images. Once all images are processed, the program generates an experiment object file, outlining the connections between the files. The images to be processed are designated as command-line arguments. Occasionally, there may be a restriction on the maximum number of arguments allowed on the command line, and the number of files could surpass this limit. In such cases, image filenames can be entered through stdin, as demonstrated in the examples below. The Format class for NMX is at modules/dxtbx/src/dxtbx/format/FormatNMX.py where beam-line-specific parameters and file format information are stored. -```consol +```console dials.import *.nxs ``` In the next step, a [search for strong pixel](https://dials.github.io/documentation/programs/dials_find_spots.html) is performed. Therefore the intensity of a pixel or pixel group is compared with its local surroundings. With the information of strong pixels, strong spots are defined. for these spots, the centroids and intensities will be calculated. the results can be visualised in the image viewer or the [browser](https://toastisme.github.io/dials_browser_experiment_viewer/) -```consol +```console dials.find_spots imported.expt find_spots.phil ``` @@ -39,20 +39,20 @@ In the [indexing](https://dials.github.io/documentation/programs/dials_index.htm As input parameters the imported.exp and strong.refl files are used. more parameters such as unit cell and spacegroup can be given. -```consol +```console dials.index imported.expt strong.refl space_group=P1 unit_cell=a,b,c,alpha,beta,gamma ``` After indexing the instrument geometry is getting [refined](https://dials.github.io/documentation/programs/dials_refine.html). -```consol +```console dials.refine indexed.refl indexed.expt detector.panels=hierarchical ``` The last step in DIALS is to integrate(https://dials.github.io/documentation/programs/dials_integrate.html) each reflex. Currently, in the dimension of the image, a simple summation is used and in the TOF dimension, a profile-fitting approach is used. -```consol +```console dev.dials.simple_tof_integrate refined.expt refined.refl ``` @@ -64,10 +64,10 @@ Currently [LSCALE](https://scripts.iucr.org/cgi-bin/paper?S0021889898015350) can Since LSCALE is not maintained anymore we currently develop a Python-based [alternative](https://github.com/mlund/pyscale) to LSCALE. start docker desktop -```consol +```console docker run -it -v $HOME:/mnt/host -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=host.docker.internal:0 lscale ``` -```consol +```console lscale < lscale.com > lscale.out ``` @@ -78,7 +78,7 @@ lscale < lscale.com > lscale.out [CTRUNCATE](https://www.ccp4.ac.uk/html/ctruncate.html) converts measured intensities into structure factors. CTRUNCATE includes corrections for weak reflections to avoid negative intensities due to background corrections. -```consol +```console Start CCP4 GUI go to all programs select Aimless From 5778a0c6e5d4ca5c66ec04165af5d942a8f3d1d0 Mon Sep 17 00:00:00 2001 From: Justin-Bergmann Date: Fri, 9 Feb 2024 11:48:39 +0000 Subject: [PATCH 069/403] Apply automatic formatting --- packages/essnmx/docs/about/work_flow_design.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/essnmx/docs/about/work_flow_design.md b/packages/essnmx/docs/about/work_flow_design.md index 28b9c111..ffc274a0 100644 --- a/packages/essnmx/docs/about/work_flow_design.md +++ b/packages/essnmx/docs/about/work_flow_design.md @@ -2,7 +2,7 @@ This is a description of the data workflow for the NMX instrument at ESS. -The [NMX](https://europeanspallationsource.se/instruments/nmx) Macromolecular Diffractometer is a time-of-flight (TOF) quasi-Laue diffractometer optimised for small samples and large unit cells dedicated to the structure determination of biological macromolecules by crystallography. +The [NMX](https://europeanspallationsource.se/instruments/nmx) Macromolecular Diffractometer is a time-of-flight (TOF) quasi-Laue diffractometer optimised for small samples and large unit cells dedicated to the structure determination of biological macromolecules by crystallography. The main scientific driver is to locate the hydrogen atoms relevant to the function of the macromolecule. ## Data reduction @@ -13,16 +13,16 @@ The main scientific driver is to locate the hydrogen atoms relevant to the funct ### From single event data to binned image-like data From single event data to binned image-like data The first step in the data reduction is to reduce the data from single event data to image-like data. -Therefore the [essNMX](https://github.com/scipp/essnmx) package is used. +Therefore the [essNMX](https://github.com/scipp/essnmx) package is used. First, the time of arrival (TOA) is converted into time of flight (TOF). -Then the single events get binned into pixels and then histogramed in the TOF dimension. +Then the single events get binned into pixels and then histogramed in the TOF dimension. These data will be written be added with some meta and instrument data to an HDF5 file. ### Spot finding and integration -For the next five steps of the data reduction from spot finding to spot integration, we use the [programme](https://dials.github.io/index.html) [DIALS](https://onlinelibrary.wiley.com/doi/10.1002/pro.4224) +For the next five steps of the data reduction from spot finding to spot integration, we use the [programme](https://dials.github.io/index.html) [DIALS](https://onlinelibrary.wiley.com/doi/10.1002/pro.4224) -First, we use [dials.import](https://dials.github.io/documentation/programs/dials_import.html) to convert image data files into a format compatible with dials. It the metadata and filenames of each image to establish relationships between different sets of images. Once all images are processed, the program generates an experiment object file, outlining the connections between the files. The images to be processed are designated as command-line arguments. Occasionally, there may be a restriction on the maximum number of arguments allowed on the command line, and the number of files could surpass this limit. In such cases, image filenames can be entered through stdin, as demonstrated in the examples below. +First, we use [dials.import](https://dials.github.io/documentation/programs/dials_import.html) to convert image data files into a format compatible with dials. It the metadata and filenames of each image to establish relationships between different sets of images. Once all images are processed, the program generates an experiment object file, outlining the connections between the files. The images to be processed are designated as command-line arguments. Occasionally, there may be a restriction on the maximum number of arguments allowed on the command line, and the number of files could surpass this limit. In such cases, image filenames can be entered through stdin, as demonstrated in the examples below. The Format class for NMX is at modules/dxtbx/src/dxtbx/format/FormatNMX.py where beam-line-specific parameters and file format information are stored. ```console @@ -60,7 +60,7 @@ dev.dials.simple_tof_integrate refined.expt refined.refl ### Scaling -Currently [LSCALE](https://scripts.iucr.org/cgi-bin/paper?S0021889898015350) can be used in a docker container which makes it indented from the OS. The source code is available on [Zenodo](https://zenodo.org/records/4381992). LSCALE is a program for scaling and normalisation of Laue intensity data. +Currently [LSCALE](https://scripts.iucr.org/cgi-bin/paper?S0021889898015350) can be used in a docker container which makes it indented from the OS. The source code is available on [Zenodo](https://zenodo.org/records/4381992). LSCALE is a program for scaling and normalisation of Laue intensity data. Since LSCALE is not maintained anymore we currently develop a Python-based [alternative](https://github.com/mlund/pyscale) to LSCALE. start docker desktop @@ -76,7 +76,7 @@ lscale < lscale.com > lscale.out [AIMLESS](https://www.ccp4.ac.uk/html/aimless.html) is used to scale multiple observations of reflections together, and merges multiple observations into an average intensity. -[CTRUNCATE](https://www.ccp4.ac.uk/html/ctruncate.html) converts measured intensities into structure factors. CTRUNCATE includes corrections for weak reflections to avoid negative intensities due to background corrections. +[CTRUNCATE](https://www.ccp4.ac.uk/html/ctruncate.html) converts measured intensities into structure factors. CTRUNCATE includes corrections for weak reflections to avoid negative intensities due to background corrections. ```console Start CCP4 GUI From 53e62d43804cbde5857264b0c3be74e67ef9b1bf Mon Sep 17 00:00:00 2001 From: Justin-Bergmann <48309261+Justin-Bergmann@users.noreply.github.com> Date: Fri, 9 Feb 2024 12:56:31 +0100 Subject: [PATCH 070/403] Update docs/about/work_flow_design.md Co-authored-by: Sunyoung Yoo --- packages/essnmx/docs/about/work_flow_design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/docs/about/work_flow_design.md b/packages/essnmx/docs/about/work_flow_design.md index ffc274a0..d33868d9 100644 --- a/packages/essnmx/docs/about/work_flow_design.md +++ b/packages/essnmx/docs/about/work_flow_design.md @@ -7,7 +7,7 @@ The main scientific driver is to locate the hydrogen atoms relevant to the funct ## Data reduction -![work_flow](https://github.com/scipp/essnmx/new/main/docs/about/NMX_work_flow.png) +![work_flow][NMX_work_flow.png] ### From single event data to binned image-like data From 54cf4e2c410ea2fb223418de26fccb75786af76a Mon Sep 17 00:00:00 2001 From: Justin-Bergmann <48309261+Justin-Bergmann@users.noreply.github.com> Date: Fri, 9 Feb 2024 12:58:14 +0100 Subject: [PATCH 071/403] Update work_flow_design.md --- packages/essnmx/docs/about/work_flow_design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/docs/about/work_flow_design.md b/packages/essnmx/docs/about/work_flow_design.md index d33868d9..dcb533d7 100644 --- a/packages/essnmx/docs/about/work_flow_design.md +++ b/packages/essnmx/docs/about/work_flow_design.md @@ -7,7 +7,7 @@ The main scientific driver is to locate the hydrogen atoms relevant to the funct ## Data reduction -![work_flow][NMX_work_flow.png] +![work_flow](NMX_work_flow.png) ### From single event data to binned image-like data From e7c76f576a6b5f318b526ba4ecdda566886221f4 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Mon, 12 Feb 2024 11:51:43 +0100 Subject: [PATCH 072/403] Update new type object, rename type alias and add default values in the loader. --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 37 +++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 6b7d7a8f..f21d5308 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -14,16 +14,19 @@ MaximumProbability = NewType("MaximumProbability", int) DefaultMaximumProbability = MaximumProbability(100_000) -ConvertedEventWeights = NewType("ConvertedEventWeights", sc.Variable) -McStasEventWeightsConverter = Callable[..., ConvertedEventWeights] -# It should be ``Callable[[MaximumProbability, sc.Variable], ConvertedEventWeights]`` -# but sciline pipeline breaks if there is a list in the type arguments. -McStasEventWeightsConverter.__name__ = "McStasEventWeightsConverter" +McStasEventProbabilities = NewType("McStasEventProbabilities", sc.Variable) +EventWeights = NewType("EventWeights", sc.Variable) +EventWeightsConverter = NewType( + "EventWeightsConverter", + Callable[[MaximumProbability, McStasEventProbabilities], EventWeights], +) +"""A function that converts McStas probability to event weights.""" ProtonCharge = NewType("ProtonCharge", sc.Variable) -McStasProtonChargeConverter = Callable[..., ProtonCharge] -# Should be ``Callable[[sc.DataArray], ProtonCharge]`` for the same reason as above. -McStasProtonChargeConverter.__name__ = "McStasProtonChargeConverter" +ProtonChargeConverter = NewType( + "ProtonChargeConverter", Callable[[EventWeights], ProtonCharge] +) +"""A function that derives arbitrary proton charge based on event weights.""" def _retrieve_event_list_name(keys: Iterable[str]) -> str: @@ -68,8 +71,8 @@ def _retrieve_crystal_rotation(file: snx.File, unit: str) -> sc.Variable: def event_weights_from_probability( - max_probability: MaximumProbability, probabilities: sc.Variable -) -> ConvertedEventWeights: + max_probability: MaximumProbability, probabilities: McStasEventProbabilities +) -> EventWeights: """Create event weights by scaling probability data. event_weights = max_probability * (probabilities / max(probabilities)) @@ -85,9 +88,7 @@ def event_weights_from_probability( """ maximum_probability = sc.scalar(max_probability, unit='counts') - return ConvertedEventWeights( - maximum_probability * (probabilities / probabilities.max()) - ) + return EventWeights(maximum_probability * (probabilities / probabilities.max())) def _compose_event_data_array( @@ -149,14 +150,14 @@ def proton_charge_from_event_data(event_da: sc.DataArray) -> ProtonCharge: def load_mcstas_nexus( file_path: InputFilepath, - event_weights_converter: McStasEventWeightsConverter, - proton_charge_converter: McStasProtonChargeConverter, + event_weights_converter: EventWeightsConverter = event_weights_from_probability, + proton_charge_converter: ProtonChargeConverter = proton_charge_from_event_data, max_probability: Optional[MaximumProbability] = None, ) -> NMXData: """Load McStas simulation result from h5(nexus) file. See :func:`~event_weights_from_probability` and - :func:`~proton_charge_from_weights` for details. + :func:`~proton_charge_from_event_data` for details. Parameters ---------- @@ -188,7 +189,9 @@ def load_mcstas_nexus( raw_data = _retrieve_raw_event_data(file) weights = event_weights_converter( max_probability or DefaultMaximumProbability, - _copy_partial_var(raw_data, idx=0, unit='counts'), # p + McStasEventProbabilities( + _copy_partial_var(raw_data, idx=0, unit='counts') + ), # p ) event_da = _compose_event_data_array( weights=weights, From 6819d54127743cdff4d88dceb77905a08c80579a Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Mon, 12 Feb 2024 11:53:55 +0100 Subject: [PATCH 073/403] Rename type alias. --- packages/essnmx/docs/examples/workflow.ipynb | 8 ++++---- packages/essnmx/tests/workflow_test.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index d5312a5b..b359dbd7 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -27,9 +27,9 @@ " InputFilepath,\n", " MaximumProbability,\n", " DefaultMaximumProbability,\n", - " McStasEventWeightsConverter,\n", + " EventWeightsConverter,\n", " event_weights_from_probability,\n", - " McStasProtonChargeConverter,\n", + " ProtonChargeConverter,\n", " proton_charge_from_event_data,\n", ")\n", "from ess.nmx.data import small_mcstas_sample\n", @@ -43,8 +43,8 @@ " InputFilepath: InputFilepath(file_path),\n", " # Additional parameters for McStas data handling.\n", " MaximumProbability: DefaultMaximumProbability,\n", - " McStasEventWeightsConverter: event_weights_from_probability,\n", - " McStasProtonChargeConverter: proton_charge_from_event_data,\n", + " EventWeightsConverter: event_weights_from_probability,\n", + " ProtonChargeConverter: proton_charge_from_event_data,\n", "}" ] }, diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index 65b6e34b..e30866e7 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -10,10 +10,10 @@ def mcstas_workflow() -> sl.Pipeline: from ess.nmx.data import small_mcstas_sample from ess.nmx.mcstas_loader import ( DefaultMaximumProbability, + EventWeightsConverter, InputFilepath, MaximumProbability, - McStasEventWeightsConverter, - McStasProtonChargeConverter, + ProtonChargeConverter, event_weights_from_probability, load_mcstas_nexus, proton_charge_from_event_data, @@ -26,8 +26,8 @@ def mcstas_workflow() -> sl.Pipeline: InputFilepath: small_mcstas_sample(), MaximumProbability: DefaultMaximumProbability, TimeBinSteps: TimeBinSteps(50), - McStasEventWeightsConverter: event_weights_from_probability, - McStasProtonChargeConverter: proton_charge_from_event_data, + EventWeightsConverter: event_weights_from_probability, + ProtonChargeConverter: proton_charge_from_event_data, }, ) From 11e92500f01846cdebe6b47beb872a75c665078f Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Mon, 19 Feb 2024 14:49:08 +0100 Subject: [PATCH 074/403] ci: use ess template --- packages/essnmx/.copier-answers.ess.yml | 3 + .../ISSUE_TEMPLATE/high-level-requirement.yml | 89 +++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 packages/essnmx/.copier-answers.ess.yml create mode 100644 packages/essnmx/.github/ISSUE_TEMPLATE/high-level-requirement.yml diff --git a/packages/essnmx/.copier-answers.ess.yml b/packages/essnmx/.copier-answers.ess.yml new file mode 100644 index 00000000..1ca4397b --- /dev/null +++ b/packages/essnmx/.copier-answers.ess.yml @@ -0,0 +1,3 @@ +# Changes here will be overwritten by Copier; NEVER EDIT MANUALLY +_commit: 815268a +_src_path: https://github.com/scipp/ess_template diff --git a/packages/essnmx/.github/ISSUE_TEMPLATE/high-level-requirement.yml b/packages/essnmx/.github/ISSUE_TEMPLATE/high-level-requirement.yml new file mode 100644 index 00000000..cedef1d9 --- /dev/null +++ b/packages/essnmx/.github/ISSUE_TEMPLATE/high-level-requirement.yml @@ -0,0 +1,89 @@ +name: High-level requirement +description: Describe a high-level requirement +title: "[Requirement] " +labels: ["requirement"] +projects: [] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to provide as many details as possible for this requirement! + - type: input + id: summary + attributes: + label: Executive summary + description: Provide a short summary of the requirement + placeholder: "Example: We need to correct for X when processing Y." + validations: + required: true + - type: textarea + id: context + attributes: + label: Context and background knowledge + description: | + - What is the context of this requirement? + - What background knowledge is required to understand it? + - Does this depend on previous tasks? Provide links! + - Is there follow-up work? + placeholder: "Example: See summary on Wikipedia, or the following paper." + validations: + required: true + - type: textarea + id: inputs + attributes: + label: Inputs + description: | + Describe in detail all the input data and data properties that are known. + This is not about test data (see below), but about general properties of data that will be used in practice. + placeholder: "Example: A single 1-D spectrum with a known wavelength range." + validations: + required: true + - type: textarea + id: methodology + attributes: + label: Methodology + description: | + Describe, e.g., the computation to be performed. + When linking to references, please refer to the specific section, page, or equation. + placeholder: "Remember you can write equations such as $n\\lambda = 2d\\sin(\\theta)$ using LaTeX syntax, as well as other Markdown formatting." + validations: + required: true + - type: textarea + id: outputs + attributes: + label: Outputs + description: | + Describe in detail all the output data and data properties. + This is not about test data (see below), but about general properties of data that will be used in practice. + placeholder: "Example: The position of the peak in the spectrum." + validations: + required: true + - type: dropdown + id: interfaces + attributes: + label: Which interfaces are required? + multiple: true + options: + - Integrated into reduction workflow + - Python module / function + - Python script + - Jupyter notebook + - Other (please describe in comments) + default: 0 + validations: + required: true + - type: textarea + id: testcases + attributes: + label: Test cases + description: How can we test this requirement? Links to tests data and reference data, or other suggestions. + validations: + required: true + - type: textarea + id: comments + attributes: + label: Comments + description: Do you have other comments that do not fall in the above categories? + placeholder: "Example: Depends on issues #1234, blocked by #666." + validations: + required: false From cc2eae2744ee09ad693287bc8f54585a87bb1bd7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Feb 2024 17:08:18 +0000 Subject: [PATCH 075/403] Bump scipp from 23.12.0 to 24.2.0 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 23.12.0 to 24.2.0. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/23.12.0...24.02.0) --- updated-dependencies: - dependency-name: scipp dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index fb39acdd..0607120a 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -41,8 +41,6 @@ idna==3.6 # via requests importlib-metadata==7.0.1 # via dask -importlib-resources==6.1.1 - # via matplotlib ipython==8.9.0 # via -r base.in jedi==0.19.1 @@ -105,7 +103,7 @@ requests==2.31.0 # via pooch sciline==24.1.0 # via -r base.in -scipp==23.12.0 +scipp==24.2.0 # via # -r base.in # scippnexus @@ -132,6 +130,4 @@ urllib3==2.1.0 wcwidth==0.2.12 # via prompt-toolkit zipp==3.17.0 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata From 0dd974bd73e22f4d1429dc7ab99cb316e667ed99 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Mon, 19 Feb 2024 10:50:58 +0100 Subject: [PATCH 076/403] Update dependencies. --- packages/essnmx/requirements/base.txt | 28 ++++++------ packages/essnmx/requirements/basetest.txt | 4 +- packages/essnmx/requirements/ci.txt | 12 ++--- packages/essnmx/requirements/dev.txt | 46 +++++++++++-------- packages/essnmx/requirements/docs.txt | 55 +++++++++++------------ packages/essnmx/requirements/nightly.txt | 20 ++++----- packages/essnmx/requirements/static.txt | 6 +-- 7 files changed, 88 insertions(+), 83 deletions(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 0607120a..81414222 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -9,7 +9,7 @@ asttokens==2.4.1 # via stack-data backcall==0.2.0 # via ipython -certifi==2023.11.17 +certifi==2024.2.2 # via requests charset-normalizer==3.3.2 # via requests @@ -21,7 +21,7 @@ contourpy==1.2.0 # via matplotlib cycler==0.12.1 # via matplotlib -dask==2023.12.1 +dask==2024.2.0 # via -r base.in decorator==5.1.1 # via ipython @@ -29,9 +29,9 @@ defusedxml==0.7.1 # via -r base.in executing==2.0.1 # via stack-data -fonttools==4.47.0 +fonttools==4.49.0 # via matplotlib -fsspec==2023.12.2 +fsspec==2024.2.0 # via dask graphviz==0.20.1 # via -r base.in @@ -49,11 +49,11 @@ kiwisolver==1.4.5 # via matplotlib locket==1.0.0 # via partd -matplotlib==3.8.2 +matplotlib==3.8.3 # via plopp matplotlib-inline==0.1.6 # via ipython -numpy==1.26.3 +numpy==1.26.4 # via # contourpy # h5py @@ -75,9 +75,9 @@ pickleshare==0.7.5 # via ipython pillow==10.2.0 # via matplotlib -platformdirs==4.1.0 +platformdirs==4.2.0 # via pooch -plopp==23.11.0 +plopp==24.1.1 # via -r base.in pooch==1.8.0 # via -r base.in @@ -101,15 +101,15 @@ pyyaml==6.0.1 # via dask requests==2.31.0 # via pooch -sciline==24.1.0 +sciline==24.2.1 # via -r base.in scipp==24.2.0 # via # -r base.in # scippnexus -scippnexus==23.12.0 +scippnexus==23.12.1 # via -r base.in -scipy==1.11.4 +scipy==1.12.0 # via scippnexus six==1.16.0 # via @@ -117,7 +117,7 @@ six==1.16.0 # python-dateutil stack-data==0.6.3 # via ipython -toolz==0.12.0 +toolz==0.12.1 # via # dask # partd @@ -125,9 +125,9 @@ traitlets==5.14.1 # via # ipython # matplotlib-inline -urllib3==2.1.0 +urllib3==2.2.1 # via requests -wcwidth==0.2.12 +wcwidth==0.2.13 # via prompt-toolkit zipp==3.17.0 # via importlib-metadata diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt index c2562f8a..bc93620f 100644 --- a/packages/essnmx/requirements/basetest.txt +++ b/packages/essnmx/requirements/basetest.txt @@ -11,9 +11,9 @@ iniconfig==2.0.0 # via pytest packaging==23.2 # via pytest -pluggy==1.3.0 +pluggy==1.4.0 # via pytest -pytest==7.4.4 +pytest==8.0.1 # via -r basetest.in tomli==2.0.1 # via pytest diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index e33fbbcf..19938753 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -7,7 +7,7 @@ # cachetools==5.3.2 # via tox -certifi==2023.11.17 +certifi==2024.2.2 # via requests chardet==5.2.0 # via tox @@ -23,7 +23,7 @@ filelock==3.13.1 # virtualenv gitdb==4.0.11 # via gitpython -gitpython==3.1.40 +gitpython==3.1.42 # via -r ci.in idna==3.6 # via requests @@ -32,11 +32,11 @@ packaging==23.2 # -r ci.in # pyproject-api # tox -platformdirs==4.1.0 +platformdirs==4.2.0 # via # tox # virtualenv -pluggy==1.3.0 +pluggy==1.4.0 # via tox pyproject-api==1.6.1 # via tox @@ -48,9 +48,9 @@ tomli==2.0.1 # via # pyproject-api # tox -tox==4.11.4 +tox==4.13.0 # via -r ci.in -urllib3==2.1.0 +urllib3==2.2.1 # via requests virtualenv==20.25.0 # via tox diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index b82a08bc..6109a3f3 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -14,8 +14,10 @@ -r wheels.txt annotated-types==0.6.0 # via pydantic -anyio==4.2.0 - # via jupyter-server +anyio==4.3.0 + # via + # httpx + # jupyter-server argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 @@ -26,14 +28,20 @@ async-lru==2.0.4 # via jupyterlab cffi==1.16.0 # via argon2-cffi-bindings -copier==9.1.0 +copier==9.1.1 # via -r dev.in -dunamai==1.19.0 +dunamai==1.19.2 # via copier fqdn==1.5.1 # via jsonschema funcy==2.0 # via copier +h11==0.14.0 + # via httpcore +httpcore==1.0.3 + # via httpx +httpx==0.26.0 + # via jupyterlab isoduration==20.11.0 # via jsonschema jinja2-ansible-filters==1.3.2 @@ -42,46 +50,46 @@ json5==0.9.14 # via jupyterlab-server jsonpointer==2.4 # via jsonschema -jsonschema[format-nongpl]==4.20.0 +jsonschema[format-nongpl]==4.21.1 # via # jupyter-events # jupyterlab-server # nbformat jupyter-events==0.9.0 # via jupyter-server -jupyter-lsp==2.2.1 +jupyter-lsp==2.2.2 # via jupyterlab -jupyter-server==2.12.2 +jupyter-server==2.12.5 # via # jupyter-lsp # jupyterlab # jupyterlab-server # notebook-shim -jupyter-server-terminals==0.5.1 +jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.0.10 +jupyterlab==4.1.1 # via -r dev.in -jupyterlab-server==2.25.2 +jupyterlab-server==2.25.3 # via jupyterlab -notebook-shim==0.2.3 +notebook-shim==0.2.4 # via jupyterlab -overrides==7.4.0 +overrides==7.7.0 # via jupyter-server pathspec==0.12.1 # via copier pip-compile-multi==2.6.3 # via -r dev.in -pip-tools==7.3.0 +pip-tools==7.4.0 # via pip-compile-multi plumbum==1.8.2 # via copier -prometheus-client==0.19.0 +prometheus-client==0.20.0 # via jupyter-server pycparser==2.21 # via cffi -pydantic==2.5.3 +pydantic==2.6.1 # via copier -pydantic-core==2.14.6 +pydantic-core==2.16.2 # via pydantic python-json-logger==2.0.7 # via jupyter-events @@ -100,14 +108,16 @@ rfc3986-validator==0.1.1 send2trash==1.8.2 # via jupyter-server sniffio==1.3.0 - # via anyio + # via + # anyio + # httpx terminado==0.18.0 # via # jupyter-server # jupyter-server-terminals toposort==1.10 # via pip-compile-multi -types-python-dateutil==2.8.19.14 +types-python-dateutil==2.8.19.20240106 # via arrow uri-template==1.3.0 # via jsonschema diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index d9dc6ed7..d544fd57 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -8,7 +8,7 @@ -r base.txt accessible-pygments==0.0.4 # via pydata-sphinx-theme -alabaster==0.7.13 +alabaster==0.7.16 # via sphinx attrs==23.2.0 # via @@ -18,7 +18,7 @@ babel==2.14.0 # via # pydata-sphinx-theme # sphinx -beautifulsoup4==4.12.2 +beautifulsoup4==4.12.3 # via # nbconvert # pydata-sphinx-theme @@ -28,7 +28,7 @@ comm==0.2.1 # via # ipykernel # ipywidgets -debugpy==1.8.0 +debugpy==1.8.1 # via ipykernel docutils==0.20.1 # via @@ -42,19 +42,19 @@ imagesize==1.4.1 # via sphinx ipydatawidgets==4.3.5 # via pythreejs -ipykernel==6.28.0 +ipykernel==6.29.2 # via -r docs.in -ipywidgets==8.1.1 +ipywidgets==8.1.2 # via # ipydatawidgets # pythreejs -jinja2==3.1.2 +jinja2==3.1.3 # via # myst-parser # nbconvert # nbsphinx # sphinx -jsonschema==4.20.0 +jsonschema==4.21.1 # via nbformat jsonschema-specifications==2023.12.1 # via jsonschema @@ -62,7 +62,7 @@ jupyter-client==8.6.0 # via # ipykernel # nbclient -jupyter-core==5.7.0 +jupyter-core==5.7.1 # via # ipykernel # jupyter-client @@ -71,13 +71,13 @@ jupyter-core==5.7.0 # nbformat jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-widgets==3.0.9 +jupyterlab-widgets==3.0.10 # via ipywidgets markdown-it-py==3.0.0 # via # mdit-py-plugins # myst-parser -markupsafe==2.1.3 +markupsafe==2.1.5 # via # jinja2 # nbconvert @@ -91,7 +91,7 @@ myst-parser==2.0.0 # via -r docs.in nbclient==0.9.0 # via nbconvert -nbconvert==7.14.0 +nbconvert==7.16.0 # via nbsphinx nbformat==5.9.2 # via @@ -100,13 +100,13 @@ nbformat==5.9.2 # nbsphinx nbsphinx==0.9.3 # via -r docs.in -nest-asyncio==1.5.8 +nest-asyncio==1.6.0 # via ipykernel -pandocfilters==1.5.0 +pandocfilters==1.5.1 # via nbconvert -psutil==5.9.7 +psutil==5.9.8 # via ipykernel -pydata-sphinx-theme==0.14.4 +pydata-sphinx-theme==0.15.2 # via -r docs.in pythreejs==2.4.2 # via -r docs.in @@ -114,15 +114,15 @@ pyzmq==25.1.2 # via # ipykernel # jupyter-client -referencing==0.32.0 +referencing==0.33.0 # via # jsonschema # jsonschema-specifications -rpds-py==0.16.2 +rpds-py==0.18.0 # via # jsonschema # referencing -scippneutron==23.11.0 +scippneutron==24.1.0 # via -r docs.in snowballstemmer==2.2.0 # via sphinx @@ -137,28 +137,23 @@ sphinx==7.2.6 # sphinx-autodoc-typehints # sphinx-copybutton # sphinx-design - # sphinxcontrib-applehelp - # sphinxcontrib-devhelp - # sphinxcontrib-htmlhelp - # sphinxcontrib-qthelp - # sphinxcontrib-serializinghtml -sphinx-autodoc-typehints==1.25.2 +sphinx-autodoc-typehints==2.0.0 # via -r docs.in sphinx-copybutton==0.5.2 # via -r docs.in sphinx-design==0.5.0 # via -r docs.in -sphinxcontrib-applehelp==1.0.7 +sphinxcontrib-applehelp==1.0.8 # via sphinx -sphinxcontrib-devhelp==1.0.5 +sphinxcontrib-devhelp==1.0.6 # via sphinx -sphinxcontrib-htmlhelp==2.0.4 +sphinxcontrib-htmlhelp==2.0.5 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.6 +sphinxcontrib-qthelp==1.0.7 # via sphinx -sphinxcontrib-serializinghtml==1.1.9 +sphinxcontrib-serializinghtml==1.1.10 # via sphinx tinycss2==1.2.1 # via nbconvert @@ -174,5 +169,5 @@ webencodings==0.5.1 # via # bleach # tinycss2 -widgetsnbextension==4.0.9 +widgetsnbextension==4.0.10 # via ipywidgets diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 3b124cfd..1454185a 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -6,7 +6,7 @@ # pip-compile-multi # -r basetest.txt -certifi==2023.11.17 +certifi==2024.2.2 # via requests charset-normalizer==3.3.2 # via requests @@ -18,13 +18,13 @@ contourpy==1.2.0 # via matplotlib cycler==0.12.1 # via matplotlib -dask==2023.12.1 +dask==2024.2.0 # via -r nightly.in defusedxml==0.7.1 # via -r nightly.in -fonttools==4.47.0 +fonttools==4.49.0 # via matplotlib -fsspec==2023.12.2 +fsspec==2024.2.0 # via dask graphviz==0.20.1 # via -r nightly.in @@ -40,9 +40,9 @@ kiwisolver==1.4.5 # via matplotlib locket==1.0.0 # via partd -matplotlib==3.8.2 +matplotlib==3.8.3 # via plopp -numpy==1.26.3 +numpy==1.26.4 # via # contourpy # h5py @@ -53,7 +53,7 @@ partd==1.4.1 # via dask pillow==10.2.0 # via matplotlib -platformdirs==4.1.0 +platformdirs==4.2.0 # via pooch plopp @ git+https://github.com/scipp/plopp@main # via -r nightly.in @@ -77,15 +77,15 @@ scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-c # scippnexus scippnexus @ git+https://github.com/scipp/scippnexus@main # via -r nightly.in -scipy==1.11.4 +scipy==1.12.0 # via scippnexus six==1.16.0 # via python-dateutil -toolz==0.12.0 +toolz==0.12.1 # via # dask # partd -urllib3==2.1.0 +urllib3==2.2.1 # via requests zipp==3.17.0 # via diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index fa660fba..68bf60c6 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -11,13 +11,13 @@ distlib==0.3.8 # via virtualenv filelock==3.13.1 # via virtualenv -identify==2.5.33 +identify==2.5.35 # via pre-commit nodeenv==1.8.0 # via pre-commit -platformdirs==4.1.0 +platformdirs==4.2.0 # via virtualenv -pre-commit==3.6.0 +pre-commit==3.6.2 # via -r static.in pyyaml==6.0.1 # via pre-commit From e26b6ec26841618bcdcb9e7333e8f74ec3cdbf2d Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Mon, 19 Feb 2024 17:54:58 +0100 Subject: [PATCH 077/403] Update dependencies including mermaid extension. --- packages/essnmx/requirements/base.txt | 2 +- packages/essnmx/requirements/docs.in | 1 + packages/essnmx/requirements/docs.txt | 6 ++++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 81414222..8da21998 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -77,7 +77,7 @@ pillow==10.2.0 # via matplotlib platformdirs==4.2.0 # via pooch -plopp==24.1.1 +plopp==24.2.0 # via -r base.in pooch==1.8.0 # via -r base.in diff --git a/packages/essnmx/requirements/docs.in b/packages/essnmx/requirements/docs.in index cc0c975f..e9076a41 100644 --- a/packages/essnmx/requirements/docs.in +++ b/packages/essnmx/requirements/docs.in @@ -10,3 +10,4 @@ sphinx-copybutton sphinx-design pythreejs # For instrument view. scippneutron +sphinxcontrib-mermaid diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index d544fd57..e0131657 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -1,4 +1,4 @@ -# SHA1:a4b21b1181ffe0d85627ecf73a8b997699993f2a +# SHA1:d900abacb6285d51223a23d628416b89bc359870 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -91,7 +91,7 @@ myst-parser==2.0.0 # via -r docs.in nbclient==0.9.0 # via nbconvert -nbconvert==7.16.0 +nbconvert==7.16.1 # via nbsphinx nbformat==5.9.2 # via @@ -151,6 +151,8 @@ sphinxcontrib-htmlhelp==2.0.5 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx +sphinxcontrib-mermaid==0.9.2 + # via -r docs.in sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 From 0f47b7ff026092d8355b9f3079ebb62071c54589 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Mon, 19 Feb 2024 18:26:27 +0100 Subject: [PATCH 078/403] Remove mermaid extension again. --- packages/essnmx/requirements/docs.in | 1 - packages/essnmx/requirements/docs.txt | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/essnmx/requirements/docs.in b/packages/essnmx/requirements/docs.in index e9076a41..cc0c975f 100644 --- a/packages/essnmx/requirements/docs.in +++ b/packages/essnmx/requirements/docs.in @@ -10,4 +10,3 @@ sphinx-copybutton sphinx-design pythreejs # For instrument view. scippneutron -sphinxcontrib-mermaid diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index e0131657..4285eae9 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -1,4 +1,4 @@ -# SHA1:d900abacb6285d51223a23d628416b89bc359870 +# SHA1:a4b21b1181ffe0d85627ecf73a8b997699993f2a # # This file is autogenerated by pip-compile-multi # To update, run: @@ -151,8 +151,6 @@ sphinxcontrib-htmlhelp==2.0.5 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-mermaid==0.9.2 - # via -r docs.in sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 From 3bd7f308839b8c86eeef4c023ac73c96cef0539d Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Fri, 16 Feb 2024 11:12:00 +0100 Subject: [PATCH 079/403] Allow missing rotation information in xml geometry to adapt McStas 3 files. --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 47 +++++++++++++------- packages/essnmx/src/ess/nmx/mcstas_xml.py | 39 ++++++++++------ 2 files changed, 56 insertions(+), 30 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index f21d5308..ca849aeb 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -9,6 +9,8 @@ PixelIDs = NewType("PixelIDs", sc.Variable) InputFilepath = NewType("InputFilepath", str) +DetectorName = NewType("DetectorName", str) +DetectorBankName = NewType("DetectorBankName", str) # McStas Configurations MaximumProbability = NewType("MaximumProbability", int) @@ -29,26 +31,33 @@ """A function that derives arbitrary proton charge based on event weights.""" -def _retrieve_event_list_name(keys: Iterable[str]) -> str: - prefix = "bank01_events_dat_list" +def _retrieve_event_list_names(keys: Iterable[str]) -> tuple[str, ...]: + import re - # (weight, x, y, n, pixel id, time of arrival) mandatory_fields = 'p_x_y_n_id_t' + # (weight, x, y, n, pixel id, time of arrival) + pattern = r"bank(\d+\d+)_events_dat_list_" + mandatory_fields - for key in keys: - if key.startswith(prefix) and mandatory_fields in key: - return key + def _filter_event_list_name(key: str) -> bool: + return re.search(pattern, key) is not None - raise ValueError("Can not find event list name.") + if not (matching_keys := tuple(filter(_filter_event_list_name, keys))): + raise ValueError("Can not find event list name.") + + return matching_keys def _retrieve_raw_event_data(file: snx.File) -> sc.Variable: """Retrieve events from the nexus file.""" - bank_name = _retrieve_event_list_name(file["entry1/data"].keys()) - # ``dim_0``: event index, ``dim_1``: property index. - return file["entry1/data/" + bank_name]["events"][()].rename_dims( - {'dim_0': 'event'} - ) + bank_names = _retrieve_event_list_names(file["entry1/data"].keys()) + + banks = [ + file["entry1/data/" + bank_name]["events"][()].rename_dims({'dim_0': 'event'}) + # ``dim_0``: event index, ``dim_1``: property index. + for bank_name in bank_names + ] + + return sc.concat(banks, 'event') def _copy_partial_var( @@ -164,12 +173,14 @@ def load_mcstas_nexus( file_path: File name to load. - event_weights_converter: + event_weights_converter: :class:`~EventWeightsConverter`, \ + default: :func:`~event_weights_from_probability` A function to convert probabilities to event weights. The function should accept the probabilities as the first argument, and return the converted event weights. - proton_charge_converter: + proton_charge_converter: :class:`~ProtonChargeConverter`, \ + default: :func:`~proton_charge_from_event_data` A function to convert the event weights to proton charge. The function should accept the event weights as the first argument, and return the proton charge. @@ -182,8 +193,12 @@ def load_mcstas_nexus( from .mcstas_xml import read_mcstas_geometry_xml + # with snx.File(file_path) as file: + # mcstas_version = _retrieve_mcstas_version(file) + geometry = read_mcstas_geometry_xml(file_path) - coords = geometry.to_coords() + detectors = [det.name for det in geometry.detectors] + coords = geometry.to_coords(*detectors) with snx.File(file_path) as file: raw_data = _retrieve_raw_event_data(file) @@ -198,7 +213,7 @@ def load_mcstas_nexus( id_list=_copy_partial_var(raw_data, idx=4, dtype='int64'), # id t_list=_copy_partial_var(raw_data, idx=5, unit='s'), # t pixel_ids=coords.pop('pixel_id'), - num_panels=len(geometry.detectors), + num_panels=len(detectors), ) proton_charge = proton_charge_converter(event_da) crystal_rotation = _retrieve_crystal_rotation( diff --git a/packages/essnmx/src/ess/nmx/mcstas_xml.py b/packages/essnmx/src/ess/nmx/mcstas_xml.py index d981a30e..d0377cc1 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas_xml.py @@ -212,7 +212,7 @@ def num_fast_pixels_per_row(self) -> int: def _collect_detector_descriptions(tree: _XML) -> Tuple[DetectorDesc, ...]: """Retrieve detector geometry descriptions from mcstas file.""" - type_list = filter_by_tag(tree, 'type') + type_list = list(filter_by_tag(tree, 'type')) simulation_settings = SimulationSettings.from_xml(tree) def _find_type_desc(det: _XML) -> _XML: @@ -245,7 +245,7 @@ class SampleDesc: name: str # From under position: sc.Variable - rotation_matrix: sc.Variable + rotation_matrix: Optional[sc.Variable] @classmethod def from_xml( @@ -254,14 +254,18 @@ def from_xml( """Create sample description from xml component.""" source_xml = select_by_type_prefix(tree, 'sampleMantid-type') location = select_by_tag(source_xml, 'location') + try: + rotation_matrix = _rotation_matrix_from_location( + location, simulation_settings.angle_unit + ) + except KeyError: + rotation_matrix = None return cls( component_type=source_xml.attrib['type'], name=source_xml.attrib['name'], position=_position_from_location(location, simulation_settings.length_unit), - rotation_matrix=_rotation_matrix_from_location( - location, simulation_settings.angle_unit - ), + rotation_matrix=rotation_matrix, ) def position_from_sample(self, other: sc.Variable) -> sc.Variable: @@ -371,24 +375,31 @@ def from_xml(cls, tree: _XML) -> 'McStasInstrument': ), ) - def to_coords(self) -> dict[str, sc.Variable]: - """Extract coordinates from the McStas instrument description.""" - slow_axes = [det.slow_axis for det in self.detectors] - fast_axes = [det.fast_axis for det in self.detectors] - origins = [ - self.sample.position_from_sample(det.position) for det in self.detectors - ] + def to_coords(self, *det_names: str) -> dict[str, sc.Variable]: + """Extract coordinates from the McStas instrument description. + + Parameters + ---------- + det_names: + Names of the detectors to extract coordinates for. + + """ + + detectors = tuple(det for det in self.detectors if det.name in det_names) + slow_axes = [det.slow_axis for det in detectors] + fast_axes = [det.fast_axis for det in detectors] + origins = [self.sample.position_from_sample(det.position) for det in detectors] detector_dim = 'panel' return { - 'pixel_id': _construct_pixel_ids(self.detectors), + 'pixel_id': _construct_pixel_ids(detectors), 'fast_axis': sc.concat(fast_axes, detector_dim), 'slow_axis': sc.concat(slow_axes, detector_dim), 'origin_position': sc.concat(origins, detector_dim), 'sample_position': self.sample.position_from_sample(self.sample.position), 'source_position': self.sample.position_from_sample(self.source.position), 'sample_name': sc.scalar(self.sample.name), - 'position': _detector_pixel_positions(self.detectors, self.sample), + 'position': _detector_pixel_positions(detectors, self.sample), } From 82cbabb39c722c445ae385ec78be4c93f1efab4b Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Fri, 16 Feb 2024 14:51:49 +0100 Subject: [PATCH 080/403] Update src/ess/nmx/mcstas_loader.py Co-authored-by: Jan-Lukas Wynen --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index ca849aeb..da4ae610 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -36,7 +36,7 @@ def _retrieve_event_list_names(keys: Iterable[str]) -> tuple[str, ...]: mandatory_fields = 'p_x_y_n_id_t' # (weight, x, y, n, pixel id, time of arrival) - pattern = r"bank(\d+\d+)_events_dat_list_" + mandatory_fields + pattern = r"bank(\d\d+)_events_dat_list_" + mandatory_fields def _filter_event_list_name(key: str) -> bool: return re.search(pattern, key) is not None From 3046483eba941cfcb00ab93ff7bcc8114cb8cb8f Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Fri, 16 Feb 2024 14:53:20 +0100 Subject: [PATCH 081/403] Remove unnecessary commented out lines. --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index da4ae610..43b165da 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -193,9 +193,6 @@ def load_mcstas_nexus( from .mcstas_xml import read_mcstas_geometry_xml - # with snx.File(file_path) as file: - # mcstas_version = _retrieve_mcstas_version(file) - geometry = read_mcstas_geometry_xml(file_path) detectors = [det.name for det in geometry.detectors] coords = geometry.to_coords(*detectors) From 52dc047e1c9311b75dfb03ecdab4ed6f1ad35f4c Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Mon, 19 Feb 2024 18:41:05 +0100 Subject: [PATCH 082/403] Use keyword-only argument in the loader. --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 43b165da..66352d1c 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -158,6 +158,7 @@ def proton_charge_from_event_data(event_da: sc.DataArray) -> ProtonCharge: def load_mcstas_nexus( + *, file_path: InputFilepath, event_weights_converter: EventWeightsConverter = event_weights_from_probability, proton_charge_converter: ProtonChargeConverter = proton_charge_from_event_data, From 14da3439bdf6daa329e60b01a27da31e29b114fe Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 20 Feb 2024 17:20:34 +0100 Subject: [PATCH 083/403] Add mcstas 3 file tests and add deprecation warnings for mcstas 2 file. --- packages/essnmx/docs/api-reference/index.md | 2 +- packages/essnmx/docs/examples/workflow.ipynb | 5 +- packages/essnmx/src/ess/nmx/__init__.py | 4 +- packages/essnmx/src/ess/nmx/data/__init__.py | 31 ++++- packages/essnmx/tests/conftest.py | 13 ++ packages/essnmx/tests/loader_test.py | 122 +++++++++++++------ packages/essnmx/tests/workflow_test.py | 23 +++- 7 files changed, 145 insertions(+), 55 deletions(-) create mode 100644 packages/essnmx/tests/conftest.py diff --git a/packages/essnmx/docs/api-reference/index.md b/packages/essnmx/docs/api-reference/index.md index 8cf99737..cf280dd3 100644 --- a/packages/essnmx/docs/api-reference/index.md +++ b/packages/essnmx/docs/api-reference/index.md @@ -24,7 +24,7 @@ :toctree: ../generated/functions :recursive: - small_mcstas_sample + small_mcstas_3_sample load_mcstas_nexus ``` diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index b359dbd7..86403515 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -32,12 +32,13 @@ " ProtonChargeConverter,\n", " proton_charge_from_event_data,\n", ")\n", - "from ess.nmx.data import small_mcstas_sample\n", + "from ess.nmx.data import small_mcstas_3_sample\n", + "from ess.nmx.data import small_mcstas_2_sample\n", "from ess.nmx.reduction import bin_time_of_arrival, TimeBinSteps\n", "\n", "providers = (load_mcstas_nexus, bin_time_of_arrival, )\n", "\n", - "file_path = small_mcstas_sample() # Replace it with your data file path\n", + "file_path = small_mcstas_2_sample() # Replace it with your data file path\n", "params = {\n", " TimeBinSteps: TimeBinSteps(50),\n", " InputFilepath: InputFilepath(file_path),\n", diff --git a/packages/essnmx/src/ess/nmx/__init__.py b/packages/essnmx/src/ess/nmx/__init__.py index f8440287..23ab38f3 100644 --- a/packages/essnmx/src/ess/nmx/__init__.py +++ b/packages/essnmx/src/ess/nmx/__init__.py @@ -11,12 +11,12 @@ del importlib -from .data import small_mcstas_sample +from .data import small_mcstas_3_sample from .mcstas_loader import InputFilepath, NMXData, load_mcstas_nexus from .reduction import NMXReducedData, TimeBinSteps __all__ = [ - "small_mcstas_sample", + "small_mcstas_3_sample", "NMXData", "InputFilepath", "load_mcstas_nexus", diff --git a/packages/essnmx/src/ess/nmx/data/__init__.py b/packages/essnmx/src/ess/nmx/data/__init__.py index 360c4eba..0ac139ff 100644 --- a/packages/essnmx/src/ess/nmx/data/__init__.py +++ b/packages/essnmx/src/ess/nmx/data/__init__.py @@ -3,7 +3,7 @@ _version = '0' -__all__ = ['small_mcstas_sample', 'get_path'] +__all__ = ['small_mcstas_2_sample', 'small_mcstas_3_sample', 'get_path'] def _make_pooch(): @@ -15,15 +15,38 @@ def _make_pooch(): retry_if_failed=3, base_url='https://public.esss.dk/groups/scipp/ess/nmx/', version=_version, - registry={'small_mcstas_sample.h5': 'md5:c3affe636397f8c9eea1d9c10a2bf487'}, + registry={ + 'small_mcstas_2_sample.h5': 'md5:c3affe636397f8c9eea1d9c10a2bf487', + 'small_mcstas_3_sample.h5': 'md5:2afaac205d13ee857ee5364e3f1957a7', + }, ) _pooch = _make_pooch() -def small_mcstas_sample(): - return get_path('small_mcstas_sample.h5') +def small_mcstas_2_sample(): + """McStas 2 file containing small number of events.""" + import warnings + + warnings.warn( + DeprecationWarning( + '``essnmx`` will not support loading files ' + 'made by McStas with version less than 3 from ``25.0.0``. ' + 'Use ``small_mcstas_3_sample`` instead.' + ), + stacklevel=2, + ) + + return get_path('small_mcstas_2_sample.h5') + + +def small_mcstas_3_sample(): + """McStas 3 file that contains only ``bank0(1-3)`` in the ``data`` group. + + Real McStas 3 file should contain more dataset under ``data`` group. + """ + return get_path('small_mcstas_3_sample.h5') def get_path(name: str) -> str: diff --git a/packages/essnmx/tests/conftest.py b/packages/essnmx/tests/conftest.py new file mode 100644 index 00000000..33d8e057 --- /dev/null +++ b/packages/essnmx/tests/conftest.py @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +# These fixtures cannot be found by pytest, +# if they are not defined in `conftest.py` under `tests` directory. +from contextlib import AbstractContextManager +from functools import partial + +import pytest + + +@pytest.fixture +def mcstas_2_deprecation_warning_context() -> partial[AbstractContextManager]: + return partial(pytest.warns, DeprecationWarning, match="McStas") diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 8d47c254..8edb96e4 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -3,15 +3,24 @@ import pathlib from typing import Generator +import numpy as np import pytest import scipp as sc - -from ess.nmx.data import small_mcstas_sample +import scippnexus as snx + +from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample +from ess.nmx.mcstas_loader import ( + DefaultMaximumProbability, + InputFilepath, + event_weights_from_probability, + load_mcstas_nexus, + proton_charge_from_event_data, +) from ess.nmx.reduction import NMXData -def check_scalar_properties(dg: NMXData): - """Test helper for NMXData. +def check_scalar_properties_mcstas_2(dg: NMXData): + """Test helper for NMXData loaded from McStas 2. Expected numbers are hard-coded based on the sample file. """ @@ -25,34 +34,10 @@ def check_scalar_properties(dg: NMXData): assert dg.sample_name == sc.scalar("sampleMantid") -def test_file_reader_mcstas() -> None: - import numpy as np - import scippnexus as snx - - from ess.nmx.mcstas_loader import ( - DefaultMaximumProbability, - InputFilepath, - event_weights_from_probability, - load_mcstas_nexus, - proton_charge_from_event_data, - ) - - file_path = InputFilepath(small_mcstas_sample()) - entry_path = "entry1/data/bank01_events_dat_list_p_x_y_n_id_t" - with snx.File(file_path) as file: - raw_data = file[entry_path]["events"][()] - data_length = raw_data.sizes['dim_0'] - - dg = load_mcstas_nexus( - file_path=file_path, - event_weights_converter=event_weights_from_probability, - proton_charge_converter=proton_charge_from_event_data, - ) +def check_nmxdata_properties(dg: NMXData) -> None: assert isinstance(dg, sc.DataGroup) assert dg.shape == (3, 1280 * 1280) - check_scalar_properties(dg) - # Check size and maximum value of weights. - assert dg.weights.bins.size().sum().value == data_length + # Check maximum value of weights. assert sc.identical( dg.weights.max().data, sc.scalar(DefaultMaximumProbability, unit='counts', dtype=float), @@ -74,13 +59,77 @@ def test_file_reader_mcstas() -> None: ) -@pytest.fixture -def tmp_mcstas_file(tmp_path: pathlib.Path) -> Generator[pathlib.Path, None, None]: +def test_file_reader_mcstas2(mcstas_2_deprecation_warning_context) -> None: + with mcstas_2_deprecation_warning_context(): + file_path = InputFilepath(small_mcstas_2_sample()) + + entry_path = "entry1/data/bank01_events_dat_list_p_x_y_n_id_t" + with snx.File(file_path) as file: + raw_data = file[entry_path]["events"][()] + data_length = raw_data.sizes['dim_0'] + + dg = load_mcstas_nexus( + file_path=file_path, + event_weights_converter=event_weights_from_probability, + proton_charge_converter=proton_charge_from_event_data, + ) + check_scalar_properties_mcstas_2(dg) + assert dg.weights.bins.size().sum().value == data_length + check_nmxdata_properties(dg) + + +def check_scalar_properties_mcstas_3(dg: NMXData): + """Test helper for NMXData loaded from McStas 3. + + Expected numbers are hard-coded based on the sample file. + """ + + assert dg.proton_charge == sc.scalar(0.0167, unit=None) + assert sc.identical(dg.crystal_rotation, sc.vector([0, 0, 0], unit='deg')) + assert sc.identical(dg.sample_position, sc.vector(value=[0, 0, 0], unit='m')) + assert sc.identical( + dg.source_position, sc.vector(value=[-0.53123, 0.0, -157.405], unit='m') + ) + assert dg.sample_name == sc.scalar("sampleMantid") + + +def test_file_reader_mcstas3() -> None: + file_path = InputFilepath(small_mcstas_3_sample()) + entry_paths = [ + f"entry1/data/bank0{i}_events_dat_list_p_x_y_n_id_t" for i in range(1, 4) + ] + with snx.File(file_path) as file: + raw_datas = [file[entry_path]["events"][()] for entry_path in entry_paths] + raw_data = sc.concat(raw_datas, dim='dim_0') + data_length = raw_data.sizes['dim_0'] + + dg = load_mcstas_nexus( + file_path=file_path, + event_weights_converter=event_weights_from_probability, + proton_charge_converter=proton_charge_from_event_data, + ) + check_scalar_properties_mcstas_3(dg) + assert dg.weights.bins.size().sum().value == data_length + check_nmxdata_properties(dg) + + +@pytest.fixture(params=[small_mcstas_2_sample, small_mcstas_3_sample]) +def tmp_mcstas_file( + tmp_path: pathlib.Path, + request: pytest.FixtureRequest, + mcstas_2_deprecation_warning_context, +) -> Generator[pathlib.Path, None, None]: import os import shutil + if request.param == small_mcstas_2_sample: + with mcstas_2_deprecation_warning_context(): + original_file_path = request.param() + else: + original_file_path = request.param() + tmp_file = tmp_path / pathlib.Path('file.h5') - shutil.copy(small_mcstas_sample(), tmp_file) + shutil.copy(original_file_path, tmp_file) yield tmp_file os.remove(tmp_file) @@ -89,13 +138,6 @@ def test_file_reader_mcstas_additional_fields(tmp_mcstas_file: pathlib.Path) -> """Check if additional fields names do not break the loader.""" import h5py - from ess.nmx.mcstas_loader import ( - InputFilepath, - event_weights_from_probability, - load_mcstas_nexus, - proton_charge_from_event_data, - ) - entry_path = "entry1/data/bank01_events_dat_list_p_x_y_n_id_t" new_entry_path = entry_path + '_L' diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index e30866e7..18699e4a 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -4,10 +4,22 @@ import sciline as sl import scipp as sc +from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample + + +@pytest.fixture(params=[small_mcstas_2_sample, small_mcstas_3_sample]) +def mcstas_file_path( + request: pytest.FixtureRequest, mcstas_2_deprecation_warning_context +) -> str: + if request.param == small_mcstas_2_sample: + with mcstas_2_deprecation_warning_context(): + return request.param() + + return request.param() + @pytest.fixture -def mcstas_workflow() -> sl.Pipeline: - from ess.nmx.data import small_mcstas_sample +def mcstas_workflow(mcstas_file_path: str) -> sl.Pipeline: from ess.nmx.mcstas_loader import ( DefaultMaximumProbability, EventWeightsConverter, @@ -23,7 +35,7 @@ def mcstas_workflow() -> sl.Pipeline: return sl.Pipeline( [load_mcstas_nexus, bin_time_of_arrival], params={ - InputFilepath: small_mcstas_sample(), + InputFilepath: mcstas_file_path, MaximumProbability: DefaultMaximumProbability, TimeBinSteps: TimeBinSteps(50), EventWeightsConverter: event_weights_from_probability, @@ -32,11 +44,10 @@ def mcstas_workflow() -> sl.Pipeline: ) -def test_pipeline_builder(mcstas_workflow: sl.Pipeline) -> None: - from ess.nmx.data import small_mcstas_sample +def test_pipeline_builder(mcstas_workflow: sl.Pipeline, mcstas_file_path: str) -> None: from ess.nmx.mcstas_loader import InputFilepath - assert mcstas_workflow.get(InputFilepath).compute() == small_mcstas_sample() + assert mcstas_workflow.get(InputFilepath).compute() == mcstas_file_path def test_pipeline_mcstas_loader(mcstas_workflow: sl.Pipeline) -> None: From 3bd9857905a2d4918767c3d06da60ad713ad2c07 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 21 Feb 2024 12:48:39 +0100 Subject: [PATCH 084/403] Drop python3.9 and add python3.12 --- packages/essnmx/.copier-answers.yml | 8 ++++---- packages/essnmx/.github/workflows/python-version-ci | 2 +- packages/essnmx/LICENSE | 2 +- packages/essnmx/conda/meta.yaml | 4 ++++ packages/essnmx/docs/conf.py | 3 ++- packages/essnmx/docs/developer/getting-started.md | 2 +- packages/essnmx/pyproject.toml | 4 ++-- packages/essnmx/requirements/base.in | 7 ++----- packages/essnmx/requirements/make_base.py | 2 +- packages/essnmx/src/ess/nmx/__init__.py | 2 +- packages/essnmx/tests/package_test.py | 2 +- packages/essnmx/tox.ini | 2 +- 12 files changed, 21 insertions(+), 19 deletions(-) diff --git a/packages/essnmx/.copier-answers.yml b/packages/essnmx/.copier-answers.yml index f4cc884b..16beda4a 100644 --- a/packages/essnmx/.copier-answers.yml +++ b/packages/essnmx/.copier-answers.yml @@ -1,13 +1,13 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY -_commit: 1abe96f +_commit: 6770edb _src_path: gh:scipp/copier_template description: Data reduction for NMX at the European Spallation Source. -max_python: '3.11' -min_python: '3.9' +max_python: '3.12' +min_python: '3.10' namespace_package: ess nightly_deps: scipp,sciline,scippnexus,plopp orgname: scipp prettyname: ESSnmx projectname: essnmx related_projects: Scipp,Sciline,Plopp,ScippNexus -year: 2023 +year: 2024 diff --git a/packages/essnmx/.github/workflows/python-version-ci b/packages/essnmx/.github/workflows/python-version-ci index bd28b9c5..c8cfe395 100644 --- a/packages/essnmx/.github/workflows/python-version-ci +++ b/packages/essnmx/.github/workflows/python-version-ci @@ -1 +1 @@ -3.9 +3.10 diff --git a/packages/essnmx/LICENSE b/packages/essnmx/LICENSE index b402aa64..54b3cf4d 100644 --- a/packages/essnmx/LICENSE +++ b/packages/essnmx/LICENSE @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2023, Scipp contributors (https://github.com/scipp) +Copyright (c) 2024, Scipp contributors (https://github.com/scipp) All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/packages/essnmx/conda/meta.yaml b/packages/essnmx/conda/meta.yaml index ab9ae253..d5ed082d 100644 --- a/packages/essnmx/conda/meta.yaml +++ b/packages/essnmx/conda/meta.yaml @@ -11,6 +11,7 @@ requirements: - setuptools - setuptools_scm run: +<<<<<<< before updating - python>=3.9 - dask - python-graphviz @@ -19,6 +20,9 @@ requirements: - scipp>=23.8.0 - scippnexus>=23.9.0 - pooch +======= + - python>=3.10 +>>>>>>> after updating test: imports: diff --git a/packages/essnmx/docs/conf.py b/packages/essnmx/docs/conf.py index 9d4285fe..bea8cd80 100644 --- a/packages/essnmx/docs/conf.py +++ b/packages/essnmx/docs/conf.py @@ -10,7 +10,7 @@ # General information about the project. project = u'ESSnmx' -copyright = u'2023 Scipp contributors' +copyright = u'2024 Scipp contributors' author = u'Scipp contributors' html_show_sourcelink = True @@ -23,6 +23,7 @@ 'sphinx.ext.intersphinx', 'sphinx.ext.mathjax', 'sphinx.ext.napoleon', + 'sphinx.ext.viewcode', 'sphinx_autodoc_typehints', 'sphinx_copybutton', 'sphinx_design', diff --git a/packages/essnmx/docs/developer/getting-started.md b/packages/essnmx/docs/developer/getting-started.md index 046d5978..a196f562 100644 --- a/packages/essnmx/docs/developer/getting-started.md +++ b/packages/essnmx/docs/developer/getting-started.md @@ -40,7 +40,7 @@ Alternatively, if you want a different workflow, take a look at ``tox.ini`` or ` Run the tests using ```sh -tox -e py39 +tox -e py310 ``` (or just `tox` if you want to run all environments). diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 8052920f..96f175c5 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -18,13 +18,13 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering", "Typing :: Typed", ] -requires-python = ">=3.9" +requires-python = ">=3.10" # IMPORTANT: # Run 'tox -e deps' after making changes here. This will update requirement files. diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index ded1646c..a302d07f 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -1,8 +1,5 @@ -# Temporary until questionary (dep of copier) updates -# See https://github.com/tmbo/questionary/blob/2df265534f3eb77aafcf70902e53e80beb1793e0/pyproject.toml#L36C43-L36C110 -prompt-toolkit==3.0.36 -# Temporary pinned until prompt-tookit conflict is resolved. -ipython==8.9.0 +# Anything above "--- END OF CUSTOM SECTION ---" +# will not be touched by ``make_base.py`` # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask diff --git a/packages/essnmx/requirements/make_base.py b/packages/essnmx/requirements/make_base.py index b26a1c2e..3b1dbabc 100644 --- a/packages/essnmx/requirements/make_base.py +++ b/packages/essnmx/requirements/make_base.py @@ -43,7 +43,7 @@ def write_dependencies(dependency_name: str, dependencies: List[str]) -> None: with open("../pyproject.toml", "rb") as toml_file: pyproject = tomli.load(toml_file) dependencies = pyproject["project"].get("dependencies") - if not dependencies: + if dependencies is None: raise RuntimeError("No dependencies found in pyproject.toml") dependencies = [dep.strip().strip('"') for dep in dependencies] diff --git a/packages/essnmx/src/ess/nmx/__init__.py b/packages/essnmx/src/ess/nmx/__init__.py index 23ab38f3..d3320b5e 100644 --- a/packages/essnmx/src/ess/nmx/__init__.py +++ b/packages/essnmx/src/ess/nmx/__init__.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) # flake8: noqa import importlib.metadata diff --git a/packages/essnmx/tests/package_test.py b/packages/essnmx/tests/package_test.py index a0ce3927..5e6fc243 100644 --- a/packages/essnmx/tests/package_test.py +++ b/packages/essnmx/tests/package_test.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) from ess import nmx as pkg diff --git a/packages/essnmx/tox.ini b/packages/essnmx/tox.ini index df80a9d1..d65754ab 100644 --- a/packages/essnmx/tox.ini +++ b/packages/essnmx/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py39 +envlist = {py310,py311,py312} isolated_build = true [testenv] From 3280dda8c362836c4815c86993fbd475be529402 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 21 Feb 2024 16:42:03 +0100 Subject: [PATCH 085/403] Resolve merge conflict. --- packages/essnmx/conda/meta.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/essnmx/conda/meta.yaml b/packages/essnmx/conda/meta.yaml index d5ed082d..9a9141de 100644 --- a/packages/essnmx/conda/meta.yaml +++ b/packages/essnmx/conda/meta.yaml @@ -11,8 +11,6 @@ requirements: - setuptools - setuptools_scm run: -<<<<<<< before updating - - python>=3.9 - dask - python-graphviz - plopp @@ -20,9 +18,7 @@ requirements: - scipp>=23.8.0 - scippnexus>=23.9.0 - pooch -======= - python>=3.10 ->>>>>>> after updating test: imports: From e29fdb3591d8617bb0a27c60609e1cee4c4b38e8 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 21 Feb 2024 12:52:37 +0100 Subject: [PATCH 086/403] Update dependencies. --- packages/essnmx/requirements/base.txt | 46 ++------------------ packages/essnmx/requirements/dev.txt | 6 +-- packages/essnmx/requirements/docs.txt | 55 ++++++++++++++++++++++++ packages/essnmx/requirements/nightly.in | 2 +- packages/essnmx/requirements/nightly.txt | 12 ++---- packages/essnmx/requirements/wheels.txt | 4 -- 6 files changed, 66 insertions(+), 59 deletions(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 8da21998..82445781 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,14 +1,10 @@ -# SHA1:f2f02404509e42e072ec3a85641b6b2fe68d380a +# SHA1:c4a2744ee02d3a805c47e68a6e258681958f71f6 # # This file is autogenerated by pip-compile-multi # To update, run: # # pip-compile-multi # -asttokens==2.4.1 - # via stack-data -backcall==0.2.0 - # via ipython certifi==2024.2.2 # via requests charset-normalizer==3.3.2 @@ -23,12 +19,8 @@ cycler==0.12.1 # via matplotlib dask==2024.2.0 # via -r base.in -decorator==5.1.1 - # via ipython defusedxml==0.7.1 # via -r base.in -executing==2.0.1 - # via stack-data fonttools==4.49.0 # via matplotlib fsspec==2024.2.0 @@ -41,18 +33,12 @@ idna==3.6 # via requests importlib-metadata==7.0.1 # via dask -ipython==8.9.0 - # via -r base.in -jedi==0.19.1 - # via ipython kiwisolver==1.4.5 # via matplotlib locket==1.0.0 # via partd matplotlib==3.8.3 # via plopp -matplotlib-inline==0.1.6 - # via ipython numpy==1.26.4 # via # contourpy @@ -65,32 +51,16 @@ packaging==23.2 # dask # matplotlib # pooch -parso==0.8.3 - # via jedi partd==1.4.1 # via dask -pexpect==4.9.0 - # via ipython -pickleshare==0.7.5 - # via ipython pillow==10.2.0 # via matplotlib platformdirs==4.2.0 # via pooch plopp==24.2.0 # via -r base.in -pooch==1.8.0 +pooch==1.8.1 # via -r base.in -prompt-toolkit==3.0.36 - # via - # -r base.in - # ipython -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -pygments==2.17.2 - # via ipython pyparsing==3.1.1 # via matplotlib python-dateutil==2.8.2 @@ -112,22 +82,12 @@ scippnexus==23.12.1 scipy==1.12.0 # via scippnexus six==1.16.0 - # via - # asttokens - # python-dateutil -stack-data==0.6.3 - # via ipython + # via python-dateutil toolz==0.12.1 # via # dask # partd -traitlets==5.14.1 - # via - # ipython - # matplotlib-inline urllib3==2.2.1 # via requests -wcwidth==0.2.13 - # via prompt-toolkit zipp==3.17.0 # via importlib-metadata diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index 6109a3f3..392873ff 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -46,7 +46,7 @@ isoduration==20.11.0 # via jsonschema jinja2-ansible-filters==1.3.2 # via copier -json5==0.9.14 +json5==0.9.17 # via jupyterlab-server jsonpointer==2.4 # via jsonschema @@ -67,7 +67,7 @@ jupyter-server==2.12.5 # notebook-shim jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.1.1 +jupyterlab==4.1.2 # via -r dev.in jupyterlab-server==2.25.3 # via jupyterlab @@ -95,7 +95,7 @@ python-json-logger==2.0.7 # via jupyter-events pyyaml-include==1.3.2 # via copier -questionary==2.0.1 +questionary==1.10.0 # via copier rfc3339-validator==0.1.4 # via diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 4285eae9..4c2cc3f2 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -10,6 +10,8 @@ accessible-pygments==0.0.4 # via pydata-sphinx-theme alabaster==0.7.16 # via sphinx +asttokens==2.4.1 + # via stack-data attrs==23.2.0 # via # jsonschema @@ -30,12 +32,18 @@ comm==0.2.1 # ipywidgets debugpy==1.8.1 # via ipykernel +decorator==5.1.1 + # via ipython docutils==0.20.1 # via # myst-parser # nbsphinx # pydata-sphinx-theme # sphinx +exceptiongroup==1.2.0 + # via ipython +executing==2.0.1 + # via stack-data fastjsonschema==2.19.1 # via nbformat imagesize==1.4.1 @@ -44,10 +52,17 @@ ipydatawidgets==4.3.5 # via pythreejs ipykernel==6.29.2 # via -r docs.in +ipython==8.21.0 + # via + # -r docs.in + # ipykernel + # ipywidgets ipywidgets==8.1.2 # via # ipydatawidgets # pythreejs +jedi==0.19.1 + # via ipython jinja2==3.1.3 # via # myst-parser @@ -81,6 +96,10 @@ markupsafe==2.1.5 # via # jinja2 # nbconvert +matplotlib-inline==0.1.6 + # via + # ipykernel + # ipython mdit-py-plugins==0.4.0 # via myst-parser mdurl==0.1.2 @@ -104,10 +123,27 @@ nest-asyncio==1.6.0 # via ipykernel pandocfilters==1.5.1 # via nbconvert +parso==0.8.3 + # via jedi +pexpect==4.9.0 + # via ipython +prompt-toolkit==3.0.43 + # via ipython psutil==5.9.8 # via ipykernel +ptyprocess==0.7.0 + # via pexpect +pure-eval==0.2.2 + # via stack-data pydata-sphinx-theme==0.15.2 # via -r docs.in +pygments==2.17.2 + # via + # accessible-pygments + # ipython + # nbconvert + # pydata-sphinx-theme + # sphinx pythreejs==2.4.2 # via -r docs.in pyzmq==25.1.2 @@ -155,16 +191,35 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx +stack-data==0.6.3 + # via ipython tinycss2==1.2.1 # via nbconvert tornado==6.4 # via # ipykernel # jupyter-client +traitlets==5.14.1 + # via + # comm + # ipykernel + # ipython + # ipywidgets + # jupyter-client + # jupyter-core + # matplotlib-inline + # nbclient + # nbconvert + # nbformat + # nbsphinx + # pythreejs + # traittypes traittypes==0.2.1 # via ipydatawidgets typing-extensions==4.9.0 # via pydata-sphinx-theme +wcwidth==0.2.13 + # via prompt-toolkit webencodings==0.5.1 # via # bleach diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index 01325973..ec3c0208 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -5,7 +5,7 @@ dask graphviz pooch defusedxml -https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl sciline @ git+https://github.com/scipp/sciline@main scippnexus @ git+https://github.com/scipp/scippnexus@main plopp @ git+https://github.com/scipp/plopp@main diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 1454185a..c571766b 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:15630cfbae70da78bb40bd86bc6b9d0737fd1dd1 +# SHA1:90858db5f2c5c8ccee07229c0c1cdb99c03ac3c3 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -34,8 +34,6 @@ idna==3.6 # via requests importlib-metadata==7.0.1 # via dask -importlib-resources==6.1.1 - # via matplotlib kiwisolver==1.4.5 # via matplotlib locket==1.0.0 @@ -57,7 +55,7 @@ platformdirs==4.2.0 # via pooch plopp @ git+https://github.com/scipp/plopp@main # via -r nightly.in -pooch==1.8.0 +pooch==1.8.1 # via -r nightly.in pyparsing==3.1.1 # via matplotlib @@ -71,7 +69,7 @@ requests==2.31.0 # via pooch sciline @ git+https://github.com/scipp/sciline@main # via -r nightly.in -scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl # via # -r nightly.in # scippnexus @@ -88,6 +86,4 @@ toolz==0.12.1 urllib3==2.2.1 # via requests zipp==3.17.0 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata diff --git a/packages/essnmx/requirements/wheels.txt b/packages/essnmx/requirements/wheels.txt index c26530af..2e33cfa3 100644 --- a/packages/essnmx/requirements/wheels.txt +++ b/packages/essnmx/requirements/wheels.txt @@ -7,8 +7,6 @@ # build==1.0.3 # via -r wheels.in -importlib-metadata==7.0.1 - # via build packaging==23.2 # via build pyproject-hooks==1.0.0 @@ -17,5 +15,3 @@ tomli==2.0.1 # via # build # pyproject-hooks -zipp==3.17.0 - # via importlib-metadata From 7bed014ed5811cafffd233856cb80179288b6cd1 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Mon, 19 Feb 2024 12:00:36 +0100 Subject: [PATCH 087/403] Improve readability and fix typos. --- .../essnmx/docs/about/work_flow_design.md | 110 ++++++++++++------ 1 file changed, 75 insertions(+), 35 deletions(-) diff --git a/packages/essnmx/docs/about/work_flow_design.md b/packages/essnmx/docs/about/work_flow_design.md index dcb533d7..af7d7e39 100644 --- a/packages/essnmx/docs/about/work_flow_design.md +++ b/packages/essnmx/docs/about/work_flow_design.md @@ -1,89 +1,129 @@ -# Design document data workflow for the NMX instrument at ESS +# Data workflow design for the NMX instrument at ESS This is a description of the data workflow for the NMX instrument at ESS. -The [NMX](https://europeanspallationsource.se/instruments/nmx) Macromolecular Diffractometer is a time-of-flight (TOF) quasi-Laue diffractometer optimised for small samples and large unit cells dedicated to the structure determination of biological macromolecules by crystallography. +The [NMX](https://europeanspallationsource.se/instruments/nmx) Macromolecular Diffractometer is a time-of-flight (TOF) +quasi-Laue diffractometer optimised for small samples and large unit cells +dedicated to the structure determination of biological macromolecules by crystallography. + The main scientific driver is to locate the hydrogen atoms relevant to the function of the macromolecule. ## Data reduction ![work_flow](NMX_work_flow.png) - -### From single event data to binned image-like data -From single event data to binned image-like data -The first step in the data reduction is to reduce the data from single event data to image-like data. +### From single event data to binned image-like data (scipp) +The first step in the data reduction is to reduce the data from single event data to image-like data.
Therefore the [essNMX](https://github.com/scipp/essnmx) package is used. -First, the time of arrival (TOA) is converted into time of flight (TOF). -Then the single events get binned into pixels and then histogramed in the TOF dimension. -These data will be written be added with some meta and instrument data to an HDF5 file. +The time of arrival (TOA) should be converted into time of flight (TOF). + +Then the single events get binned into pixels and then histogramed in the TOF dimension.
+This result can be exported to an HDF5 file +along with additional metadata and instrument coordinates (pixel IDs). + +See [workflow example](../examples/workflow_example) for more details. + +### Spot finding and integration (DIAL) +For the next five steps of the data reduction from spot finding to spot integration, +we use a program called [DIALS](https://dials.github.io/index.html) [^1]. +[^1]: [DIAL as a toolkit](https://onlinelibrary.wiley.com/doi/10.1002/pro.4224) -### Spot finding and integration -For the next five steps of the data reduction from spot finding to spot integration, we use the [programme](https://dials.github.io/index.html) [DIALS](https://onlinelibrary.wiley.com/doi/10.1002/pro.4224) +#### 1. Import Image-like Files -First, we use [dials.import](https://dials.github.io/documentation/programs/dials_import.html) to convert image data files into a format compatible with dials. It the metadata and filenames of each image to establish relationships between different sets of images. Once all images are processed, the program generates an experiment object file, outlining the connections between the files. The images to be processed are designated as command-line arguments. Occasionally, there may be a restriction on the maximum number of arguments allowed on the command line, and the number of files could surpass this limit. In such cases, image filenames can be entered through stdin, as demonstrated in the examples below. +First, we use [dials.import](https://dials.github.io/documentation/programs/dials_import.html) to convert image data files into a format compatible with dials. + +It processes the metadata and filenames of each image to establish relationships between different sets of images.
+Once all images are processed, the program generates an experiment object file, outlining the connections between the files.
+The images to be processed are designated as command-line arguments.
+Occasionally, there may be a restriction on the maximum number of arguments allowed on the command line, and the number of files could surpass this limit.
+In such cases, image filenames can be entered through stdin, as demonstrated in the examples below.
The Format class for NMX is at modules/dxtbx/src/dxtbx/format/FormatNMX.py where beam-line-specific parameters and file format information are stored. ```console dials.import *.nxs ``` -In the next step, a [search for strong pixel](https://dials.github.io/documentation/programs/dials_find_spots.html) is performed. Therefore the intensity of a pixel or pixel group is compared with its local surroundings. With the information of strong pixels, strong spots are defined. for these spots, the centroids and intensities will be calculated. the results can be visualised in the image viewer or the [browser](https://toastisme.github.io/dials_browser_experiment_viewer/) +#### 2. Search for Strong Pixels + +The next step is to [search for strong pixels](https://dials.github.io/documentation/programs/dials_find_spots.html).
+In this step, the intensity of each pixel or a pixel group is compared with its local surroundings.
+With the information of strong pixels, strong spots are defined.
+To find these spots, the centroids and intensities will be calculated.
+The results can be visualised in the image viewer or the [dial browser](https://toastisme.github.io/dials_browser_experiment_viewer/). + ```console dials.find_spots imported.expt find_spots.phil ``` -In the [indexing](https://dials.github.io/documentation/programs/dials_index.html) step the unit cell is determined. a list of indexed reflexes and an instrument model including a crystal model is returned. One-dimensional and three-dimensional fast Fourier transform-based methods are available. +#### 3. Index Instrument Geometry + +In the [indexing](https://dials.github.io/documentation/programs/dials_index.html) step the unit cell is determined.
+A list of indexed reflexes and an instrument model including a crystal model is returned.
+One-dimensional and three-dimensional fast Fourier transform-based methods are available. -As input parameters the imported.exp and strong.refl files are used. more parameters such as unit cell and spacegroup can be given. +As input parameters the ``imported.exp`` and ``strong.refl`` files are used.
+More parameters such as ``unit cell`` and ``spacegroup`` can be given. ```console dials.index imported.expt strong.refl space_group=P1 unit_cell=a,b,c,alpha,beta,gamma ``` +#### 4. Refine the Diffraction Geometry +The result of indexing the instrument geometry is then used to get refined diffraction geometry [^2]. +[^2]: (https://dials.github.io/documentation/programs/dials_refine) -After indexing the instrument geometry is getting [refined](https://dials.github.io/documentation/programs/dials_refine.html). ```console dials.refine indexed.refl indexed.expt detector.panels=hierarchical ``` -The last step in DIALS is to integrate(https://dials.github.io/documentation/programs/dials_integrate.html) each reflex. Currently, in the dimension of the image, a simple summation is used and in the TOF dimension, a profile-fitting approach is used. +#### 5. Integrate Reflexes + +The last step in DIALS is to integrate each reflex.[^3] +[^3]: (https://dials.github.io/documentation/programs/dials_integrate.html) + +Currently, different approach is used to integrate the dimension of the image and the dimension of TOF.
+In the dimension of the image, a simple summation is used +and in the TOF dimension, a profile-fitting approach is used. ```console dev.dials.simple_tof_integrate refined.expt refined.refl ``` +### Scaling (LSCALE/pyscale) +Currently [LSCALE](https://scripts.iucr.org/cgi-bin/paper?S0021889898015350) can be used in a docker container which makes it indented from the OS.
+LSCALE is a program for scaling and normalisation of Laue intensity data.
+The source code is available on [Zenodo](https://zenodo.org/records/4381992).
+Since LSCALE is not maintained anymore we are currently developing a Python-based alternative to LSCALE called pyscale[^4]. +[^4]: ``pyscale`` is under development and lives in a private repository. Please ask for access to the repository to the [owner](https://github.com/mlund) if needed. - - -### Scaling -Currently [LSCALE](https://scripts.iucr.org/cgi-bin/paper?S0021889898015350) can be used in a docker container which makes it indented from the OS. The source code is available on [Zenodo](https://zenodo.org/records/4381992). LSCALE is a program for scaling and normalisation of Laue intensity data. -Since LSCALE is not maintained anymore we currently develop a Python-based [alternative](https://github.com/mlund/pyscale) to LSCALE. - -start docker desktop +**To start docker desktop** ```console docker run -it -v $HOME:/mnt/host -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=host.docker.internal:0 lscale ``` +**Command to run ``lscale``** ```console lscale < lscale.com > lscale.out ``` -### AIMLESS and CTRUNCATE +### Merge Intensities and Derive Structure Factors (CCP4, AIMLESS and CTRUNCATE) +[AIMLESS](https://www.ccp4.ac.uk/html/aimless.html) and [CTRUNCATE](https://www.ccp4.ac.uk/html/ctruncate.html) are sub-programs of [CCP4](https://www.ccp4.ac.uk/html/). -[AIMLESS](https://www.ccp4.ac.uk/html/aimless.html) is used to scale multiple observations of reflections together, and merges multiple observations into an average intensity. +[AIMLESS](https://www.ccp4.ac.uk/html/aimless.html) can scale multiple observations of reflections together.
+It can also merge multiple observations into an average intensity. +[CTRUNCATE](https://www.ccp4.ac.uk/html/ctruncate.html) converts measured intensities into structure factors.
+CTRUNCATE includes corrections for weak reflections to avoid negative intensities due to background corrections. -[CTRUNCATE](https://www.ccp4.ac.uk/html/ctruncate.html) converts measured intensities into structure factors. CTRUNCATE includes corrections for weak reflections to avoid negative intensities due to background corrections. +This step can be done via GUI interfaces of ``CCP4``. +1. Start ``CCP4`` GUI +2. Go to ``all programs`` +3. Select ``Aimless`` +4. Select ``scaled *mtz file`` -```console -Start CCP4 GUI -go to all programs -select Aimless -select scaled *.mtz file -``` -usually, standard parameters are fine but parameters can be modified. +Parameters can be modified.
+Standard parameters are fine in most cases. -This results in a final *mtz file which can be used in a standard protein crystallographic program to solve and refine the structure. +The ``mtz`` file can be used in a standard protein crystallographic program to solve and refine the structure. From 4bbef92683847dba9a7296aef460f8bb8f3dd74b Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Mon, 19 Feb 2024 18:24:31 +0100 Subject: [PATCH 088/403] Include data workflow overview page. --- .../{work_flow_design.md => data_workflow_overview.md} | 9 ++++----- packages/essnmx/docs/about/index.md | 8 ++++++++ 2 files changed, 12 insertions(+), 5 deletions(-) rename packages/essnmx/docs/about/{work_flow_design.md => data_workflow_overview.md} (96%) diff --git a/packages/essnmx/docs/about/work_flow_design.md b/packages/essnmx/docs/about/data_workflow_overview.md similarity index 96% rename from packages/essnmx/docs/about/work_flow_design.md rename to packages/essnmx/docs/about/data_workflow_overview.md index af7d7e39..17c010b7 100644 --- a/packages/essnmx/docs/about/work_flow_design.md +++ b/packages/essnmx/docs/about/data_workflow_overview.md @@ -1,6 +1,5 @@ -# Data workflow design for the NMX instrument at ESS - -This is a description of the data workflow for the NMX instrument at ESS. +# Data Workflow Overview +This is an overall description of the data workflow for the NMX instrument at ESS. The [NMX](https://europeanspallationsource.se/instruments/nmx) Macromolecular Diffractometer is a time-of-flight (TOF) quasi-Laue diffractometer optimised for small samples and large unit cells @@ -10,7 +9,7 @@ The main scientific driver is to locate the hydrogen atoms relevant to the funct ## Data reduction -![work_flow](NMX_work_flow.png) +![Workflow Overview](NMX_work_flow.png) ### From single event data to binned image-like data (scipp) The first step in the data reduction is to reduce the data from single event data to image-like data.
@@ -24,7 +23,7 @@ along with additional metadata and instrument coordinates (pixel IDs). See [workflow example](../examples/workflow_example) for more details. -### Spot finding and integration (DIAL) +### Spot finding and integration (DIALS) For the next five steps of the data reduction from spot finding to spot integration, we use a program called [DIALS](https://dials.github.io/index.html) [^1]. [^1]: [DIAL as a toolkit](https://onlinelibrary.wiley.com/doi/10.1002/pro.4224) diff --git a/packages/essnmx/docs/about/index.md b/packages/essnmx/docs/about/index.md index 22ab8121..3453d9d6 100644 --- a/packages/essnmx/docs/about/index.md +++ b/packages/essnmx/docs/about/index.md @@ -1,5 +1,13 @@ # About +```{toctree} +--- +maxdepth: 3 +--- + +data_workflow_overview +``` + ## Development ESSnmx is an open source project by the [European Spallation Source ERIC](https://europeanspallationsource.se/) (ESS). From e59a817042f6b53d589374d86f21acaa73d3f556 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 21 Feb 2024 13:21:48 +0100 Subject: [PATCH 089/403] Update to use sciline pipeline wrapper. --- packages/essnmx/docs/examples/workflow.ipynb | 23 ++++++++------------ 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index 86403515..e5ad0b6a 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -5,9 +5,9 @@ "metadata": {}, "source": [ "# Workflow\n", + "In this example, we will use McStas 3 simulation file.\n", "\n", - "## Collect Parameters and Providers\n", - "### Simulation(McStas) Data\n", + "## Build Pipeline (Collect Parameters and Providers)\n", "There is a dedicated loader, ``load_mcstas_nexus`` for ``McStas`` simulation data workflow.
\n", "``MaximumProbability`` can be manually provided to the loader
\n", "to derive more realistic number of events.
\n", @@ -33,12 +33,11 @@ " proton_charge_from_event_data,\n", ")\n", "from ess.nmx.data import small_mcstas_3_sample\n", - "from ess.nmx.data import small_mcstas_2_sample\n", "from ess.nmx.reduction import bin_time_of_arrival, TimeBinSteps\n", "\n", "providers = (load_mcstas_nexus, bin_time_of_arrival, )\n", "\n", - "file_path = small_mcstas_2_sample() # Replace it with your data file path\n", + "file_path = small_mcstas_3_sample() # Replace it with your data file path\n", "params = {\n", " TimeBinSteps: TimeBinSteps(50),\n", " InputFilepath: InputFilepath(file_path),\n", @@ -60,7 +59,7 @@ "The reason of having them as parameters not as providers is,\n", "\n", "1. They are not part of general reduction, which are only for McStas cases.\n", - "2. They are better done while the file is open and read in the loader.\n" + "2. They are better done while the file is open and read in the loader." ] }, { @@ -69,14 +68,10 @@ "metadata": {}, "outputs": [], "source": [ - "from typing import get_type_hints\n", - "param_reprs = {key.__name__: value for key, value in params.items()}\n", - "prov_reprs = {\n", - " get_type_hints(prov)['return'].__name__: prov.__name__ for prov in providers\n", - "}\n", - "\n", - "# Providers and parameters to be used for pipeline\n", - "sc.DataGroup(**prov_reprs, **param_reprs)" + "import sciline as sl\n", + "\n", + "nmx_pl = sl.Pipeline(list(providers), params=params)\n", + "nmx_pl" ] }, { @@ -194,7 +189,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.18" + "version": "3.10.13" } }, "nbformat": 4, From 08c13c256e67a05f0dca6317bce6fcb9fe114a4a Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Fri, 1 Mar 2024 10:38:57 +0100 Subject: [PATCH 090/403] Update data_workflow_overview.md --- packages/essnmx/docs/about/data_workflow_overview.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/docs/about/data_workflow_overview.md b/packages/essnmx/docs/about/data_workflow_overview.md index 17c010b7..ca0de56a 100644 --- a/packages/essnmx/docs/about/data_workflow_overview.md +++ b/packages/essnmx/docs/about/data_workflow_overview.md @@ -72,7 +72,7 @@ dials.index imported.expt strong.refl space_group=P1 unit_cell=a,b,c,alpha,beta, #### 4. Refine the Diffraction Geometry The result of indexing the instrument geometry is then used to get refined diffraction geometry [^2]. -[^2]: (https://dials.github.io/documentation/programs/dials_refine) +[^2]: https://dials.github.io/documentation/programs/dials_refine ```console dials.refine indexed.refl indexed.expt detector.panels=hierarchical @@ -81,7 +81,7 @@ dials.refine indexed.refl indexed.expt detector.panels=hierarchical #### 5. Integrate Reflexes The last step in DIALS is to integrate each reflex.[^3] -[^3]: (https://dials.github.io/documentation/programs/dials_integrate.html) +[^3]: https://dials.github.io/documentation/programs/dials_integrate.html Currently, different approach is used to integrate the dimension of the image and the dimension of TOF.
In the dimension of the image, a simple summation is used From cf620b88ff9d201af227605bd51648ae504c3832 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Fri, 1 Mar 2024 13:09:19 +0100 Subject: [PATCH 091/403] Add missing dependency in conda meta. --- packages/essnmx/conda/meta.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/essnmx/conda/meta.yaml b/packages/essnmx/conda/meta.yaml index 9a9141de..767706ef 100644 --- a/packages/essnmx/conda/meta.yaml +++ b/packages/essnmx/conda/meta.yaml @@ -18,6 +18,7 @@ requirements: - scipp>=23.8.0 - scippnexus>=23.9.0 - pooch + - defusedxml - python>=3.10 test: From c421e838a48ee1b2f2c7097bcd0995a419a19e64 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 7 Mar 2024 17:12:00 +0100 Subject: [PATCH 092/403] Update dependencies. --- packages/essnmx/conda/meta.yaml | 2 ++ packages/essnmx/pyproject.toml | 2 ++ packages/essnmx/requirements/base.in | 2 ++ packages/essnmx/requirements/base.txt | 28 +++++++++++++++++------ packages/essnmx/requirements/basetest.txt | 2 +- packages/essnmx/requirements/ci.txt | 6 ++--- packages/essnmx/requirements/dev.txt | 20 ++++++++-------- packages/essnmx/requirements/docs.txt | 12 ++++++---- packages/essnmx/requirements/mypy.txt | 2 +- packages/essnmx/requirements/nightly.in | 4 +++- packages/essnmx/requirements/nightly.txt | 28 +++++++++++++++++------ packages/essnmx/requirements/static.txt | 2 +- packages/essnmx/requirements/wheels.txt | 6 ++++- 13 files changed, 79 insertions(+), 37 deletions(-) diff --git a/packages/essnmx/conda/meta.yaml b/packages/essnmx/conda/meta.yaml index 767706ef..40645fb8 100644 --- a/packages/essnmx/conda/meta.yaml +++ b/packages/essnmx/conda/meta.yaml @@ -20,6 +20,8 @@ requirements: - pooch - defusedxml - python>=3.10 + - gemmi + - pandas test: imports: diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 96f175c5..1227ab17 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -37,6 +37,8 @@ dependencies = [ "scipp>=23.8.0", "scippnexus>=23.9.0", "pooch", + "pandas", + "gemmi", "defusedxml", ] diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index a302d07f..ceeae5e3 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -9,4 +9,6 @@ sciline>=23.9.1 scipp>=23.8.0 scippnexus>=23.9.0 pooch +pandas +gemmi defusedxml diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 82445781..16fa8066 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:c4a2744ee02d3a805c47e68a6e258681958f71f6 +# SHA1:755d32c56a7dbb9e3068b6bc07621e630837e6dc # # This file is autogenerated by pip-compile-multi # To update, run: @@ -17,7 +17,7 @@ contourpy==1.2.0 # via matplotlib cycler==0.12.1 # via matplotlib -dask==2024.2.0 +dask==2024.2.1 # via -r base.in defusedxml==0.7.1 # via -r base.in @@ -25,14 +25,18 @@ fonttools==4.49.0 # via matplotlib fsspec==2024.2.0 # via dask +gemmi==0.6.5 + # via -r base.in graphviz==0.20.1 # via -r base.in h5py==3.10.0 # via scippnexus idna==3.6 # via requests -importlib-metadata==7.0.1 +importlib-metadata==7.0.2 # via dask +importlib-resources==6.1.3 + # via matplotlib kiwisolver==1.4.5 # via matplotlib locket==1.0.0 @@ -44,6 +48,7 @@ numpy==1.26.4 # contourpy # h5py # matplotlib + # pandas # scipp # scipy packaging==23.2 @@ -51,6 +56,8 @@ packaging==23.2 # dask # matplotlib # pooch +pandas==2.2.1 + # via -r base.in partd==1.4.1 # via dask pillow==10.2.0 @@ -61,12 +68,15 @@ plopp==24.2.0 # via -r base.in pooch==1.8.1 # via -r base.in -pyparsing==3.1.1 +pyparsing==3.1.2 # via matplotlib -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # matplotlib + # pandas # scippnexus +pytz==2024.1 + # via pandas pyyaml==6.0.1 # via dask requests==2.31.0 @@ -77,7 +87,7 @@ scipp==24.2.0 # via # -r base.in # scippnexus -scippnexus==23.12.1 +scippnexus==24.3.1 # via -r base.in scipy==1.12.0 # via scippnexus @@ -87,7 +97,11 @@ toolz==0.12.1 # via # dask # partd +tzdata==2024.1 + # via pandas urllib3==2.2.1 # via requests zipp==3.17.0 - # via importlib-metadata + # via + # importlib-metadata + # importlib-resources diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt index bc93620f..a1fc27b6 100644 --- a/packages/essnmx/requirements/basetest.txt +++ b/packages/essnmx/requirements/basetest.txt @@ -13,7 +13,7 @@ packaging==23.2 # via pytest pluggy==1.4.0 # via pytest -pytest==8.0.1 +pytest==8.0.2 # via -r basetest.in tomli==2.0.1 # via pytest diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index 19938753..6b0041cf 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -5,7 +5,7 @@ # # pip-compile-multi # -cachetools==5.3.2 +cachetools==5.3.3 # via tox certifi==2024.2.2 # via requests @@ -48,9 +48,9 @@ tomli==2.0.1 # via # pyproject-api # tox -tox==4.13.0 +tox==4.14.1 # via -r ci.in urllib3==2.2.1 # via requests -virtualenv==20.25.0 +virtualenv==20.25.1 # via tox diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index 392873ff..33262f44 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -38,15 +38,15 @@ funcy==2.0 # via copier h11==0.14.0 # via httpcore -httpcore==1.0.3 +httpcore==1.0.4 # via httpx -httpx==0.26.0 +httpx==0.27.0 # via jupyterlab isoduration==20.11.0 # via jsonschema jinja2-ansible-filters==1.3.2 # via copier -json5==0.9.17 +json5==0.9.22 # via jupyterlab-server jsonpointer==2.4 # via jsonschema @@ -57,9 +57,9 @@ jsonschema[format-nongpl]==4.21.1 # nbformat jupyter-events==0.9.0 # via jupyter-server -jupyter-lsp==2.2.2 +jupyter-lsp==2.2.4 # via jupyterlab -jupyter-server==2.12.5 +jupyter-server==2.13.0 # via # jupyter-lsp # jupyterlab @@ -67,7 +67,7 @@ jupyter-server==2.12.5 # notebook-shim jupyter-server-terminals==0.5.2 # via jupyter-server -jupyterlab==4.1.2 +jupyterlab==4.1.4 # via -r dev.in jupyterlab-server==2.25.3 # via jupyterlab @@ -79,7 +79,7 @@ pathspec==0.12.1 # via copier pip-compile-multi==2.6.3 # via -r dev.in -pip-tools==7.4.0 +pip-tools==7.4.1 # via pip-compile-multi plumbum==1.8.2 # via copier @@ -87,9 +87,9 @@ prometheus-client==0.20.0 # via jupyter-server pycparser==2.21 # via cffi -pydantic==2.6.1 +pydantic==2.6.3 # via copier -pydantic-core==2.16.2 +pydantic-core==2.16.3 # via pydantic python-json-logger==2.0.7 # via jupyter-events @@ -107,7 +107,7 @@ rfc3986-validator==0.1.1 # jupyter-events send2trash==1.8.2 # via jupyter-server -sniffio==1.3.0 +sniffio==1.3.1 # via # anyio # httpx diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 4c2cc3f2..69128c7c 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -50,9 +50,9 @@ imagesize==1.4.1 # via sphinx ipydatawidgets==4.3.5 # via pythreejs -ipykernel==6.29.2 +ipykernel==6.29.3 # via -r docs.in -ipython==8.21.0 +ipython==8.18.1 # via # -r docs.in # ipykernel @@ -110,7 +110,7 @@ myst-parser==2.0.0 # via -r docs.in nbclient==0.9.0 # via nbconvert -nbconvert==7.16.1 +nbconvert==7.16.2 # via nbsphinx nbformat==5.9.2 # via @@ -216,8 +216,10 @@ traitlets==5.14.1 # traittypes traittypes==0.2.1 # via ipydatawidgets -typing-extensions==4.9.0 - # via pydata-sphinx-theme +typing-extensions==4.10.0 + # via + # ipython + # pydata-sphinx-theme wcwidth==0.2.13 # via prompt-toolkit webencodings==0.5.1 diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index ac285686..49722576 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -10,5 +10,5 @@ mypy==1.8.0 # via -r mypy.in mypy-extensions==1.0.0 # via mypy -typing-extensions==4.9.0 +typing-extensions==4.10.0 # via mypy diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index ec3c0208..8a7ccc99 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -4,8 +4,10 @@ dask graphviz pooch +pandas +gemmi defusedxml -https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl sciline @ git+https://github.com/scipp/sciline@main scippnexus @ git+https://github.com/scipp/scippnexus@main plopp @ git+https://github.com/scipp/plopp@main diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index c571766b..a3dedaf8 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:90858db5f2c5c8ccee07229c0c1cdb99c03ac3c3 +# SHA1:5ab5fa78fc427b600d881ae674cc81dcf0cc3eeb # # This file is autogenerated by pip-compile-multi # To update, run: @@ -18,7 +18,7 @@ contourpy==1.2.0 # via matplotlib cycler==0.12.1 # via matplotlib -dask==2024.2.0 +dask==2024.2.1 # via -r nightly.in defusedxml==0.7.1 # via -r nightly.in @@ -26,14 +26,18 @@ fonttools==4.49.0 # via matplotlib fsspec==2024.2.0 # via dask +gemmi==0.6.5 + # via -r nightly.in graphviz==0.20.1 # via -r nightly.in h5py==3.10.0 # via scippnexus idna==3.6 # via requests -importlib-metadata==7.0.1 +importlib-metadata==7.0.2 # via dask +importlib-resources==6.1.3 + # via matplotlib kiwisolver==1.4.5 # via matplotlib locket==1.0.0 @@ -45,8 +49,11 @@ numpy==1.26.4 # contourpy # h5py # matplotlib + # pandas # scipp # scipy +pandas==2.2.1 + # via -r nightly.in partd==1.4.1 # via dask pillow==10.2.0 @@ -57,19 +64,22 @@ plopp @ git+https://github.com/scipp/plopp@main # via -r nightly.in pooch==1.8.1 # via -r nightly.in -pyparsing==3.1.1 +pyparsing==3.1.2 # via matplotlib -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # matplotlib + # pandas # scippnexus +pytz==2024.1 + # via pandas pyyaml==6.0.1 # via dask requests==2.31.0 # via pooch sciline @ git+https://github.com/scipp/sciline@main # via -r nightly.in -scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl # via # -r nightly.in # scippnexus @@ -83,7 +93,11 @@ toolz==0.12.1 # via # dask # partd +tzdata==2024.1 + # via pandas urllib3==2.2.1 # via requests zipp==3.17.0 - # via importlib-metadata + # via + # importlib-metadata + # importlib-resources diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index 68bf60c6..0e3af8ac 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -21,7 +21,7 @@ pre-commit==3.6.2 # via -r static.in pyyaml==6.0.1 # via pre-commit -virtualenv==20.25.0 +virtualenv==20.25.1 # via pre-commit # The following packages are considered to be unsafe in a requirements file: diff --git a/packages/essnmx/requirements/wheels.txt b/packages/essnmx/requirements/wheels.txt index 2e33cfa3..a0234e2c 100644 --- a/packages/essnmx/requirements/wheels.txt +++ b/packages/essnmx/requirements/wheels.txt @@ -5,8 +5,10 @@ # # pip-compile-multi # -build==1.0.3 +build==1.1.1 # via -r wheels.in +importlib-metadata==7.0.2 + # via build packaging==23.2 # via build pyproject-hooks==1.0.0 @@ -15,3 +17,5 @@ tomli==2.0.1 # via # build # pyproject-hooks +zipp==3.17.0 + # via importlib-metadata From b2ace175e2984163e752fadbb927d98ba2d7684c Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Fri, 22 Mar 2024 15:12:41 +0100 Subject: [PATCH 093/403] Update nightly.in --- packages/essnmx/requirements/nightly.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index 8a7ccc99..245b5831 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -7,7 +7,7 @@ pooch pandas gemmi defusedxml -https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl sciline @ git+https://github.com/scipp/sciline@main scippnexus @ git+https://github.com/scipp/scippnexus@main plopp @ git+https://github.com/scipp/plopp@main From 83b1a378797cb521cbcffe8bb44a44c96f408ff5 Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Fri, 22 Mar 2024 15:13:38 +0100 Subject: [PATCH 094/403] Update nightly.txt --- packages/essnmx/requirements/nightly.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index a3dedaf8..f671ace6 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -79,7 +79,7 @@ requests==2.31.0 # via pooch sciline @ git+https://github.com/scipp/sciline@main # via -r nightly.in -scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl # via # -r nightly.in # scippnexus From cf31381a9ffb7fa647899867cef1987007aee827 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Fri, 15 Mar 2024 13:09:58 +0100 Subject: [PATCH 095/403] fix: load single bank --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 50 +++++--------------- 1 file changed, 13 insertions(+), 37 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 66352d1c..126613d0 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) -from typing import Callable, Iterable, NewType, Optional +from typing import Callable, NewType, Optional import scipp as sc import scippnexus as snx @@ -31,33 +31,12 @@ """A function that derives arbitrary proton charge based on event weights.""" -def _retrieve_event_list_names(keys: Iterable[str]) -> tuple[str, ...]: - import re - - mandatory_fields = 'p_x_y_n_id_t' - # (weight, x, y, n, pixel id, time of arrival) - pattern = r"bank(\d\d+)_events_dat_list_" + mandatory_fields - - def _filter_event_list_name(key: str) -> bool: - return re.search(pattern, key) is not None - - if not (matching_keys := tuple(filter(_filter_event_list_name, keys))): - raise ValueError("Can not find event list name.") - - return matching_keys - - -def _retrieve_raw_event_data(file: snx.File) -> sc.Variable: +def _retrieve_raw_event_data(file: snx.File, bank_name: str) -> sc.Variable: """Retrieve events from the nexus file.""" - bank_names = _retrieve_event_list_names(file["entry1/data"].keys()) - - banks = [ - file["entry1/data/" + bank_name]["events"][()].rename_dims({'dim_0': 'event'}) - # ``dim_0``: event index, ``dim_1``: property index. - for bank_name in bank_names - ] - - return sc.concat(banks, 'event') + bank_name = f'{bank_name}_events_dat_list_p_x_y_n_id_t' + return file["entry1/data/" + bank_name]["events"][()].rename_dims( + {'dim_0': 'event'} + ) def _copy_partial_var( @@ -106,7 +85,6 @@ def _compose_event_data_array( id_list: sc.Variable, t_list: sc.Variable, pixel_ids: sc.Variable, - num_panels: int, ) -> sc.DataArray: """Combine data with coordinates loaded from the nexus file. @@ -123,15 +101,10 @@ def _compose_event_data_array( pixel_ids: All possible pixel IDs of the detector. - - num_panels: - The number of (detector) panels used in the experiment. - """ events = sc.DataArray(data=weights, coords={'t': t_list, 'id': id_list}) - grouped: sc.DataArray = events.group(pixel_ids) - return grouped.fold(dim='id', sizes={'panel': num_panels, 'id': -1}) + return events.group(pixel_ids).fold(dim='id', sizes={'panel': 1, 'id': -1}) def proton_charge_from_event_data(event_da: sc.DataArray) -> ProtonCharge: @@ -148,7 +121,7 @@ def proton_charge_from_event_data(event_da: sc.DataArray) -> ProtonCharge: Parameters ---------- event_da: - The event data binned in detector panel and pixel id dimensions. + The event data binned in pixel id """ # Arbitrary number to scale the proton charge @@ -163,6 +136,7 @@ def load_mcstas_nexus( event_weights_converter: EventWeightsConverter = event_weights_from_probability, proton_charge_converter: ProtonChargeConverter = proton_charge_from_event_data, max_probability: Optional[MaximumProbability] = None, + detector_bank_name: DetectorBankName, ) -> NMXData: """Load McStas simulation result from h5(nexus) file. @@ -190,6 +164,9 @@ def load_mcstas_nexus( The maximum probability to scale the weights. If not provided, ``DefaultMaximumProbability`` is used. + detector_bank_name: + Name of the detector bank to load events from. + """ from .mcstas_xml import read_mcstas_geometry_xml @@ -199,7 +176,7 @@ def load_mcstas_nexus( coords = geometry.to_coords(*detectors) with snx.File(file_path) as file: - raw_data = _retrieve_raw_event_data(file) + raw_data = _retrieve_raw_event_data(file, detector_bank_name) weights = event_weights_converter( max_probability or DefaultMaximumProbability, McStasEventProbabilities( @@ -211,7 +188,6 @@ def load_mcstas_nexus( id_list=_copy_partial_var(raw_data, idx=4, dtype='int64'), # id t_list=_copy_partial_var(raw_data, idx=5, unit='s'), # t pixel_ids=coords.pop('pixel_id'), - num_panels=len(detectors), ) proton_charge = proton_charge_converter(event_da) crystal_rotation = _retrieve_crystal_rotation( From 93ad46cf5f751f6836f798815d7207429a429518 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Fri, 15 Mar 2024 13:12:31 +0100 Subject: [PATCH 096/403] fix: add bank to example --- packages/essnmx/docs/examples/workflow.ipynb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index e5ad0b6a..a74cd315 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -31,6 +31,7 @@ " event_weights_from_probability,\n", " ProtonChargeConverter,\n", " proton_charge_from_event_data,\n", + " DetectorBankName,\n", ")\n", "from ess.nmx.data import small_mcstas_3_sample\n", "from ess.nmx.reduction import bin_time_of_arrival, TimeBinSteps\n", @@ -45,6 +46,7 @@ " MaximumProbability: DefaultMaximumProbability,\n", " EventWeightsConverter: event_weights_from_probability,\n", " ProtonChargeConverter: proton_charge_from_event_data,\n", + " DetectorBankName: 'bank01',\n", "}" ] }, From 7aaba2c59c4963974ec528595067d4ad4906f56a Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Fri, 15 Mar 2024 14:05:41 +0100 Subject: [PATCH 097/403] test: fix some of the tests --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 2 ++ packages/essnmx/tests/loader_test.py | 12 +++++++----- packages/essnmx/tests/workflow_test.py | 6 ++++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 126613d0..308069eb 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -34,6 +34,7 @@ def _retrieve_raw_event_data(file: snx.File, bank_name: str) -> sc.Variable: """Retrieve events from the nexus file.""" bank_name = f'{bank_name}_events_dat_list_p_x_y_n_id_t' + (bank_name,) = (name for name in file["entry1/data"].keys() if bank_name in name) return file["entry1/data/" + bank_name]["events"][()].rename_dims( {'dim_0': 'event'} ) @@ -173,6 +174,7 @@ def load_mcstas_nexus( geometry = read_mcstas_geometry_xml(file_path) detectors = [det.name for det in geometry.detectors] + detectors = [geometry.detectors[0].name] coords = geometry.to_coords(*detectors) with snx.File(file_path) as file: diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 8edb96e4..4ece764f 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -72,6 +72,7 @@ def test_file_reader_mcstas2(mcstas_2_deprecation_warning_context) -> None: file_path=file_path, event_weights_converter=event_weights_from_probability, proton_charge_converter=proton_charge_from_event_data, + detector_bank_name='bank01', ) check_scalar_properties_mcstas_2(dg) assert dg.weights.bins.size().sum().value == data_length @@ -93,11 +94,10 @@ def check_scalar_properties_mcstas_3(dg: NMXData): assert dg.sample_name == sc.scalar("sampleMantid") -def test_file_reader_mcstas3() -> None: +@pytest.mark.parametrize('bank_id', ('01', '02', '03')) +def test_file_reader_mcstas3(bank_id) -> None: file_path = InputFilepath(small_mcstas_3_sample()) - entry_paths = [ - f"entry1/data/bank0{i}_events_dat_list_p_x_y_n_id_t" for i in range(1, 4) - ] + entry_paths = [f"entry1/data/bank{bank_id}_events_dat_list_p_x_y_n_id_t"] with snx.File(file_path) as file: raw_datas = [file[entry_path]["events"][()] for entry_path in entry_paths] raw_data = sc.concat(raw_datas, dim='dim_0') @@ -107,6 +107,7 @@ def test_file_reader_mcstas3() -> None: file_path=file_path, event_weights_converter=event_weights_from_probability, proton_charge_converter=proton_charge_from_event_data, + detector_bank_name=f'bank{bank_id}', ) check_scalar_properties_mcstas_3(dg) assert dg.weights.bins.size().sum().value == data_length @@ -150,7 +151,8 @@ def test_file_reader_mcstas_additional_fields(tmp_mcstas_file: pathlib.Path) -> file_path=InputFilepath(str(tmp_mcstas_file)), event_weights_converter=event_weights_from_probability, proton_charge_converter=proton_charge_from_event_data, + detector_bank_name='bank01', ) assert isinstance(dg, sc.DataGroup) - assert dg.shape == (3, 1280 * 1280) + assert dg.shape == (1, 1280 * 1280) diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index 18699e4a..11695ee7 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -22,6 +22,7 @@ def mcstas_file_path( def mcstas_workflow(mcstas_file_path: str) -> sl.Pipeline: from ess.nmx.mcstas_loader import ( DefaultMaximumProbability, + DetectorBankName, EventWeightsConverter, InputFilepath, MaximumProbability, @@ -40,6 +41,7 @@ def mcstas_workflow(mcstas_file_path: str) -> sl.Pipeline: TimeBinSteps: TimeBinSteps(50), EventWeightsConverter: event_weights_from_probability, ProtonChargeConverter: proton_charge_from_event_data, + DetectorBankName: 'bank01', }, ) @@ -56,7 +58,7 @@ def test_pipeline_mcstas_loader(mcstas_workflow: sl.Pipeline) -> None: nmx_data = mcstas_workflow.compute(NMXData) assert isinstance(nmx_data, sc.DataGroup) - assert nmx_data.sizes['panel'] == 3 + assert nmx_data.sizes['panel'] == 1 assert nmx_data.sizes['id'] == 1280 * 1280 @@ -66,5 +68,5 @@ def test_pipeline_mcstas_reduction(mcstas_workflow: sl.Pipeline) -> None: nmx_reduced_data = mcstas_workflow.compute(NMXReducedData) assert isinstance(nmx_reduced_data, sc.DataGroup) - assert nmx_reduced_data.sizes['panel'] == 3 + assert nmx_reduced_data.sizes['panel'] == 1 assert nmx_reduced_data.sizes['t'] == 50 From b500996937dd5656df342d2e309f0faae90e714c Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Fri, 15 Mar 2024 14:14:11 +0100 Subject: [PATCH 098/403] temporary workaround --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 308069eb..a9ddf0b6 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -173,9 +173,13 @@ def load_mcstas_nexus( from .mcstas_xml import read_mcstas_geometry_xml geometry = read_mcstas_geometry_xml(file_path) - detectors = [det.name for det in geometry.detectors] - detectors = [geometry.detectors[0].name] - coords = geometry.to_coords(*detectors) + bank_name_to_detector_name = dict( + zip( + (f'bank0{i}' for i in range(1, 4)), + ('nD_Mantid_0', 'nD_Mantid_1', 'nD_Mantid_2'), + ) + ) + coords = geometry.to_coords(bank_name_to_detector_name[detector_bank_name]) with snx.File(file_path) as file: raw_data = _retrieve_raw_event_data(file, detector_bank_name) From 19327f14ad98192a6dcf8958b93af7fc8c764c43 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 19 Mar 2024 11:12:00 +0100 Subject: [PATCH 099/403] remove panel dimension from geometry loader --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 2 +- packages/essnmx/src/ess/nmx/mcstas_xml.py | 49 ++++++++-------- packages/essnmx/src/ess/nmx/reduction.py | 11 +--- packages/essnmx/tests/loader_test.py | 61 ++++++++++---------- packages/essnmx/tests/workflow_test.py | 2 - 5 files changed, 56 insertions(+), 69 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index a9ddf0b6..cfac1e69 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -105,7 +105,7 @@ def _compose_event_data_array( """ events = sc.DataArray(data=weights, coords={'t': t_list, 'id': id_list}) - return events.group(pixel_ids).fold(dim='id', sizes={'panel': 1, 'id': -1}) + return events.group(pixel_ids) def proton_charge_from_event_data(event_da: sc.DataArray) -> ProtonCharge: diff --git a/packages/essnmx/src/ess/nmx/mcstas_xml.py b/packages/essnmx/src/ess/nmx/mcstas_xml.py index d0377cc1..82b2cbed 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas_xml.py @@ -308,13 +308,14 @@ def from_xml( ) -def _construct_pixel_ids(detector_descs: Tuple[DetectorDesc, ...]) -> sc.Variable: - """Pixel IDs for all detectors.""" - intervals = [ - (desc.id_start, desc.id_start + desc.total_pixels) for desc in detector_descs - ] - ids = [sc.arange('id', start, stop, unit=None) for start, stop in intervals] - return sc.concat(ids, 'id') +def _construct_pixel_ids(detector_desc: DetectorDesc) -> sc.Variable: + """Pixel IDs for detector.""" + return sc.arange( + 'id', + detector_desc.id_start, + detector_desc.id_start + detector_desc.total_pixels, + unit=None, + ) def _pixel_positions( @@ -341,15 +342,12 @@ def _pixel_positions( def _detector_pixel_positions( - detector_descs: Tuple[DetectorDesc, ...], sample: SampleDesc + detector_desc: DetectorDesc, sample: SampleDesc ) -> sc.Variable: - """Position of pixels of all detectors.""" - - positions = [ - _pixel_positions(detector, sample.position_from_sample(detector.position)) - for detector in detector_descs - ] - return sc.concat(positions, 'panel') + """Position of pixels of detector.""" + return _pixel_positions( + detector_desc, sample.position_from_sample(detector_desc.position) + ) @dataclass @@ -375,7 +373,7 @@ def from_xml(cls, tree: _XML) -> 'McStasInstrument': ), ) - def to_coords(self, *det_names: str) -> dict[str, sc.Variable]: + def to_coords(self, det_name: str) -> dict[str, sc.Variable]: """Extract coordinates from the McStas instrument description. Parameters @@ -385,21 +383,20 @@ def to_coords(self, *det_names: str) -> dict[str, sc.Variable]: """ - detectors = tuple(det for det in self.detectors if det.name in det_names) - slow_axes = [det.slow_axis for det in detectors] - fast_axes = [det.fast_axis for det in detectors] - origins = [self.sample.position_from_sample(det.position) for det in detectors] - detector_dim = 'panel' + (detector,) = (det for det in self.detectors if det.name == det_name) + slow_axis = detector.slow_axis + fast_axis = detector.fast_axis + origin = self.sample.position_from_sample(detector.position) return { - 'pixel_id': _construct_pixel_ids(detectors), - 'fast_axis': sc.concat(fast_axes, detector_dim), - 'slow_axis': sc.concat(slow_axes, detector_dim), - 'origin_position': sc.concat(origins, detector_dim), + 'pixel_id': _construct_pixel_ids(detector), + 'fast_axis': fast_axis, + 'slow_axis': slow_axis, + 'origin_position': origin, 'sample_position': self.sample.position_from_sample(self.sample.position), 'source_position': self.sample.position_from_sample(self.source.position), 'sample_name': sc.scalar(self.sample.name), - 'position': _detector_pixel_positions(detectors, self.sample), + 'position': _detector_pixel_positions(detector, self.sample), } diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 31eaf7ec..f2fb21ad 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -64,7 +64,7 @@ def sample_position(self) -> sc.Variable: class NMXData(_SharedFields, sc.DataGroup): @property def weights(self) -> sc.DataArray: - """Event data grouped by pixel id and panel.""" + """Event data grouped by pixel id.""" return self['weights'] @@ -140,7 +140,6 @@ def _create_sample_group(self, nx_entry: h5py.Group) -> h5py.Group: def _create_instrument_group(self, nx_entry: h5py.Group) -> h5py.Group: nx_instrument = nx_entry.create_group("NXinstrument") - nx_instrument.attrs["nr_detector"] = self.origin_position.sizes['panel'] nx_instrument.create_dataset("proton_charge", data=self.proton_charge.value) nx_detector_1 = nx_instrument.create_group("detector_1") @@ -148,9 +147,7 @@ def _create_instrument_group(self, nx_entry: h5py.Group) -> h5py.Group: self._create_compressed_dataset( root_entry=nx_detector_1, name="counts", - var=self.counts.fold( - 'id', sizes={'panel': 1, 'id': self.counts.sizes['id']} - ), + var=self.counts.fold('id', sizes={'id': self.counts.sizes['id']}), ) # Time of arrival bin edges self._create_dataset_from_var( @@ -242,9 +239,7 @@ def bin_time_of_arrival( ) -> NMXReducedData: """Bin time of arrival data into ``time_bin_step`` bins.""" - counts: sc.DataArray = nmx_data.weights.flatten(dims=['panel', 'id'], to='id').hist( - t=time_bin_step - ) + counts: sc.DataArray = nmx_data.weights.hist(t=time_bin_step) counts.unit = 'counts' return NMXReducedData( diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 4ece764f..609b0c73 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -3,10 +3,10 @@ import pathlib from typing import Generator -import numpy as np import pytest import scipp as sc import scippnexus as snx +from scipp.testing import assert_allclose, assert_identical from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample from ess.nmx.mcstas_loader import ( @@ -25,44 +25,32 @@ def check_scalar_properties_mcstas_2(dg: NMXData): Expected numbers are hard-coded based on the sample file. """ - assert dg.proton_charge == sc.scalar(0.15430000000000002, unit=None) - assert sc.identical(dg.crystal_rotation, sc.vector([20, 0, 90], unit='deg')) - assert sc.identical(dg.sample_position, sc.vector(value=[0, 0, 0], unit='m')) - assert sc.identical( + assert_identical(dg.proton_charge, sc.scalar(0.15430000000000002, unit=None)) + assert_identical(dg.crystal_rotation, sc.vector([20, 0, 90], unit='deg')) + assert_identical(dg.sample_position, sc.vector(value=[0, 0, 0], unit='m')) + assert_identical( dg.source_position, sc.vector(value=[-0.53123, 0.0, -157.405], unit='m') ) assert dg.sample_name == sc.scalar("sampleMantid") -def check_nmxdata_properties(dg: NMXData) -> None: +def check_nmxdata_properties(dg: NMXData, fast_axis, slow_axis) -> None: assert isinstance(dg, sc.DataGroup) - assert dg.shape == (3, 1280 * 1280) + assert dg.shape == (1280 * 1280,) # Check maximum value of weights. - assert sc.identical( + assert_identical( dg.weights.max().data, sc.scalar(DefaultMaximumProbability, unit='counts', dtype=float), ) - # Expected values are provided by the IDS - # based on the simulation settings of the sample file. - assert np.all( - np.round(dg.fast_axis.values, 2) - == sc.vectors( - dims=['panel'], - values=[(1.0, 0.0, -0.01), (-0.01, 0.0, -1.0), (0.01, 0.0, 1.0)], - ).values, - ) - assert sc.identical( - dg.slow_axis, - sc.vectors( - dims=['panel'], values=[[0.0, 1.0, 0.0], [0.0, 1.0, 0.0], [0.0, 1.0, 0.0]] - ), - ) + assert_allclose(dg.fast_axis, sc.vector(fast_axis), atol=sc.scalar(0.005)) + assert_identical(dg.slow_axis, sc.vector(slow_axis)) def test_file_reader_mcstas2(mcstas_2_deprecation_warning_context) -> None: with mcstas_2_deprecation_warning_context(): file_path = InputFilepath(small_mcstas_2_sample()) + fast_axis, slow_axis = (1.0, 0.0, -0.01), (0.0, 1.0, 0.0) entry_path = "entry1/data/bank01_events_dat_list_p_x_y_n_id_t" with snx.File(file_path) as file: raw_data = file[entry_path]["events"][()] @@ -76,7 +64,7 @@ def test_file_reader_mcstas2(mcstas_2_deprecation_warning_context) -> None: ) check_scalar_properties_mcstas_2(dg) assert dg.weights.bins.size().sum().value == data_length - check_nmxdata_properties(dg) + check_nmxdata_properties(dg, fast_axis, slow_axis) def check_scalar_properties_mcstas_3(dg: NMXData): @@ -85,17 +73,26 @@ def check_scalar_properties_mcstas_3(dg: NMXData): Expected numbers are hard-coded based on the sample file. """ - assert dg.proton_charge == sc.scalar(0.0167, unit=None) - assert sc.identical(dg.crystal_rotation, sc.vector([0, 0, 0], unit='deg')) - assert sc.identical(dg.sample_position, sc.vector(value=[0, 0, 0], unit='m')) - assert sc.identical( + assert_identical(dg.proton_charge, sc.scalar(0.0167, unit=None)) + assert_identical(dg.crystal_rotation, sc.vector([0, 0, 0], unit='deg')) + assert_identical(dg.sample_position, sc.vector(value=[0, 0, 0], unit='m')) + assert_identical( dg.source_position, sc.vector(value=[-0.53123, 0.0, -157.405], unit='m') ) assert dg.sample_name == sc.scalar("sampleMantid") -@pytest.mark.parametrize('bank_id', ('01', '02', '03')) -def test_file_reader_mcstas3(bank_id) -> None: +@pytest.mark.parametrize( + 'bank_id, fast_axis, slow_axis', + ( + # Expected values are provided by the IDS + # based on the simulation settings of the sample file. + ('01', (1.0, 0.0, -0.01), (0.0, 1.0, 0.0)), + ('02', (-0.01, 0.0, -1.0), (0.0, 1.0, 0.0)), + ('03', (0.01, 0.0, 1.0), (0.0, 1.0, 0.0)), + ), +) +def test_file_reader_mcstas3(bank_id, fast_axis, slow_axis) -> None: file_path = InputFilepath(small_mcstas_3_sample()) entry_paths = [f"entry1/data/bank{bank_id}_events_dat_list_p_x_y_n_id_t"] with snx.File(file_path) as file: @@ -111,7 +108,7 @@ def test_file_reader_mcstas3(bank_id) -> None: ) check_scalar_properties_mcstas_3(dg) assert dg.weights.bins.size().sum().value == data_length - check_nmxdata_properties(dg) + check_nmxdata_properties(dg, fast_axis, slow_axis) @pytest.fixture(params=[small_mcstas_2_sample, small_mcstas_3_sample]) @@ -155,4 +152,4 @@ def test_file_reader_mcstas_additional_fields(tmp_mcstas_file: pathlib.Path) -> ) assert isinstance(dg, sc.DataGroup) - assert dg.shape == (1, 1280 * 1280) + assert dg.shape == (1280 * 1280,) diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index 11695ee7..247c6b5d 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -58,7 +58,6 @@ def test_pipeline_mcstas_loader(mcstas_workflow: sl.Pipeline) -> None: nmx_data = mcstas_workflow.compute(NMXData) assert isinstance(nmx_data, sc.DataGroup) - assert nmx_data.sizes['panel'] == 1 assert nmx_data.sizes['id'] == 1280 * 1280 @@ -68,5 +67,4 @@ def test_pipeline_mcstas_reduction(mcstas_workflow: sl.Pipeline) -> None: nmx_reduced_data = mcstas_workflow.compute(NMXReducedData) assert isinstance(nmx_reduced_data, sc.DataGroup) - assert nmx_reduced_data.sizes['panel'] == 1 assert nmx_reduced_data.sizes['t'] == 50 From ee463c219e8ad80f89670c28711eb47aef61ce6e Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 19 Mar 2024 13:39:41 +0100 Subject: [PATCH 100/403] update fake proton charge in test --- packages/essnmx/tests/loader_test.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 609b0c73..f11ec3ca 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -25,7 +25,10 @@ def check_scalar_properties_mcstas_2(dg: NMXData): Expected numbers are hard-coded based on the sample file. """ - assert_identical(dg.proton_charge, sc.scalar(0.15430000000000002, unit=None)) + assert_identical( + dg.proton_charge, + sc.scalar(1e-4 * dg['weights'].bins.size().sum().data.values, unit=None), + ) assert_identical(dg.crystal_rotation, sc.vector([20, 0, 90], unit='deg')) assert_identical(dg.sample_position, sc.vector(value=[0, 0, 0], unit='m')) assert_identical( @@ -73,7 +76,10 @@ def check_scalar_properties_mcstas_3(dg: NMXData): Expected numbers are hard-coded based on the sample file. """ - assert_identical(dg.proton_charge, sc.scalar(0.0167, unit=None)) + assert_identical( + dg.proton_charge, + sc.scalar(1e-4 * dg['weights'].bins.size().sum().data.values, unit=None), + ) assert_identical(dg.crystal_rotation, sc.vector([0, 0, 0], unit='deg')) assert_identical(dg.sample_position, sc.vector(value=[0, 0, 0], unit='m')) assert_identical( From 869fcd22deb6ddbdd4dbe61b0e1e7d3e4be3e025 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 19 Mar 2024 16:05:21 +0100 Subject: [PATCH 101/403] look for bank and detector association, handle case with and without panels --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 40 +++++++++---- packages/essnmx/src/ess/nmx/mcstas_xml.py | 59 +++++++++++--------- packages/essnmx/tests/loader_test.py | 29 +++++++--- 3 files changed, 84 insertions(+), 44 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index cfac1e69..fe2a8189 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +import re from typing import Callable, NewType, Optional import scipp as sc @@ -103,7 +104,6 @@ def _compose_event_data_array( pixel_ids: All possible pixel IDs of the detector. """ - events = sc.DataArray(data=weights, coords={'t': t_list, 'id': id_list}) return events.group(pixel_ids) @@ -131,6 +131,19 @@ def proton_charge_from_event_data(event_da: sc.DataArray) -> ProtonCharge: return ProtonCharge(_proton_charge_scale_factor * event_da.bins.size().sum().data) +def _bank_names_to_detector_names(file_path): + with snx.File(file_path) as file: + description = file['entry1/instrument/description'][()] + detector_component_regex = r'^COMPONENT (?P.*) = Monitor_nD\(\n(?:(?!COMPONENT)(?!filename)(?:.|\s))*(?:filename = \"(?P[^\"]*)\")?' # noqa: E501 + matches = re.finditer(detector_component_regex, description, re.MULTILINE) + bank_names_to_detector_names = {} + for m in matches: + bank_names_to_detector_names.setdefault(m.group('bank_name'), []).append( + m.group('detector_name') + ) + return bank_names_to_detector_names + + def load_mcstas_nexus( *, file_path: InputFilepath, @@ -173,28 +186,33 @@ def load_mcstas_nexus( from .mcstas_xml import read_mcstas_geometry_xml geometry = read_mcstas_geometry_xml(file_path) - bank_name_to_detector_name = dict( - zip( - (f'bank0{i}' for i in range(1, 4)), - ('nD_Mantid_0', 'nD_Mantid_1', 'nD_Mantid_2'), - ) + + detector_names = next( + det_names + for bank_name, det_names in _bank_names_to_detector_names(file_path).items() + if detector_bank_name in bank_name ) - coords = geometry.to_coords(bank_name_to_detector_name[detector_bank_name]) + coords = geometry.to_coords(*detector_names) with snx.File(file_path) as file: raw_data = _retrieve_raw_event_data(file, detector_bank_name) weights = event_weights_converter( max_probability or DefaultMaximumProbability, - McStasEventProbabilities( - _copy_partial_var(raw_data, idx=0, unit='counts') - ), # p + McStasEventProbabilities(_copy_partial_var(raw_data, idx=0, unit='counts')), ) event_da = _compose_event_data_array( weights=weights, id_list=_copy_partial_var(raw_data, idx=4, dtype='int64'), # id t_list=_copy_partial_var(raw_data, idx=5, unit='s'), # t - pixel_ids=coords.pop('pixel_id'), + pixel_ids=coords.pop('pixel_id'), # p ) + if len(detector_names) > 1: + # If the events come from several detector panels, reshape to reflect that. + # This assumes each panel has the same number of pixels + # and that the pixel_ids associated with each panel consist of one interval. + event_da = event_da.fold( + dim='id', sizes={'panel': len(detector_names), 'id': -1} + ) proton_charge = proton_charge_converter(event_da) crystal_rotation = _retrieve_crystal_rotation( file, geometry.simulation_settings.angle_unit diff --git a/packages/essnmx/src/ess/nmx/mcstas_xml.py b/packages/essnmx/src/ess/nmx/mcstas_xml.py index 82b2cbed..a1a81fc3 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas_xml.py @@ -308,14 +308,13 @@ def from_xml( ) -def _construct_pixel_ids(detector_desc: DetectorDesc) -> sc.Variable: - """Pixel IDs for detector.""" - return sc.arange( - 'id', - detector_desc.id_start, - detector_desc.id_start + detector_desc.total_pixels, - unit=None, - ) +def _construct_pixel_ids(detector_descs: Tuple[DetectorDesc, ...]) -> sc.Variable: + """Pixel IDs for all detectors.""" + intervals = [ + (desc.id_start, desc.id_start + desc.total_pixels) for desc in detector_descs + ] + ids = [sc.arange('id', start, stop, unit=None) for start, stop in intervals] + return sc.concat(ids, 'id') def _pixel_positions( @@ -342,12 +341,15 @@ def _pixel_positions( def _detector_pixel_positions( - detector_desc: DetectorDesc, sample: SampleDesc + detector_descs: Tuple[DetectorDesc, ...], sample: SampleDesc ) -> sc.Variable: - """Position of pixels of detector.""" - return _pixel_positions( - detector_desc, sample.position_from_sample(detector_desc.position) - ) + """Position of pixels of all detectors.""" + + positions = [ + _pixel_positions(detector, sample.position_from_sample(detector.position)) + for detector in detector_descs + ] + return sc.concat(positions, 'panel') @dataclass @@ -373,7 +375,7 @@ def from_xml(cls, tree: _XML) -> 'McStasInstrument': ), ) - def to_coords(self, det_name: str) -> dict[str, sc.Variable]: + def to_coords(self, *det_names: str) -> dict[str, sc.Variable]: """Extract coordinates from the McStas instrument description. Parameters @@ -383,21 +385,28 @@ def to_coords(self, det_name: str) -> dict[str, sc.Variable]: """ - (detector,) = (det for det in self.detectors if det.name == det_name) - slow_axis = detector.slow_axis - fast_axis = detector.fast_axis - origin = self.sample.position_from_sample(detector.position) - - return { - 'pixel_id': _construct_pixel_ids(detector), - 'fast_axis': fast_axis, - 'slow_axis': slow_axis, - 'origin_position': origin, + detectors = tuple(det for det in self.detectors if det.name in det_names) + slow_axes = [det.slow_axis for det in detectors] + fast_axes = [det.fast_axis for det in detectors] + origins = [self.sample.position_from_sample(det.position) for det in detectors] + detector_dim = 'panel' + + coords = { + 'pixel_id': _construct_pixel_ids(detectors), + 'fast_axis': sc.concat(fast_axes, detector_dim), + 'slow_axis': sc.concat(slow_axes, detector_dim), + 'origin_position': sc.concat(origins, detector_dim), 'sample_position': self.sample.position_from_sample(self.sample.position), 'source_position': self.sample.position_from_sample(self.source.position), 'sample_name': sc.scalar(self.sample.name), - 'position': _detector_pixel_positions(detector, self.sample), + 'position': _detector_pixel_positions(detectors, self.sample), } + if len(det_names) == 1: + coords = { + c: sc.squeeze(v, detector_dim) if detector_dim in v.sizes else v + for c, v in coords.items() + } + return coords def read_mcstas_geometry_xml(file_path: Union[Path, str]) -> McStasInstrument: diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index f11ec3ca..2aed1023 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -37,23 +37,37 @@ def check_scalar_properties_mcstas_2(dg: NMXData): assert dg.sample_name == sc.scalar("sampleMantid") -def check_nmxdata_properties(dg: NMXData, fast_axis, slow_axis) -> None: +def check_nmxdata_properties(dg: NMXData, fast_axis, slow_axis, npanels=None) -> None: assert isinstance(dg, sc.DataGroup) - assert dg.shape == (1280 * 1280,) + assert ( + dg.shape + == ( + npanels, + 1280 * 1280, + ) + if npanels + else (1280 * 1280,) + ) # Check maximum value of weights. assert_identical( dg.weights.max().data, sc.scalar(DefaultMaximumProbability, unit='counts', dtype=float), ) - assert_allclose(dg.fast_axis, sc.vector(fast_axis), atol=sc.scalar(0.005)) - assert_identical(dg.slow_axis, sc.vector(slow_axis)) + assert_allclose(dg.fast_axis, fast_axis, atol=sc.scalar(0.005)) + assert_identical(dg.slow_axis, slow_axis) def test_file_reader_mcstas2(mcstas_2_deprecation_warning_context) -> None: with mcstas_2_deprecation_warning_context(): file_path = InputFilepath(small_mcstas_2_sample()) - fast_axis, slow_axis = (1.0, 0.0, -0.01), (0.0, 1.0, 0.0) + fast_axis = sc.vectors( + dims=['panel'], values=((1.0, 0.0, -0.01), (-0.01, 0.0, -1.0), (0.01, 0.0, 1.0)) + ) + slow_axis = sc.vectors( + dims=['panel'], values=((0.0, 1.0, 0.0), (0.0, 1.0, 0.0), (0.0, 1.0, 0.0)) + ) + entry_path = "entry1/data/bank01_events_dat_list_p_x_y_n_id_t" with snx.File(file_path) as file: raw_data = file[entry_path]["events"][()] @@ -67,7 +81,7 @@ def test_file_reader_mcstas2(mcstas_2_deprecation_warning_context) -> None: ) check_scalar_properties_mcstas_2(dg) assert dg.weights.bins.size().sum().value == data_length - check_nmxdata_properties(dg, fast_axis, slow_axis) + check_nmxdata_properties(dg, fast_axis, slow_axis, npanels=3) def check_scalar_properties_mcstas_3(dg: NMXData): @@ -114,7 +128,7 @@ def test_file_reader_mcstas3(bank_id, fast_axis, slow_axis) -> None: ) check_scalar_properties_mcstas_3(dg) assert dg.weights.bins.size().sum().value == data_length - check_nmxdata_properties(dg, fast_axis, slow_axis) + check_nmxdata_properties(dg, sc.vector(fast_axis), sc.vector(slow_axis)) @pytest.fixture(params=[small_mcstas_2_sample, small_mcstas_3_sample]) @@ -158,4 +172,3 @@ def test_file_reader_mcstas_additional_fields(tmp_mcstas_file: pathlib.Path) -> ) assert isinstance(dg, sc.DataGroup) - assert dg.shape == (1280 * 1280,) From 6ac64e86c6e356f25c8de05b7b497286b071e8b5 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 19 Mar 2024 16:07:11 +0100 Subject: [PATCH 102/403] fix --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index fe2a8189..0bb06526 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -204,7 +204,7 @@ def load_mcstas_nexus( weights=weights, id_list=_copy_partial_var(raw_data, idx=4, dtype='int64'), # id t_list=_copy_partial_var(raw_data, idx=5, unit='s'), # t - pixel_ids=coords.pop('pixel_id'), # p + pixel_ids=coords.pop('pixel_id'), ) if len(detector_names) > 1: # If the events come from several detector panels, reshape to reflect that. From 279dbb526544534597f07682c8e1bcd2acbc6b26 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 19 Mar 2024 16:12:35 +0100 Subject: [PATCH 103/403] fix: remove unnecessary reshape --- packages/essnmx/src/ess/nmx/reduction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index f2fb21ad..8c3f921d 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -147,7 +147,7 @@ def _create_instrument_group(self, nx_entry: h5py.Group) -> h5py.Group: self._create_compressed_dataset( root_entry=nx_detector_1, name="counts", - var=self.counts.fold('id', sizes={'id': self.counts.sizes['id']}), + var=self.counts, ) # Time of arrival bin edges self._create_dataset_from_var( From 089696eb2c1098079205e011a47421d00bec3f01 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Wed, 20 Mar 2024 09:44:24 +0100 Subject: [PATCH 104/403] handle case when filename unset and add explanation --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 0bb06526..95f1881a 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -134,13 +134,25 @@ def proton_charge_from_event_data(event_da: sc.DataArray) -> ProtonCharge: def _bank_names_to_detector_names(file_path): with snx.File(file_path) as file: description = file['entry1/instrument/description'][()] - detector_component_regex = r'^COMPONENT (?P.*) = Monitor_nD\(\n(?:(?!COMPONENT)(?!filename)(?:.|\s))*(?:filename = \"(?P[^\"]*)\")?' # noqa: E501 + detector_component_regex = ( + # Start of the detector component definition, contains the detector name. + r'^COMPONENT (?P.*) = Monitor_nD\(\n' + # Some uninteresting lines, we're looking for 'filename'. + # Make sure no new component begins. + r'(?:(?!COMPONENT)(?!filename)(?:.|\s))*' + # The line that defines the filename of the file that stores the + # events associated with the detector. + r'(?:filename = \"(?P[^\"]*)\")?' + ) matches = re.finditer(detector_component_regex, description, re.MULTILINE) bank_names_to_detector_names = {} for m in matches: - bank_names_to_detector_names.setdefault(m.group('bank_name'), []).append( - m.group('detector_name') - ) + bank_names_to_detector_names.setdefault( + # If filename was not set for the detector the filename for the + # event data defaults to the name of the detector. + m.group('bank_name') or m.group('detector_name'), + [], + ).append(m.group('detector_name')) return bank_names_to_detector_names From d9d4097e1c2e7bfb136639a1a5873a72e790f45a Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Fri, 22 Mar 2024 09:09:08 +0100 Subject: [PATCH 105/403] test: tests for description parser --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 24 ++++-- packages/essnmx/tests/loader_test.py | 40 ++++++++- .../tests/mcstas_description_examples.py | 84 +++++++++++++++++++ 3 files changed, 137 insertions(+), 11 deletions(-) create mode 100644 packages/essnmx/tests/mcstas_description_examples.py diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 95f1881a..feb88309 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) import re -from typing import Callable, NewType, Optional +from typing import Callable, Dict, List, NewType, Optional import scipp as sc import scippnexus as snx @@ -131,9 +131,17 @@ def proton_charge_from_event_data(event_da: sc.DataArray) -> ProtonCharge: return ProtonCharge(_proton_charge_scale_factor * event_da.bins.size().sum().data) -def _bank_names_to_detector_names(file_path): +def read_bank_names_to_detector_names(file_path: str) -> Dict[str, List[str]]: with snx.File(file_path) as file: description = file['entry1/instrument/description'][()] + + return bank_names_to_detector_names(description) + + +def bank_names_to_detector_names(description: str) -> Dict[str, List[str]]: + """Associates event data names with the names of the detectors + where the events were detected""" + detector_component_regex = ( # Start of the detector component definition, contains the detector name. r'^COMPONENT (?P.*) = Monitor_nD\(\n' @@ -162,7 +170,7 @@ def load_mcstas_nexus( event_weights_converter: EventWeightsConverter = event_weights_from_probability, proton_charge_converter: ProtonChargeConverter = proton_charge_from_event_data, max_probability: Optional[MaximumProbability] = None, - detector_bank_name: DetectorBankName, + detector_bank_prefix: DetectorBankName, ) -> NMXData: """Load McStas simulation result from h5(nexus) file. @@ -190,8 +198,8 @@ def load_mcstas_nexus( The maximum probability to scale the weights. If not provided, ``DefaultMaximumProbability`` is used. - detector_bank_name: - Name of the detector bank to load events from. + detector_bank_prefix: + Prefix of the detector bank to load events from. """ @@ -201,13 +209,13 @@ def load_mcstas_nexus( detector_names = next( det_names - for bank_name, det_names in _bank_names_to_detector_names(file_path).items() - if detector_bank_name in bank_name + for bank_name, det_names in read_bank_names_to_detector_names(file_path).items() + if detector_bank_prefix in bank_name ) coords = geometry.to_coords(*detector_names) with snx.File(file_path) as file: - raw_data = _retrieve_raw_event_data(file, detector_bank_name) + raw_data = _retrieve_raw_event_data(file, detector_bank_prefix) weights = event_weights_converter( max_probability or DefaultMaximumProbability, McStasEventProbabilities(_copy_partial_var(raw_data, idx=0, unit='counts')), diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 2aed1023..e56a45f0 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) import pathlib +import sys from typing import Generator import pytest @@ -12,12 +13,21 @@ from ess.nmx.mcstas_loader import ( DefaultMaximumProbability, InputFilepath, + bank_names_to_detector_names, event_weights_from_probability, load_mcstas_nexus, proton_charge_from_event_data, ) from ess.nmx.reduction import NMXData +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +from mcstas_description_examples import ( # noqa: E402 + no_detectors, + one_detector_no_filename, + two_detectors_same_filename, + two_detectors_two_filenames, +) + def check_scalar_properties_mcstas_2(dg: NMXData): """Test helper for NMXData loaded from McStas 2. @@ -77,7 +87,7 @@ def test_file_reader_mcstas2(mcstas_2_deprecation_warning_context) -> None: file_path=file_path, event_weights_converter=event_weights_from_probability, proton_charge_converter=proton_charge_from_event_data, - detector_bank_name='bank01', + detector_bank_prefix='bank01', ) check_scalar_properties_mcstas_2(dg) assert dg.weights.bins.size().sum().value == data_length @@ -124,7 +134,7 @@ def test_file_reader_mcstas3(bank_id, fast_axis, slow_axis) -> None: file_path=file_path, event_weights_converter=event_weights_from_probability, proton_charge_converter=proton_charge_from_event_data, - detector_bank_name=f'bank{bank_id}', + detector_bank_prefix=f'bank{bank_id}', ) check_scalar_properties_mcstas_3(dg) assert dg.weights.bins.size().sum().value == data_length @@ -168,7 +178,31 @@ def test_file_reader_mcstas_additional_fields(tmp_mcstas_file: pathlib.Path) -> file_path=InputFilepath(str(tmp_mcstas_file)), event_weights_converter=event_weights_from_probability, proton_charge_converter=proton_charge_from_event_data, - detector_bank_name='bank01', + detector_bank_prefix='bank01', ) assert isinstance(dg, sc.DataGroup) + + +def test_bank_names_to_detector_names_two_detectors(): + res = bank_names_to_detector_names(two_detectors_two_filenames) + assert len(res) == 2 + assert all(len(v) == 1 for v in res.values()) + + +def test_bank_names_to_detector_names_same_filename(): + res = bank_names_to_detector_names(two_detectors_same_filename) + assert len(res) == 1 + assert all(len(v) == 2 for v in res.values()) + + +def test_bank_names_to_detector_names_no_detectors(): + res = bank_names_to_detector_names(no_detectors) + assert len(res) == 0 + + +def test_bank_names_to_detector_names_no_filename(): + res = bank_names_to_detector_names(one_detector_no_filename) + assert len(res) == 1 + ((bank, (detector,)),) = res.items() + assert bank == detector diff --git a/packages/essnmx/tests/mcstas_description_examples.py b/packages/essnmx/tests/mcstas_description_examples.py new file mode 100644 index 00000000..beb9a24e --- /dev/null +++ b/packages/essnmx/tests/mcstas_description_examples.py @@ -0,0 +1,84 @@ +# flake8: noqa + +no_detectors = """ +SPLIT 999 COMPONENT Xtal = Single_crystal( + order = 1, + p_transmit=0.001, + reflections = "Rubredoxin.lau", + xwidth = XtalSize_width, + yheight = XtalSize_height, + zdepth = XtalSize_depth, + mosaic = XtalMosaicity, + delta_d_d=1e-4) + AT (0, 0, deltaz) RELATIVE PREVIOUS + ROTATED (XtalPhiX,XtalPhiY, XtalPhiZ) RELATIVE armSample + EXTEND %{ + if (!SCATTERED) {ABSORB;} + %} + +COMPONENT Sphere = PSD_monitor_4PI( + nx = 360, ny = 360, filename = "4pi", radius = 0.2, + restore_neutron = 1) + +AT (0, 0, deltaz) RELATIVE armSample +""" + +two_detectors_two_filenames = """ +COMPONENT nD_Mantid_0 = Monitor_nD( + options ="mantid square x limits=[0 0.512] bins=1280 y limits=[0 0.512] bins=1280, neutron pixel min=1 t, list all neutrons", + xmin = 0, + xmax = 0.512, + ymin = 0, + ymax = 0.512, + restore_neutron = 1, + filename = "bank01_events.dat") + AT (-0.25, -0.25, 0.29) RELATIVE armSample + ROTATED (0, 0, 0) RELATIVE armSample + +COMPONENT nD_Mantid_1 = Monitor_nD( + options ="mantid square x limits=[0 0.512] bins=1280 y limits=[0 0.512] bins=1280, neutron pixel min=2000000 t, list all neutrons", + xmin = 0, + xmax = 0.512, + ymin = 0, + ymax = 0.512, + restore_neutron = 1, + filename = "bank02_events.dat") + AT (-0.29, -0.25, 0.25) RELATIVE armSample + ROTATED (0, 90, 0) RELATIVE armSample +""" + +one_detector_no_filename = """ +COMPONENT nD_Mantid_2 = Monitor_nD( + options ="mantid square x limits=[0 0.512] bins=1280 y limits=[0 0.512] bins=1280, neutron pixel min=2000000 t, list all neutrons", + xmin = 0, + xmax = 0.512, + ymin = 0, + ymax = 0.512, + restore_neutron = 1, + AT (-0.29, -0.25, 0.25) RELATIVE armSample + ROTATED (0, 90, 0) RELATIVE armSample +""" + +two_detectors_same_filename = """ +COMPONENT nD_Mantid_0 = Monitor_nD( + options ="mantid square x limits=[0 0.512] bins=1280 y limits=[0 0.512] bins=1280, neutron pixel min=1 t, list all neutrons", + xmin = 0, + xmax = 0.512, + ymin = 0, + ymax = 0.512, + restore_neutron = 1, + filename = "bank01_events.dat") + AT (-0.25, -0.25, 0.29) RELATIVE armSample + ROTATED (0, 0, 0) RELATIVE armSample + +COMPONENT nD_Mantid_1 = Monitor_nD( + options ="mantid square x limits=[0 0.512] bins=1280 y limits=[0 0.512] bins=1280, neutron pixel min=2000000 t, list all neutrons", + xmin = 0, + xmax = 0.512, + ymin = 0, + ymax = 0.512, + restore_neutron = 1, + filename = "bank01_events.dat") + AT (-0.29, -0.25, 0.25) RELATIVE armSample + ROTATED (0, 90, 0) RELATIVE armSample +""" From 357b6cbfe5ab117d288553ad6e47e48a3da92f32 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Fri, 22 Mar 2024 09:15:32 +0100 Subject: [PATCH 106/403] remove overloads with same return value --- packages/essnmx/src/ess/nmx/reduction.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 8c3f921d..7d7b182d 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) import io import pathlib -from typing import NewType, Optional, Union, overload +from typing import NewType, Optional, Union import h5py import scipp as sc @@ -194,18 +194,6 @@ def _create_source_group(self, nx_entry: h5py.Group) -> h5py.Group: nx_source["target_material"] = "W" return nx_source - @overload - def export_as_nexus(self, output_file_base: str) -> None: - ... - - @overload - def export_as_nexus(self, output_file_base: pathlib.Path) -> None: - ... - - @overload - def export_as_nexus(self, output_file_base: io.BytesIO) -> None: - ... - def export_as_nexus( self, output_file_base: Union[str, pathlib.Path, io.BytesIO] ) -> None: From 787978c2b3bb8a899d43639890a158d0ec0c75a9 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Fri, 22 Mar 2024 19:03:50 +0100 Subject: [PATCH 107/403] fix: load single detector based on DetectorIndex --- packages/essnmx/docs/examples/workflow.ipynb | 89 +++---- packages/essnmx/src/ess/nmx/__init__.py | 10 +- packages/essnmx/src/ess/nmx/const.py | 5 + packages/essnmx/src/ess/nmx/mcstas_loader.py | 248 +++++++------------ packages/essnmx/src/ess/nmx/mcstas_xml.py | 25 +- packages/essnmx/src/ess/nmx/reduction.py | 21 +- packages/essnmx/src/ess/nmx/types.py | 36 +++ packages/essnmx/tests/loader_test.py | 140 ++++++----- packages/essnmx/tests/workflow_test.py | 37 +-- 9 files changed, 286 insertions(+), 325 deletions(-) create mode 100644 packages/essnmx/src/ess/nmx/const.py create mode 100644 packages/essnmx/src/ess/nmx/types.py diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index a74cd315..0af563e3 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -8,9 +8,8 @@ "In this example, we will use McStas 3 simulation file.\n", "\n", "## Build Pipeline (Collect Parameters and Providers)\n", - "There is a dedicated loader, ``load_mcstas_nexus`` for ``McStas`` simulation data workflow.
\n", - "``MaximumProbability`` can be manually provided to the loader
\n", - "to derive more realistic number of events.
\n", + "Import the providers from ``load_mcstas_nexus`` to use the ``McStas`` simulation data workflow.
\n", + "``MaximumProbability`` can be manually provided to derive more realistic number of events.
\n", "It is because ``weights`` are given as probability, not number of events in a McStas file.
" ] }, @@ -20,48 +19,30 @@ "metadata": {}, "outputs": [], "source": [ - "# Collect parameters and providers\n", - "import scipp as sc\n", - "from ess.nmx.mcstas_loader import load_mcstas_nexus\n", - "from ess.nmx.mcstas_loader import (\n", - " InputFilepath,\n", - " MaximumProbability,\n", - " DefaultMaximumProbability,\n", - " EventWeightsConverter,\n", - " event_weights_from_probability,\n", - " ProtonChargeConverter,\n", - " proton_charge_from_event_data,\n", - " DetectorBankName,\n", - ")\n", - "from ess.nmx.data import small_mcstas_3_sample\n", - "from ess.nmx.reduction import bin_time_of_arrival, TimeBinSteps\n", - "\n", - "providers = (load_mcstas_nexus, bin_time_of_arrival, )\n", + "import sciline as sl\n", "\n", - "file_path = small_mcstas_3_sample() # Replace it with your data file path\n", - "params = {\n", - " TimeBinSteps: TimeBinSteps(50),\n", - " InputFilepath: InputFilepath(file_path),\n", - " # Additional parameters for McStas data handling.\n", - " MaximumProbability: DefaultMaximumProbability,\n", - " EventWeightsConverter: event_weights_from_probability,\n", - " ProtonChargeConverter: proton_charge_from_event_data,\n", - " DetectorBankName: 'bank01',\n", - "}" + "from ess.nmx.mcstas_loader import providers as loader_providers\n", + "from ess.nmx.mcstas_xml import read_mcstas_geometry_xml\n", + "from ess.nmx.data import small_mcstas_3_sample, small_mcstas_2_sample\n", + "from ess.nmx.types import *\n", + "from ess.nmx.reduction import NMXData, NMXReducedData, bin_time_of_arrival, TimeBinSteps\n", + "\n", + "\n", + "pl = sl.Pipeline((*loader_providers, read_mcstas_geometry_xml, bin_time_of_arrival))\n", + "# Replace with the path to your own file\n", + "pl[FilePath] = small_mcstas_3_sample()\n", + "pl[MaximumProbability] = 10000\n", + "pl[TimeBinSteps] = 50\n", + "# DetectorIndex selects what detector panels to include in the run\n", + "# in this case we select all three panels.\n", + "pl.set_param_table(sl.ParamTable(RunID, {DetectorIndex: range(3)}, index=range(3)))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "``event weights converter`` and ``proton_charge_from_event_data`` are\n", - "\n", - "set as parameters for reproducibility of workflow and accessibility to the documentation.\n", - "\n", - "The reason of having them as parameters not as providers is,\n", - "\n", - "1. They are not part of general reduction, which are only for McStas cases.\n", - "2. They are better done while the file is open and read in the loader." + "To see what the pipeline can produce, display it:" ] }, { @@ -70,10 +51,7 @@ "metadata": {}, "outputs": [], "source": [ - "import sciline as sl\n", - "\n", - "nmx_pl = sl.Pipeline(list(providers), params=params)\n", - "nmx_pl" + "pl" ] }, { @@ -89,12 +67,7 @@ "metadata": {}, "outputs": [], "source": [ - "import sciline as sl\n", - "from ess.nmx.mcstas_loader import NMXData\n", - "from ess.nmx.reduction import NMXReducedData\n", - "\n", - "nmx_pl = sl.Pipeline(list(providers), params=params)\n", - "nmx_workflow = nmx_pl.get(NMXReducedData)\n", + "nmx_workflow = pl.get(NMXReducedData)\n", "nmx_workflow.visualize()" ] }, @@ -111,8 +84,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Event data grouped by detector panel and pixel id.\n", - "dg = nmx_workflow.compute(NMXData)\n", + "# Event data grouped by pixel id for each of the selected detectors\n", + "dg = nmx_workflow.compute(sl.Series[RunID, NMXData])\n", "dg" ] }, @@ -122,8 +95,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Binned data.\n", - "\n", + "# Data from all selected detectors binned by panel, pixel and timeslice\n", "binned_dg = nmx_workflow.compute(NMXReducedData)\n", "binned_dg" ] @@ -143,6 +115,15 @@ "```" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "binned_dg.export_as_nexus('test.nxs')" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -166,8 +147,8 @@ "source": [ "import scippneutron as scn\n", "\n", - "da = dg['weights']\n", - "da.coords['position'] = dg['position']\n", + "da = dg[0]['weights']\n", + "da.coords['position'] = dg[0]['position']['panel', 0]\n", "# Plot one out of 100 pixels to reduce size of docs output\n", "view = scn.instrument_view(da['id', ::100].hist(), pixel_size=0.0075)\n", "view.children[0].toolbar.cameraz()\n", diff --git a/packages/essnmx/src/ess/nmx/__init__.py b/packages/essnmx/src/ess/nmx/__init__.py index d3320b5e..6c6373a7 100644 --- a/packages/essnmx/src/ess/nmx/__init__.py +++ b/packages/essnmx/src/ess/nmx/__init__.py @@ -12,14 +12,16 @@ del importlib from .data import small_mcstas_3_sample -from .mcstas_loader import InputFilepath, NMXData, load_mcstas_nexus from .reduction import NMXReducedData, TimeBinSteps +from .types import MaximumProbability + +default_parameters = {MaximumProbability: 10000} + +del MaximumProbability __all__ = [ "small_mcstas_3_sample", - "NMXData", - "InputFilepath", - "load_mcstas_nexus", "NMXReducedData", "TimeBinSteps", + "default_parameters", ] diff --git a/packages/essnmx/src/ess/nmx/const.py b/packages/essnmx/src/ess/nmx/const.py new file mode 100644 index 00000000..12bb34ef --- /dev/null +++ b/packages/essnmx/src/ess/nmx/const.py @@ -0,0 +1,5 @@ +DETECTOR_DIM = 'panel' +DETECTOR_SHAPE = (1280, 1280) +PIXEL_DIM = 'id' +TOF_DIM = 't' +DETECTOR_DIM = 'panel' diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index feb88309..73e916ac 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -1,114 +1,103 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) import re -from typing import Callable, Dict, List, NewType, Optional +from typing import Dict, List import scipp as sc import scippnexus as snx +from .const import PIXEL_DIM, TOF_DIM +from .mcstas_xml import McStasInstrument, read_mcstas_geometry_xml from .reduction import NMXData - -PixelIDs = NewType("PixelIDs", sc.Variable) -InputFilepath = NewType("InputFilepath", str) -DetectorName = NewType("DetectorName", str) -DetectorBankName = NewType("DetectorBankName", str) - -# McStas Configurations -MaximumProbability = NewType("MaximumProbability", int) -DefaultMaximumProbability = MaximumProbability(100_000) - -McStasEventProbabilities = NewType("McStasEventProbabilities", sc.Variable) -EventWeights = NewType("EventWeights", sc.Variable) -EventWeightsConverter = NewType( - "EventWeightsConverter", - Callable[[MaximumProbability, McStasEventProbabilities], EventWeights], +from .types import ( + CrystalRotation, + DetectorBankPrefix, + DetectorIndex, + DetectorName, + EventData, + FilePath, + MaximumProbability, + ProtonCharge, + RawEventData, ) -"""A function that converts McStas probability to event weights.""" -ProtonCharge = NewType("ProtonCharge", sc.Variable) -ProtonChargeConverter = NewType( - "ProtonChargeConverter", Callable[[EventWeights], ProtonCharge] -) -"""A function that derives arbitrary proton charge based on event weights.""" - -def _retrieve_raw_event_data(file: snx.File, bank_name: str) -> sc.Variable: - """Retrieve events from the nexus file.""" - bank_name = f'{bank_name}_events_dat_list_p_x_y_n_id_t' - (bank_name,) = (name for name in file["entry1/data"].keys() if bank_name in name) - return file["entry1/data/" + bank_name]["events"][()].rename_dims( - {'dim_0': 'event'} - ) +def detector_name_from_index(index: DetectorIndex) -> DetectorName: + return f'nD_Mantid_{index}' -def _copy_partial_var( - var: sc.Variable, idx: int, unit: Optional[str] = None, dtype: Optional[str] = None -) -> sc.Variable: - """Retrieve a property from a variable.""" - var = var['dim_1', idx].astype(dtype or var.dtype, copy=True) - if unit is not None: - var.unit = sc.Unit(unit) - return var +def event_data_bank_name( + detector_name: DetectorName, file_path: FilePath +) -> DetectorBankPrefix: + '''Finds the filename associated with a detector''' + for bank_name, det_names in read_bank_names_to_detector_names(file_path).items(): + if detector_name in det_names: + return bank_name.partition('.')[0] -def _retrieve_crystal_rotation(file: snx.File, unit: str) -> sc.Variable: +def raw_event_data( + file_path: FilePath, + bank_prefix: DetectorBankPrefix, + detector_name: DetectorName, + instrument: McStasInstrument, +) -> RawEventData: + """Retrieve events from the nexus file.""" + coords = instrument.to_coords(detector_name) + bank_name = f'{bank_prefix}_dat_list_p_x_y_n_id_t' + with snx.File(file_path, 'r') as f: + root = f["entry1/data"] + (bank_name,) = (name for name in root.keys() if bank_name in name) + data = root[bank_name]["events"][()].rename_dims({'dim_0': 'event'}) + return sc.DataArray( + coords={ + PIXEL_DIM: sc.array( + dims=['event'], + values=data['dim_1', 4].values, + dtype='int64', + unit=None, + ), + TOF_DIM: sc.array( + dims=['event'], values=data['dim_1', 5].values, unit='s' + ), + }, + data=sc.array( + dims=['event'], values=data['dim_1', 0].values, unit='counts' + ), + ).group(coords.pop('pixel_id')) + + +def crystal_rotation( + file_path: FilePath, instrument: McStasInstrument +) -> CrystalRotation: """Retrieve crystal rotation from the file.""" - - return sc.vector( - value=[file[f"entry1/simulation/Param/XtalPhi{key}"][...] for key in "XYZ"], - unit=unit, - ) + with snx.File(file_path, 'r') as file: + return sc.vector( + value=[file[f"entry1/simulation/Param/XtalPhi{key}"][...] for key in "XYZ"], + unit=instrument.simulation_settings.angle_unit, + ) def event_weights_from_probability( - max_probability: MaximumProbability, probabilities: McStasEventProbabilities -) -> EventWeights: + da: RawEventData, + max_probability: MaximumProbability, +) -> EventData: """Create event weights by scaling probability data. event_weights = max_probability * (probabilities / max(probabilities)) Parameters ---------- - probabilities: - The probabilities of the events. + da: + The probabilities of the events max_probability: The maximum probability to scale the weights. """ - maximum_probability = sc.scalar(max_probability, unit='counts') - - return EventWeights(maximum_probability * (probabilities / probabilities.max())) + return sc.scalar(max_probability, unit='counts') * da / da.max() -def _compose_event_data_array( - *, - weights: sc.Variable, - id_list: sc.Variable, - t_list: sc.Variable, - pixel_ids: sc.Variable, -) -> sc.DataArray: - """Combine data with coordinates loaded from the nexus file. - - Parameters - ---------- - weights: - The weights of the events. - - id_list: - The pixel IDs of the events. - - t_list: - The time of arrival of the events. - - pixel_ids: - All possible pixel IDs of the detector. - """ - events = sc.DataArray(data=weights, coords={'t': t_list, 'id': id_list}) - return events.group(pixel_ids) - - -def proton_charge_from_event_data(event_da: sc.DataArray) -> ProtonCharge: +def proton_charge_from_event_data(da: EventData) -> ProtonCharge: """Make up the proton charge from the event data array. Proton charge is proportional to the number of neutrons, @@ -122,19 +111,16 @@ def proton_charge_from_event_data(event_da: sc.DataArray) -> ProtonCharge: Parameters ---------- event_da: - The event data binned in pixel id + The event data """ # Arbitrary number to scale the proton charge - _proton_charge_scale_factor = sc.scalar(1 / 10_000, unit=None) - - return ProtonCharge(_proton_charge_scale_factor * event_da.bins.size().sum().data) + return ProtonCharge(sc.scalar(1 / 10_000, unit=None) * da.bins.size().sum().data) def read_bank_names_to_detector_names(file_path: str) -> Dict[str, List[str]]: with snx.File(file_path) as file: description = file['entry1/instrument/description'][()] - return bank_names_to_detector_names(description) @@ -164,83 +150,31 @@ def bank_names_to_detector_names(description: str) -> Dict[str, List[str]]: return bank_names_to_detector_names -def load_mcstas_nexus( +def load_mcstas( *, - file_path: InputFilepath, - event_weights_converter: EventWeightsConverter = event_weights_from_probability, - proton_charge_converter: ProtonChargeConverter = proton_charge_from_event_data, - max_probability: Optional[MaximumProbability] = None, - detector_bank_prefix: DetectorBankName, + da: EventData, + proton_charge: ProtonCharge, + crystal_rotation: CrystalRotation, + detector_name: DetectorName, + instrument: McStasInstrument, ) -> NMXData: - """Load McStas simulation result from h5(nexus) file. - - See :func:`~event_weights_from_probability` and - :func:`~proton_charge_from_event_data` for details. - - Parameters - ---------- - file_path: - File name to load. - - event_weights_converter: :class:`~EventWeightsConverter`, \ - default: :func:`~event_weights_from_probability` - A function to convert probabilities to event weights. - The function should accept the probabilities as the first argument, - and return the converted event weights. - - proton_charge_converter: :class:`~ProtonChargeConverter`, \ - default: :func:`~proton_charge_from_event_data` - A function to convert the event weights to proton charge. - The function should accept the event weights as the first argument, - and return the proton charge. - - max_probability: - The maximum probability to scale the weights. - If not provided, ``DefaultMaximumProbability`` is used. - - detector_bank_prefix: - Prefix of the detector bank to load events from. - - """ - - from .mcstas_xml import read_mcstas_geometry_xml - - geometry = read_mcstas_geometry_xml(file_path) - - detector_names = next( - det_names - for bank_name, det_names in read_bank_names_to_detector_names(file_path).items() - if detector_bank_prefix in bank_name - ) - coords = geometry.to_coords(*detector_names) - - with snx.File(file_path) as file: - raw_data = _retrieve_raw_event_data(file, detector_bank_prefix) - weights = event_weights_converter( - max_probability or DefaultMaximumProbability, - McStasEventProbabilities(_copy_partial_var(raw_data, idx=0, unit='counts')), - ) - event_da = _compose_event_data_array( - weights=weights, - id_list=_copy_partial_var(raw_data, idx=4, dtype='int64'), # id - t_list=_copy_partial_var(raw_data, idx=5, unit='s'), # t - pixel_ids=coords.pop('pixel_id'), - ) - if len(detector_names) > 1: - # If the events come from several detector panels, reshape to reflect that. - # This assumes each panel has the same number of pixels - # and that the pixel_ids associated with each panel consist of one interval. - event_da = event_da.fold( - dim='id', sizes={'panel': len(detector_names), 'id': -1} - ) - proton_charge = proton_charge_converter(event_da) - crystal_rotation = _retrieve_crystal_rotation( - file, geometry.simulation_settings.angle_unit - ) - + coords = instrument.to_coords(detector_name) + coords.pop('pixel_id') return NMXData( - weights=event_da, + weights=da, proton_charge=proton_charge, crystal_rotation=crystal_rotation, **coords, ) + + +providers = ( + read_mcstas_geometry_xml, + detector_name_from_index, + event_data_bank_name, + raw_event_data, + event_weights_from_probability, + proton_charge_from_event_data, + crystal_rotation, + load_mcstas, +) diff --git a/packages/essnmx/src/ess/nmx/mcstas_xml.py b/packages/essnmx/src/ess/nmx/mcstas_xml.py index a1a81fc3..7e796006 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas_xml.py @@ -2,12 +2,14 @@ # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) # McStas instrument geometry xml description related functions. from dataclasses import dataclass -from pathlib import Path from types import MappingProxyType -from typing import Iterable, Optional, Protocol, Tuple, TypeVar, Union +from typing import Iterable, Optional, Protocol, Tuple, TypeVar import scipp as sc +from .const import DETECTOR_DIM +from .types import FilePath + T = TypeVar('T') @@ -279,7 +281,6 @@ def position_from_sample(self, other: sc.Variable) -> sc.Variable: Position of the other object in 3D vector. """ - return other - self.position @@ -344,12 +345,11 @@ def _detector_pixel_positions( detector_descs: Tuple[DetectorDesc, ...], sample: SampleDesc ) -> sc.Variable: """Position of pixels of all detectors.""" - positions = [ _pixel_positions(detector, sample.position_from_sample(detector.position)) for detector in detector_descs ] - return sc.concat(positions, 'panel') + return sc.concat(positions, DETECTOR_DIM) @dataclass @@ -384,32 +384,25 @@ def to_coords(self, *det_names: str) -> dict[str, sc.Variable]: Names of the detectors to extract coordinates for. """ - detectors = tuple(det for det in self.detectors if det.name in det_names) slow_axes = [det.slow_axis for det in detectors] fast_axes = [det.fast_axis for det in detectors] origins = [self.sample.position_from_sample(det.position) for det in detectors] - detector_dim = 'panel' coords = { 'pixel_id': _construct_pixel_ids(detectors), - 'fast_axis': sc.concat(fast_axes, detector_dim), - 'slow_axis': sc.concat(slow_axes, detector_dim), - 'origin_position': sc.concat(origins, detector_dim), + 'fast_axis': sc.concat(fast_axes, DETECTOR_DIM), + 'slow_axis': sc.concat(slow_axes, DETECTOR_DIM), + 'origin_position': sc.concat(origins, DETECTOR_DIM), 'sample_position': self.sample.position_from_sample(self.sample.position), 'source_position': self.sample.position_from_sample(self.source.position), 'sample_name': sc.scalar(self.sample.name), 'position': _detector_pixel_positions(detectors, self.sample), } - if len(det_names) == 1: - coords = { - c: sc.squeeze(v, detector_dim) if detector_dim in v.sizes else v - for c, v in coords.items() - } return coords -def read_mcstas_geometry_xml(file_path: Union[Path, str]) -> McStasInstrument: +def read_mcstas_geometry_xml(file_path: FilePath) -> McStasInstrument: """Retrieve geometry parameters from mcstas file""" import h5py from defusedxml.ElementTree import fromstring diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 7d7b182d..22528c2b 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -5,8 +5,13 @@ from typing import NewType, Optional, Union import h5py +import sciline import scipp as sc +from .const import DETECTOR_DIM +from .mcstas_xml import McStasInstrument +from .types import DetectorName, RunID + TimeBinSteps = NewType("TimeBinSteps", int) @@ -140,7 +145,7 @@ def _create_sample_group(self, nx_entry: h5py.Group) -> h5py.Group: def _create_instrument_group(self, nx_entry: h5py.Group) -> h5py.Group: nx_instrument = nx_entry.create_group("NXinstrument") - nx_instrument.create_dataset("proton_charge", data=self.proton_charge.value) + nx_instrument.create_dataset("proton_charge", data=self.proton_charge.values) nx_detector_1 = nx_instrument.create_group("detector_1") # Detector counts @@ -223,14 +228,20 @@ def export_as_nexus( def bin_time_of_arrival( - nmx_data: NMXData, time_bin_step: TimeBinSteps + nmx_data: sciline.Series[RunID, NMXData], + detector_name: sciline.Series[RunID, DetectorName], + instrument: McStasInstrument, + time_bin_step: TimeBinSteps, ) -> NMXReducedData: """Bin time of arrival data into ``time_bin_step`` bins.""" - counts: sc.DataArray = nmx_data.weights.hist(t=time_bin_step) - counts.unit = 'counts' + nmx_data = list(nmx_data.values()) + nmx_data = sc.concat(nmx_data, DETECTOR_DIM) + counts = nmx_data.pop('weights').hist(t=time_bin_step) + new_coords = instrument.to_coords(*detector_name.values()) + new_coords.pop('pixel_id') return NMXReducedData( counts=counts, - **{key: nmx_data[key] for key in nmx_data.keys() if key != 'weights'}, + **{**nmx_data, **new_coords}, ) diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py new file mode 100644 index 00000000..1da72152 --- /dev/null +++ b/packages/essnmx/src/ess/nmx/types.py @@ -0,0 +1,36 @@ +from typing import Any, NewType + +import scipp as sc + +FilePath = NewType("FilePath", str) +"""File name of a file containing the results of a McStas run""" + +DetectorIndex = NewType("DetectorIndex", int) +"""Index of the detector to load. Index ordered by the id:s of the pixels""" + +DetectorName = NewType("DetectorName", str) +"""Name of the detector to load""" + +DetectorBankPrefix = NewType("DetectorBankPrefix", str) +"""Prefix identifying the event data array containing +the events from the selected detector""" + +MaximumProbability = NewType("MaximumProbability", int) +"""Maximum number of counts after scaling the event counts""" + +RawEventData = NewType("RawEventData", sc.DataArray) +"""DataArray containing the event counts read from the McStas file, +has coordinates 'id' and 't' """ + +EventData = NewType("EventData", sc.DataArray) +"""The scaled RawEventData""" + +ProtonCharge = NewType("ProtonCharge", sc.Variable) +"""The proton charge signal""" + +CrystalRotation = NewType("CrystalRotation", sc.Variable) +"""Rotation of the crystal""" + +RunID = NewType("RunId", int) + +DetectorGeometry = NewType("DetectorGeometry", Any) diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index e56a45f0..9c68a215 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -5,20 +5,23 @@ from typing import Generator import pytest +import sciline as sl import scipp as sc import scippnexus as snx from scipp.testing import assert_allclose, assert_identical +from ess.nmx import default_parameters +from ess.nmx.const import DETECTOR_DIM from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample -from ess.nmx.mcstas_loader import ( - DefaultMaximumProbability, - InputFilepath, - bank_names_to_detector_names, - event_weights_from_probability, - load_mcstas_nexus, - proton_charge_from_event_data, -) +from ess.nmx.mcstas_loader import bank_names_to_detector_names +from ess.nmx.mcstas_loader import providers as loader_providers from ess.nmx.reduction import NMXData +from ess.nmx.types import ( + DetectorBankPrefix, + DetectorIndex, + FilePath, + MaximumProbability, +) sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) from mcstas_description_examples import ( # noqa: E402 @@ -34,7 +37,6 @@ def check_scalar_properties_mcstas_2(dg: NMXData): Expected numbers are hard-coded based on the sample file. """ - assert_identical( dg.proton_charge, sc.scalar(1e-4 * dg['weights'].bins.size().sum().data.values, unit=None), @@ -47,51 +49,53 @@ def check_scalar_properties_mcstas_2(dg: NMXData): assert dg.sample_name == sc.scalar("sampleMantid") -def check_nmxdata_properties(dg: NMXData, fast_axis, slow_axis, npanels=None) -> None: +def check_nmxdata_properties(dg: NMXData, fast_axis, slow_axis) -> None: assert isinstance(dg, sc.DataGroup) - assert ( - dg.shape - == ( - npanels, - 1280 * 1280, - ) - if npanels - else (1280 * 1280,) - ) + assert dg.shape == (1280 * 1280, 1) # Check maximum value of weights. - assert_identical( + assert_allclose( dg.weights.max().data, - sc.scalar(DefaultMaximumProbability, unit='counts', dtype=float), + sc.scalar(default_parameters[MaximumProbability], unit='counts', dtype=float), + atol=sc.scalar(1e-10, unit='counts'), + rtol=sc.scalar(1e-8), + ) + assert_allclose( + sc.squeeze(dg.fast_axis, DETECTOR_DIM), fast_axis, atol=sc.scalar(0.005) ) - assert_allclose(dg.fast_axis, fast_axis, atol=sc.scalar(0.005)) - assert_identical(dg.slow_axis, slow_axis) + assert_identical(sc.squeeze(dg.slow_axis, DETECTOR_DIM), slow_axis) -def test_file_reader_mcstas2(mcstas_2_deprecation_warning_context) -> None: +@pytest.mark.parametrize( + 'detector_index, fast_axis, slow_axis', + ( + # Expected values are provided by the IDS + # based on the simulation settings of the sample file. + (0, (1.0, 0.0, -0.01), (0.0, 1.0, 0.0)), + (1, (-0.01, 0.0, -1.0), (0.0, 1.0, 0.0)), + (2, (0.01, 0.0, 1.0), (0.0, 1.0, 0.0)), + ), +) +def test_file_reader_mcstas2( + detector_index, fast_axis, slow_axis, mcstas_2_deprecation_warning_context +) -> None: with mcstas_2_deprecation_warning_context(): - file_path = InputFilepath(small_mcstas_2_sample()) - - fast_axis = sc.vectors( - dims=['panel'], values=((1.0, 0.0, -0.01), (-0.01, 0.0, -1.0), (0.01, 0.0, 1.0)) - ) - slow_axis = sc.vectors( - dims=['panel'], values=((0.0, 1.0, 0.0), (0.0, 1.0, 0.0), (0.0, 1.0, 0.0)) + file_path = small_mcstas_2_sample() + + fast_axis = sc.vector(fast_axis) + slow_axis = sc.vector(slow_axis) + + pl = sl.Pipeline( + loader_providers, + params={ + FilePath: file_path, + DetectorIndex: detector_index, + **default_parameters, + }, ) + dg = pl.compute(NMXData) - entry_path = "entry1/data/bank01_events_dat_list_p_x_y_n_id_t" - with snx.File(file_path) as file: - raw_data = file[entry_path]["events"][()] - data_length = raw_data.sizes['dim_0'] - - dg = load_mcstas_nexus( - file_path=file_path, - event_weights_converter=event_weights_from_probability, - proton_charge_converter=proton_charge_from_event_data, - detector_bank_prefix='bank01', - ) check_scalar_properties_mcstas_2(dg) - assert dg.weights.bins.size().sum().value == data_length - check_nmxdata_properties(dg, fast_axis, slow_axis, npanels=3) + check_nmxdata_properties(dg, fast_axis, slow_axis) def check_scalar_properties_mcstas_3(dg: NMXData): @@ -99,7 +103,6 @@ def check_scalar_properties_mcstas_3(dg: NMXData): Expected numbers are hard-coded based on the sample file. """ - assert_identical( dg.proton_charge, sc.scalar(1e-4 * dg['weights'].bins.size().sum().data.values, unit=None), @@ -113,29 +116,33 @@ def check_scalar_properties_mcstas_3(dg: NMXData): @pytest.mark.parametrize( - 'bank_id, fast_axis, slow_axis', + 'detector_index, fast_axis, slow_axis', ( # Expected values are provided by the IDS # based on the simulation settings of the sample file. - ('01', (1.0, 0.0, -0.01), (0.0, 1.0, 0.0)), - ('02', (-0.01, 0.0, -1.0), (0.0, 1.0, 0.0)), - ('03', (0.01, 0.0, 1.0), (0.0, 1.0, 0.0)), + (0, (1.0, 0.0, -0.01), (0.0, 1.0, 0.0)), + (1, (-0.01, 0.0, -1.0), (0.0, 1.0, 0.0)), + (2, (0.01, 0.0, 1.0), (0.0, 1.0, 0.0)), ), ) -def test_file_reader_mcstas3(bank_id, fast_axis, slow_axis) -> None: - file_path = InputFilepath(small_mcstas_3_sample()) - entry_paths = [f"entry1/data/bank{bank_id}_events_dat_list_p_x_y_n_id_t"] +def test_file_reader_mcstas3(detector_index, fast_axis, slow_axis) -> None: + file_path = small_mcstas_3_sample() + + pl = sl.Pipeline( + loader_providers, + params={ + FilePath: file_path, + DetectorIndex: detector_index, + **default_parameters, + }, + ) + dg, bank = pl.compute((NMXData, DetectorBankPrefix)).values() + + entry_path = f"entry1/data/{bank}_dat_list_p_x_y_n_id_t" with snx.File(file_path) as file: - raw_datas = [file[entry_path]["events"][()] for entry_path in entry_paths] - raw_data = sc.concat(raw_datas, dim='dim_0') + raw_data = file[entry_path]["events"][()] data_length = raw_data.sizes['dim_0'] - dg = load_mcstas_nexus( - file_path=file_path, - event_weights_converter=event_weights_from_probability, - proton_charge_converter=proton_charge_from_event_data, - detector_bank_prefix=f'bank{bank_id}', - ) check_scalar_properties_mcstas_3(dg) assert dg.weights.bins.size().sum().value == data_length check_nmxdata_properties(dg, sc.vector(fast_axis), sc.vector(slow_axis)) @@ -174,12 +181,15 @@ def test_file_reader_mcstas_additional_fields(tmp_mcstas_file: pathlib.Path) -> del file[entry_path] file[new_entry_path] = dataset - dg = load_mcstas_nexus( - file_path=InputFilepath(str(tmp_mcstas_file)), - event_weights_converter=event_weights_from_probability, - proton_charge_converter=proton_charge_from_event_data, - detector_bank_prefix='bank01', + pl = sl.Pipeline( + loader_providers, + params={ + FilePath: str(tmp_mcstas_file), + DetectorIndex: 0, + **default_parameters, + }, ) + dg = pl.compute(NMXData) assert isinstance(dg, sc.DataGroup) diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index 247c6b5d..00401f9d 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -4,7 +4,11 @@ import sciline as sl import scipp as sc +from ess.nmx import default_parameters from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample +from ess.nmx.mcstas_loader import providers as load_providers +from ess.nmx.reduction import TimeBinSteps, bin_time_of_arrival +from ess.nmx.types import DetectorIndex, FilePath, RunID @pytest.fixture(params=[small_mcstas_2_sample, small_mcstas_3_sample]) @@ -20,36 +24,21 @@ def mcstas_file_path( @pytest.fixture def mcstas_workflow(mcstas_file_path: str) -> sl.Pipeline: - from ess.nmx.mcstas_loader import ( - DefaultMaximumProbability, - DetectorBankName, - EventWeightsConverter, - InputFilepath, - MaximumProbability, - ProtonChargeConverter, - event_weights_from_probability, - load_mcstas_nexus, - proton_charge_from_event_data, - ) - from ess.nmx.reduction import TimeBinSteps, bin_time_of_arrival - - return sl.Pipeline( - [load_mcstas_nexus, bin_time_of_arrival], + pl = sl.Pipeline( + [*load_providers, bin_time_of_arrival], params={ - InputFilepath: mcstas_file_path, - MaximumProbability: DefaultMaximumProbability, - TimeBinSteps: TimeBinSteps(50), - EventWeightsConverter: event_weights_from_probability, - ProtonChargeConverter: proton_charge_from_event_data, - DetectorBankName: 'bank01', + FilePath: mcstas_file_path, + TimeBinSteps: 50, + DetectorIndex: 0, + **default_parameters, }, ) + pl.set_param_table(sl.ParamTable(RunID, {DetectorIndex: range(3)}, index=range(3))) + return pl def test_pipeline_builder(mcstas_workflow: sl.Pipeline, mcstas_file_path: str) -> None: - from ess.nmx.mcstas_loader import InputFilepath - - assert mcstas_workflow.get(InputFilepath).compute() == mcstas_file_path + assert mcstas_workflow.get(FilePath).compute() == mcstas_file_path def test_pipeline_mcstas_loader(mcstas_workflow: sl.Pipeline) -> None: From 1d0f91ed9a41bd8a36245963a6788774b590bff7 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Mon, 8 Apr 2024 13:51:11 +0200 Subject: [PATCH 108/403] fix --- packages/essnmx/src/ess/nmx/mcstas_xml.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_xml.py b/packages/essnmx/src/ess/nmx/mcstas_xml.py index 7e796006..93f18265 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas_xml.py @@ -389,7 +389,7 @@ def to_coords(self, *det_names: str) -> dict[str, sc.Variable]: fast_axes = [det.fast_axis for det in detectors] origins = [self.sample.position_from_sample(det.position) for det in detectors] - coords = { + return { 'pixel_id': _construct_pixel_ids(detectors), 'fast_axis': sc.concat(fast_axes, DETECTOR_DIM), 'slow_axis': sc.concat(slow_axes, DETECTOR_DIM), @@ -399,7 +399,6 @@ def to_coords(self, *det_names: str) -> dict[str, sc.Variable]: 'sample_name': sc.scalar(self.sample.name), 'position': _detector_pixel_positions(detectors, self.sample), } - return coords def read_mcstas_geometry_xml(file_path: FilePath) -> McStasInstrument: From 48a867abd27ace8021a901ee0d12873f1c32af56 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Mon, 8 Apr 2024 13:55:40 +0200 Subject: [PATCH 109/403] clarification --- packages/essnmx/tests/workflow_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index 00401f9d..35481123 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -29,7 +29,6 @@ def mcstas_workflow(mcstas_file_path: str) -> sl.Pipeline: params={ FilePath: mcstas_file_path, TimeBinSteps: 50, - DetectorIndex: 0, **default_parameters, }, ) @@ -45,6 +44,7 @@ def test_pipeline_mcstas_loader(mcstas_workflow: sl.Pipeline) -> None: """Test if the loader graph is complete.""" from ess.nmx.mcstas_loader import NMXData + mcstas_workflow[DetectorIndex] = 0 nmx_data = mcstas_workflow.compute(NMXData) assert isinstance(nmx_data, sc.DataGroup) assert nmx_data.sizes['id'] == 1280 * 1280 From 896ada803f8ca8db7f98630676ce859532ac1709 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Mon, 8 Apr 2024 14:31:02 +0200 Subject: [PATCH 110/403] fix: move TimeBinSteps to types and update docs --- packages/essnmx/docs/api-reference/index.md | 4 +--- packages/essnmx/docs/examples/workflow.ipynb | 15 +++++---------- packages/essnmx/src/ess/nmx/__init__.py | 4 ++-- packages/essnmx/src/ess/nmx/reduction.py | 10 ++++------ packages/essnmx/src/ess/nmx/types.py | 6 ++++-- packages/essnmx/tests/workflow_test.py | 6 +++--- 6 files changed, 19 insertions(+), 26 deletions(-) diff --git a/packages/essnmx/docs/api-reference/index.md b/packages/essnmx/docs/api-reference/index.md index cf280dd3..17ee733c 100644 --- a/packages/essnmx/docs/api-reference/index.md +++ b/packages/essnmx/docs/api-reference/index.md @@ -12,8 +12,6 @@ NMXData NMXReducedData - InputFilepath - TimeBinSteps ``` @@ -25,7 +23,6 @@ :recursive: small_mcstas_3_sample - load_mcstas_nexus ``` @@ -39,5 +36,6 @@ data mcstas_loader + types ``` diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index 0af563e3..32eee68c 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -25,8 +25,7 @@ "from ess.nmx.mcstas_xml import read_mcstas_geometry_xml\n", "from ess.nmx.data import small_mcstas_3_sample, small_mcstas_2_sample\n", "from ess.nmx.types import *\n", - "from ess.nmx.reduction import NMXData, NMXReducedData, bin_time_of_arrival, TimeBinSteps\n", - "\n", + "from ess.nmx.reduction import NMXData, NMXReducedData, bin_time_of_arrival\n", "\n", "pl = sl.Pipeline((*loader_providers, read_mcstas_geometry_xml, bin_time_of_arrival))\n", "# Replace with the path to your own file\n", @@ -35,7 +34,7 @@ "pl[TimeBinSteps] = 50\n", "# DetectorIndex selects what detector panels to include in the run\n", "# in this case we select all three panels.\n", - "pl.set_param_table(sl.ParamTable(RunID, {DetectorIndex: range(3)}, index=range(3)))" + "pl.set_param_series(DetectorIndex, range(3))" ] }, { @@ -85,7 +84,7 @@ "outputs": [], "source": [ "# Event data grouped by pixel id for each of the selected detectors\n", - "dg = nmx_workflow.compute(sl.Series[RunID, NMXData])\n", + "dg = nmx_workflow.compute(sl.Series[DetectorIndex, NMXData])\n", "dg" ] }, @@ -108,11 +107,7 @@ "\n", "``NMXReducedData`` object has a method to export the data into nexus or h5 file.\n", "\n", - "You can save the result as ``test.nxs`` for example.\n", - "\n", - "```python\n", - "binned_dg.export_as_nexus('test.nxs')\n", - "```" + "You can save the result as ``test.nxs``, for example:\n" ] }, { @@ -121,7 +116,7 @@ "metadata": {}, "outputs": [], "source": [ - "binned_dg.export_as_nexus('test.nxs')" + "#binned_dg.export_as_nexus('test.nxs')" ] }, { diff --git a/packages/essnmx/src/ess/nmx/__init__.py b/packages/essnmx/src/ess/nmx/__init__.py index 6c6373a7..dddbbaec 100644 --- a/packages/essnmx/src/ess/nmx/__init__.py +++ b/packages/essnmx/src/ess/nmx/__init__.py @@ -12,7 +12,7 @@ del importlib from .data import small_mcstas_3_sample -from .reduction import NMXReducedData, TimeBinSteps +from .reduction import NMXData, NMXReducedData from .types import MaximumProbability default_parameters = {MaximumProbability: 10000} @@ -22,6 +22,6 @@ __all__ = [ "small_mcstas_3_sample", "NMXReducedData", - "TimeBinSteps", + "NMXData", "default_parameters", ] diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 22528c2b..fc0bc191 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) import io import pathlib -from typing import NewType, Optional, Union +from typing import Optional, Union import h5py import sciline @@ -10,9 +10,7 @@ from .const import DETECTOR_DIM from .mcstas_xml import McStasInstrument -from .types import DetectorName, RunID - -TimeBinSteps = NewType("TimeBinSteps", int) +from .types import DetectorIndex, DetectorName, TimeBinSteps class _SharedFields(sc.DataGroup): @@ -228,8 +226,8 @@ def export_as_nexus( def bin_time_of_arrival( - nmx_data: sciline.Series[RunID, NMXData], - detector_name: sciline.Series[RunID, DetectorName], + nmx_data: sciline.Series[DetectorIndex, NMXData], + detector_name: sciline.Series[DetectorIndex, DetectorName], instrument: McStasInstrument, time_bin_step: TimeBinSteps, ) -> NMXReducedData: diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 1da72152..e9ed7077 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -31,6 +31,8 @@ CrystalRotation = NewType("CrystalRotation", sc.Variable) """Rotation of the crystal""" -RunID = NewType("RunId", int) - DetectorGeometry = NewType("DetectorGeometry", Any) +"""Description of the geometry of the detector banks""" + +TimeBinSteps = NewType("TimeBinSteps", int) +"""Number of bins in the binning of the time coordinate""" diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index 35481123..e0ea3861 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -7,8 +7,8 @@ from ess.nmx import default_parameters from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample from ess.nmx.mcstas_loader import providers as load_providers -from ess.nmx.reduction import TimeBinSteps, bin_time_of_arrival -from ess.nmx.types import DetectorIndex, FilePath, RunID +from ess.nmx.reduction import bin_time_of_arrival +from ess.nmx.types import DetectorIndex, FilePath, TimeBinSteps @pytest.fixture(params=[small_mcstas_2_sample, small_mcstas_3_sample]) @@ -32,7 +32,7 @@ def mcstas_workflow(mcstas_file_path: str) -> sl.Pipeline: **default_parameters, }, ) - pl.set_param_table(sl.ParamTable(RunID, {DetectorIndex: range(3)}, index=range(3))) + pl.set_param_series(DetectorIndex, range(3)) return pl From 56d4d716bf9714119124f4859cc1d4f2739e51db Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Wed, 10 Apr 2024 16:42:10 +0200 Subject: [PATCH 111/403] fix: use constants consistently --- packages/essnmx/src/ess/nmx/mcstas_xml.py | 8 ++++---- packages/essnmx/src/ess/nmx/reduction.py | 6 +++--- packages/essnmx/tests/loader_test.py | 4 ++-- packages/essnmx/tests/workflow_test.py | 5 +++-- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_xml.py b/packages/essnmx/src/ess/nmx/mcstas_xml.py index 93f18265..b42b1952 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas_xml.py @@ -7,7 +7,7 @@ import scipp as sc -from .const import DETECTOR_DIM +from .const import DETECTOR_DIM, PIXEL_DIM from .types import FilePath T = TypeVar('T') @@ -314,8 +314,8 @@ def _construct_pixel_ids(detector_descs: Tuple[DetectorDesc, ...]) -> sc.Variabl intervals = [ (desc.id_start, desc.id_start + desc.total_pixels) for desc in detector_descs ] - ids = [sc.arange('id', start, stop, unit=None) for start, stop in intervals] - return sc.concat(ids, 'id') + ids = [sc.arange(PIXEL_DIM, start, stop, unit=None) for start, stop in intervals] + return sc.concat(ids, PIXEL_DIM) def _pixel_positions( @@ -325,7 +325,7 @@ def _pixel_positions( Position of each pixel is relative to the position_offset. """ - pixel_idx = sc.arange('id', detector.total_pixels) + pixel_idx = sc.arange(PIXEL_DIM, detector.total_pixels) n_col = sc.scalar(detector.num_fast_pixels_per_row) pixel_n_slow = pixel_idx // n_col diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index fc0bc191..d0d92a88 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -8,7 +8,7 @@ import sciline import scipp as sc -from .const import DETECTOR_DIM +from .const import DETECTOR_DIM, PIXEL_DIM, TOF_DIM from .mcstas_xml import McStasInstrument from .types import DetectorIndex, DetectorName, TimeBinSteps @@ -155,7 +155,7 @@ def _create_instrument_group(self, nx_entry: h5py.Group) -> h5py.Group: # Time of arrival bin edges self._create_dataset_from_var( root_entry=nx_detector_1, - var=self.counts.coords['t'], + var=self.counts.coords[TOF_DIM], name="t_bin", long_name="t_bin TOF (ms)", ) @@ -163,7 +163,7 @@ def _create_instrument_group(self, nx_entry: h5py.Group) -> h5py.Group: self._create_compressed_dataset( root_entry=nx_detector_1, name="pixel_id", - var=self.counts.coords['id'], + var=self.counts.coords[PIXEL_DIM], long_name="pixel ID", ) return nx_instrument diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 9c68a215..527943bf 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -11,7 +11,7 @@ from scipp.testing import assert_allclose, assert_identical from ess.nmx import default_parameters -from ess.nmx.const import DETECTOR_DIM +from ess.nmx.const import DETECTOR_DIM, DETECTOR_SHAPE from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample from ess.nmx.mcstas_loader import bank_names_to_detector_names from ess.nmx.mcstas_loader import providers as loader_providers @@ -51,7 +51,7 @@ def check_scalar_properties_mcstas_2(dg: NMXData): def check_nmxdata_properties(dg: NMXData, fast_axis, slow_axis) -> None: assert isinstance(dg, sc.DataGroup) - assert dg.shape == (1280 * 1280, 1) + assert dg.shape == (DETECTOR_SHAPE[0] * DETECTOR_SHAPE[1], 1) # Check maximum value of weights. assert_allclose( dg.weights.max().data, diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index e0ea3861..d29f29e5 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -5,6 +5,7 @@ import scipp as sc from ess.nmx import default_parameters +from ess.nmx.const import DETECTOR_SHAPE, PIXEL_DIM, TOF_DIM from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample from ess.nmx.mcstas_loader import providers as load_providers from ess.nmx.reduction import bin_time_of_arrival @@ -47,7 +48,7 @@ def test_pipeline_mcstas_loader(mcstas_workflow: sl.Pipeline) -> None: mcstas_workflow[DetectorIndex] = 0 nmx_data = mcstas_workflow.compute(NMXData) assert isinstance(nmx_data, sc.DataGroup) - assert nmx_data.sizes['id'] == 1280 * 1280 + assert nmx_data.sizes[PIXEL_DIM] == DETECTOR_SHAPE[0] * DETECTOR_SHAPE[1] def test_pipeline_mcstas_reduction(mcstas_workflow: sl.Pipeline) -> None: @@ -56,4 +57,4 @@ def test_pipeline_mcstas_reduction(mcstas_workflow: sl.Pipeline) -> None: nmx_reduced_data = mcstas_workflow.compute(NMXReducedData) assert isinstance(nmx_reduced_data, sc.DataGroup) - assert nmx_reduced_data.sizes['t'] == 50 + assert nmx_reduced_data.sizes[TOF_DIM] == 50 From 5f3514479049f97c946c48b429f6f98992b0c0a3 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 11 Apr 2024 17:33:35 +0200 Subject: [PATCH 112/403] Add MTZ sample files. --- packages/essnmx/src/ess/nmx/data/__init__.py | 13 +++++++--- packages/essnmx/src/ess/nmx/mtz_io.py | 27 ++++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 packages/essnmx/src/ess/nmx/mtz_io.py diff --git a/packages/essnmx/src/ess/nmx/data/__init__.py b/packages/essnmx/src/ess/nmx/data/__init__.py index 0ac139ff..8fec7b37 100644 --- a/packages/essnmx/src/ess/nmx/data/__init__.py +++ b/packages/essnmx/src/ess/nmx/data/__init__.py @@ -1,14 +1,13 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +import pooch _version = '0' __all__ = ['small_mcstas_2_sample', 'small_mcstas_3_sample', 'get_path'] -def _make_pooch(): - import pooch - +def _make_pooch() -> pooch.Pooch: return pooch.create( path=pooch.os_cache('essnmx'), env='ESSNMX_DATA_DIR', @@ -18,6 +17,7 @@ def _make_pooch(): registry={ 'small_mcstas_2_sample.h5': 'md5:c3affe636397f8c9eea1d9c10a2bf487', 'small_mcstas_3_sample.h5': 'md5:2afaac205d13ee857ee5364e3f1957a7', + 'mtz_samples.tar.gz': 'md5:bed1eaf604bbe8725c1f6a20ca79fcc0', }, ) @@ -57,3 +57,10 @@ def get_path(name: str) -> str: paths to custom files. """ return _pooch.fetch(name) + + +def get_small_mtz_samples() -> list[str]: + """Return a list of path to MTZ sample files.""" + from pooch.processors import Untar + + return _pooch.fetch('mtz_samples.tar.gz', processor=Untar()) diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py new file mode 100644 index 00000000..93c5c5cc --- /dev/null +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +from typing import NewType + +import gemmi +import numpy as np +import pandas as pd + +FileName = NewType("FileName", str) + +MTZFilepath = NewType("MTZFilepath", str) +RawMtz = NewType("RawMtz", gemmi.Mtz) +MtzDataFrame = NewType("MtzDataFrame", pd.DataFrame) + + +def read_mtz_file(file_path: MTZFilepath) -> RawMtz: + '''read mtz file''' + + return RawMtz(gemmi.read_mtz_file(file_path)) + + +def mtz_to_pandas(mtz: RawMtz) -> MtzDataFrame: + return MtzDataFrame( + pd.DataFrame( # Recommended in the gemmi documentation. + data=np.array(mtz, copy=False), columns=mtz.column_labels() + ) + ) From d6a1e6b6d5597f3615a747c42213fdcd91365e96 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 11 Apr 2024 17:34:19 +0200 Subject: [PATCH 113/403] MTZ IO wrapper. --- packages/essnmx/src/ess/nmx/mtz_io.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index 93c5c5cc..2241695c 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -6,11 +6,14 @@ import numpy as np import pandas as pd -FileName = NewType("FileName", str) - MTZFilepath = NewType("MTZFilepath", str) +"""Path to the mtz file""" + RawMtz = NewType("RawMtz", gemmi.Mtz) +"""The mtz file as a gemmi object""" + MtzDataFrame = NewType("MtzDataFrame", pd.DataFrame) +"""The mtz file as a pandas DataFrame""" def read_mtz_file(file_path: MTZFilepath) -> RawMtz: From 266db1df2ba6b85775d25320d8f196c40dcc578f Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 11 Apr 2024 17:34:42 +0200 Subject: [PATCH 114/403] MTZ IO helper example. --- packages/essnmx/docs/examples/scaling.ipynb | 54 +++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 packages/essnmx/docs/examples/scaling.ipynb diff --git a/packages/essnmx/docs/examples/scaling.ipynb b/packages/essnmx/docs/examples/scaling.ipynb new file mode 100644 index 00000000..70f7be38 --- /dev/null +++ b/packages/essnmx/docs/examples/scaling.ipynb @@ -0,0 +1,54 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Scaling\n", + "\n", + "## MTZ IO\n", + "\n", + "``ess.nmx`` has ``MTZ`` IO helper functions.\n", + "They can be used as providers in a workflow of scaling routine.\n", + "\n", + "They are wrapping ``MTZ`` IO functions of ``gemmi``." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ess.nmx.mtz_io import read_mtz_file, mtz_to_pandas\n", + "from ess.nmx.data import get_small_mtz_samples\n", + "\n", + "small_mtz_sample = get_small_mtz_samples()[0]\n", + "mtz = read_mtz_file(small_mtz_sample)\n", + "df = mtz_to_pandas(mtz)\n", + "df.head()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nmx-dev-310", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 451396bab7b12a8fc25f10fafc6ef4d4f5b0b670 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 11 Apr 2024 17:53:46 +0200 Subject: [PATCH 115/403] Add MTZ IO tests. --- packages/essnmx/tests/mtz_io_test.py | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 packages/essnmx/tests/mtz_io_test.py diff --git a/packages/essnmx/tests/mtz_io_test.py b/packages/essnmx/tests/mtz_io_test.py new file mode 100644 index 00000000..3bc2218b --- /dev/null +++ b/packages/essnmx/tests/mtz_io_test.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +import gemmi +import pytest + +from ess.nmx.data import get_small_mtz_samples +from ess.nmx.mtz_io import mtz_to_pandas, read_mtz_file + + +@pytest.fixture(params=get_small_mtz_samples()) +def file_path(request) -> str: + return request.param + + +def test_gemmi_mtz(file_path: str) -> None: + mtz = read_mtz_file(file_path) + assert mtz.spacegroup == gemmi.SpaceGroup('C 1 2 1') # Hard-coded value + assert len(mtz.columns[0]) == 100 # Number of samples, hard-coded value + + +@pytest.fixture +def gemmi_mtz_object(file_path: str) -> gemmi.Mtz: + return read_mtz_file(file_path) + + +def test_mtz_to_pandas_dataframe(gemmi_mtz_object: gemmi.Mtz) -> None: + df = mtz_to_pandas(gemmi_mtz_object) + assert set(df.columns) == set(gemmi_mtz_object.column_labels()) + # Check if the test data are not all-same + first_column_name, second_column_name = df.columns[0:2] + assert not all(df[first_column_name] == df[second_column_name]) + + # Check if the data are the same + for column in gemmi_mtz_object.columns: + assert column.label in df.columns + assert all(df[column.label] == column.array) From 1d88c5a084c584b7f3f8d6731e2f6ea309fe2972 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Fri, 12 Apr 2024 16:01:16 +0200 Subject: [PATCH 116/403] Add mtz reducing and merging method. --- packages/essnmx/docs/examples/index.md | 1 + packages/essnmx/docs/examples/scaling.ipynb | 54 ----- .../docs/examples/scaling_workflow.ipynb | 130 ++++++++++++ packages/essnmx/src/ess/nmx/mtz_io.py | 186 +++++++++++++++++- packages/essnmx/tests/mtz_io_test.py | 118 ++++++++++- 5 files changed, 424 insertions(+), 65 deletions(-) delete mode 100644 packages/essnmx/docs/examples/scaling.ipynb create mode 100644 packages/essnmx/docs/examples/scaling_workflow.ipynb diff --git a/packages/essnmx/docs/examples/index.md b/packages/essnmx/docs/examples/index.md index 76ac4ef0..9c3f900d 100644 --- a/packages/essnmx/docs/examples/index.md +++ b/packages/essnmx/docs/examples/index.md @@ -6,4 +6,5 @@ maxdepth: 2 --- workflow +scaling_workflow ``` diff --git a/packages/essnmx/docs/examples/scaling.ipynb b/packages/essnmx/docs/examples/scaling.ipynb deleted file mode 100644 index 70f7be38..00000000 --- a/packages/essnmx/docs/examples/scaling.ipynb +++ /dev/null @@ -1,54 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Scaling\n", - "\n", - "## MTZ IO\n", - "\n", - "``ess.nmx`` has ``MTZ`` IO helper functions.\n", - "They can be used as providers in a workflow of scaling routine.\n", - "\n", - "They are wrapping ``MTZ`` IO functions of ``gemmi``." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from ess.nmx.mtz_io import read_mtz_file, mtz_to_pandas\n", - "from ess.nmx.data import get_small_mtz_samples\n", - "\n", - "small_mtz_sample = get_small_mtz_samples()[0]\n", - "mtz = read_mtz_file(small_mtz_sample)\n", - "df = mtz_to_pandas(mtz)\n", - "df.head()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "nmx-dev-310", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb new file mode 100644 index 00000000..72a3cce7 --- /dev/null +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -0,0 +1,130 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Scaling\n", + "\n", + "## MTZ IO\n", + "\n", + "``ess.nmx`` has ``MTZ`` IO helper functions.\n", + "They can be used as providers in a workflow of scaling routine.\n", + "\n", + "They are wrapping ``MTZ`` IO functions of ``gemmi``." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ess.nmx.mtz_io import read_mtz_file, mtz_to_pandas, MTZFilepath\n", + "from ess.nmx.data import get_small_mtz_samples\n", + "\n", + "small_mtz_sample = get_small_mtz_samples()[0]\n", + "mtz = read_mtz_file(MTZFilepath(small_mtz_sample))\n", + "df = mtz_to_pandas(mtz)\n", + "df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Build Pipeline\n", + "\n", + "Scaling routine includes:\n", + "- Reducing individual MTZ dataset\n", + "- Merging MTZ dataset \n", + "- Reducing merged MTZ dataset\n", + "\n", + "These operations are done on pandas dataframe as recommended in ``gemmi``.\n", + "And multiple MTZ files are expected, so we need to use ``sciline.ParamTable``.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sciline as sl\n", + "from ess.nmx.mtz_io import mtz_io_providers\n", + "from ess.nmx.mtz_io import MTZFilepath, MTZFileIndex, NMXMtzDataFrame, SpaceGroupDesc\n", + "\n", + "pl = sl.Pipeline(\n", + " providers=mtz_io_providers,\n", + " params={\n", + " SpaceGroupDesc: \"C 1 2 1\"\n", + " # Replace with the correct space group if needed\n", + " },\n", + ")\n", + "\n", + "file_path_table = sl.ParamTable(\n", + " row_dim=MTZFileIndex, columns={MTZFilepath: get_small_mtz_samples()}\n", + ")\n", + "\n", + "pl.set_param_table(file_path_table)\n", + "pl" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Build Workflow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "scaling_nmx_workflow = pl.get(NMXMtzDataFrame)\n", + "scaling_nmx_workflow.visualize(graph_attr={\"rankdir\": \"LR\"})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute Desired Type" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "merged_df = scaling_nmx_workflow.compute(NMXMtzDataFrame)\n", + "merged_df.head()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nmx-dev-310", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index 2241695c..5c0149e1 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -1,30 +1,202 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) -from typing import NewType +from typing import NewType, Optional import gemmi import numpy as np import pandas as pd +import sciline as sl +# Index types for param table. +MTZFileIndex = NewType("MTZFileIndex", int) +"""The index of the mtz file when iterating over multiple mtz files.""" + +# User defined or configurable types MTZFilepath = NewType("MTZFilepath", str) """Path to the mtz file""" +LambdaBinSize = NewType("LambdaBinSize", int) +"""The number of bins to use for binning wavelength(lambda).""" +SpaceGroupDesc = NewType("SpaceGroupDesc", str) +"""The space group description. e.g. 'P 21 21 21'""" +DEFAULT_SPACE_GROUP_DESC = SpaceGroupDesc("P 21 21 21") +"""The default space group description to use if not found in the mtz files.""" +# Computed types RawMtz = NewType("RawMtz", gemmi.Mtz) """The mtz file as a gemmi object""" - -MtzDataFrame = NewType("MtzDataFrame", pd.DataFrame) -"""The mtz file as a pandas DataFrame""" +RawMtzDataFrame = NewType("RawMtzDataFrame", pd.DataFrame) +"""The raw mtz dataframe.""" +SpaceGroup = NewType("SpaceGroup", gemmi.SpaceGroup) +"""The space group.""" +RapioAsu = NewType("RapioAsu", gemmi.ReciprocalAsu) +"""The reciprocal asymmetric unit.""" +MergedMtzDataFrame = NewType("MergedMtzDataFrame", pd.DataFrame) +"""The merged mtz dataframe with derived columns.""" +NMXMtzDataFrame = NewType("NMXMtzDataFrame", pd.DataFrame) +"""The reduced mtz dataframe with derived columns.""" def read_mtz_file(file_path: MTZFilepath) -> RawMtz: - '''read mtz file''' + """read mtz file""" return RawMtz(gemmi.read_mtz_file(file_path)) -def mtz_to_pandas(mtz: RawMtz) -> MtzDataFrame: - return MtzDataFrame( +def mtz_to_pandas(mtz: gemmi.Mtz) -> pd.DataFrame: + """Converts the mtz file to a pandas dataframe. + + It is equivalent to the following code: + ```python + import numpy as np + import pandas as pd + + data = np.array(mtz, copy=False) + columns = mtz.column_labels() + return pd.DataFrame(data, columns=columns) + ``` + It is recommended in the gemmi documentation. + + """ + + return RawMtzDataFrame( pd.DataFrame( # Recommended in the gemmi documentation. data=np.array(mtz, copy=False), columns=mtz.column_labels() ) ) + + +def reduce_single_mtz(mtz: RawMtz) -> RawMtzDataFrame: + """Select and derive columns from the original ``MtzDataFrame``. + + Parameters + ---------- + mtz: + The raw mtz dataset. + + Returns + ------- + : + The new mtz dataframe with derived columns. + The derived columns are: + + Notes + ----- + :class:`pandas.DataFrame` is the data structure + that the rest of the steps are using, + but :class:`gemmi.Mtz` has :func:`gemmi.Mtz:calculate_d` + that can derive the ``d`` using ``HKL``. + This part of the method must be called on each mtz file separately. + + """ + from .mtz_io import mtz_to_pandas + + orig_df = mtz_to_pandas(mtz) + mtz_df = pd.DataFrame() + + # HKL should always be integer. + mtz_df[["H", "K", "L"]] = orig_df[["H", "K", "L"]].astype(int) + mtz_df["hkl"] = mtz_df[["H", "K", "L"]].values.tolist() + + def _calculate_d(row: pd.Series) -> float: + return mtz.get_cell().calculate_d(row["hkl"]) + + mtz_df["d"] = mtz_df.apply(_calculate_d, axis=1) + # $(2d)^{-2} = \sin^2(\theta)/\lambda^2 + mtz_df["resolution"] = (1 / mtz_df["d"]) ** 2 / 4 + + mtz_df["I_div_SIGI"] = orig_df["I"] / orig_df["SIGI"] + + return RawMtzDataFrame(mtz_df) + + +def get_space_group( + mtzs: sl.Series[MTZFileIndex, RawMtz], + spacegroup_desc: Optional[SpaceGroupDesc] = None, +) -> SpaceGroup: + """Retrieves spacegroup from file or uses parameter. + + Manually provided space group description is prioritized over + space group descriptions found in the mtz files. + Spacegroup is always expected in any MTZ files, but it may be missing. + + Parameters + ---------- + mtzs: + A series of raw mtz datasets. + + spacegroup_desc: + The space group description to use if not found in the mtz files. + If None, ``DEFAULT_SPACE_GROUP_DESC`` is used. + + Returns + ------- + SpaceGroup + The space group. + + Raises + ------ + ValueError + If multiple or no space groups are found + but space group description is not provided. + + """ + space_groups = { + sgrp.short_name(): sgrp + for mtz in mtzs.values() + if (sgrp := mtz.spacegroup) is not None + } + if spacegroup_desc is not None: # Use the provided space group description + return SpaceGroup(gemmi.SpaceGroup(spacegroup_desc)) + elif len(space_groups) > 1: + raise ValueError(f"Multiple space groups found: {space_groups}") + elif len(space_groups) == 1: + return SpaceGroup(space_groups.popitem()[1]) + else: + raise ValueError( + "No space group found and no space group description provided." + ) + + +def get_reciprocal_asu(spacegroup: SpaceGroup) -> RapioAsu: + """Returns the reciprocal asymmetric unit from the space group.""" + + return RapioAsu(gemmi.ReciprocalAsu(spacegroup)) + + +def merge_mtz_dataframes( + mtz_dfs: sl.Series[MTZFileIndex, RawMtzDataFrame], +) -> MergedMtzDataFrame: + """Merge multiple mtz dataframes into one.""" + + return MergedMtzDataFrame(pd.concat(mtz_dfs.values(), ignore_index=True)) + + +def reduce_merged_mtz_dataframe( + *, + merged_mtz_df: MergedMtzDataFrame, + rapio_asu: RapioAsu, + sg: SpaceGroup, +) -> NMXMtzDataFrame: + """Reduces the shallow copy of a merged mtz dataframes. + + This method must be called after merging multiple mtz dataframes. + """ + merged_df = merged_mtz_df.copy(deep=False) + + def _rapio_asu_to_asu(row: pd.Series) -> list[int]: + return rapio_asu.to_asu(row["hkl"], sg.operations())[0] + + merged_df["hkl_eq"] = merged_df.apply(_rapio_asu_to_asu, axis=1) + + return NMXMtzDataFrame(merged_df) + + +mtz_io_providers = ( + read_mtz_file, + reduce_single_mtz, + get_space_group, + get_reciprocal_asu, + merge_mtz_dataframes, + reduce_merged_mtz_dataframe, +) +"""The providers related to the MTZ IO.""" diff --git a/packages/essnmx/tests/mtz_io_test.py b/packages/essnmx/tests/mtz_io_test.py index 3bc2218b..0f6c9153 100644 --- a/packages/essnmx/tests/mtz_io_test.py +++ b/packages/essnmx/tests/mtz_io_test.py @@ -2,9 +2,23 @@ # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) import gemmi import pytest +import sciline as sl from ess.nmx.data import get_small_mtz_samples -from ess.nmx.mtz_io import mtz_to_pandas, read_mtz_file +from ess.nmx.mtz_io import DEFAULT_SPACE_GROUP_DESC # P 21 21 21 +from ess.nmx.mtz_io import ( + MergedMtzDataFrame, + MTZFileIndex, + MTZFilepath, + RawMtz, + get_reciprocal_asu, + get_space_group, + merge_mtz_dataframes, + mtz_to_pandas, + read_mtz_file, + reduce_merged_mtz_dataframe, + reduce_single_mtz, +) @pytest.fixture(params=get_small_mtz_samples()) @@ -13,14 +27,14 @@ def file_path(request) -> str: def test_gemmi_mtz(file_path: str) -> None: - mtz = read_mtz_file(file_path) - assert mtz.spacegroup == gemmi.SpaceGroup('C 1 2 1') # Hard-coded value + mtz = read_mtz_file(MTZFilepath(file_path)) + assert mtz.spacegroup == gemmi.SpaceGroup("C 1 2 1") # Hard-coded value assert len(mtz.columns[0]) == 100 # Number of samples, hard-coded value @pytest.fixture def gemmi_mtz_object(file_path: str) -> gemmi.Mtz: - return read_mtz_file(file_path) + return read_mtz_file(MTZFilepath(file_path)) def test_mtz_to_pandas_dataframe(gemmi_mtz_object: gemmi.Mtz) -> None: @@ -34,3 +48,99 @@ def test_mtz_to_pandas_dataframe(gemmi_mtz_object: gemmi.Mtz) -> None: for column in gemmi_mtz_object.columns: assert column.label in df.columns assert all(df[column.label] == column.array) + + +def test_mtz_to_reduced_pandas_dataframe(gemmi_mtz_object: gemmi.Mtz) -> None: + df = reduce_single_mtz(RawMtz(gemmi_mtz_object)) + for expected_colum in ["hkl", "d", "resolution", "I_div_SIGI", *"HKL"]: + assert expected_colum in df.columns + + for hkl_column in "HKL": + assert hkl_column in df.columns + assert df[hkl_column].dtype == int + + assert "hkl_eq" not in df.columns # It should be done on merged dataframes + + +@pytest.fixture +def mtz_series() -> sl.Series[MTZFileIndex, RawMtz]: + return sl.Series( + row_dim=MTZFileIndex, + items={ + MTZFileIndex(i_file): read_mtz_file(MTZFilepath(file_path)) + for i_file, file_path in enumerate(get_small_mtz_samples()) + }, + ) + + +def test_get_space_group(mtz_series: sl.Series[MTZFileIndex, RawMtz]) -> None: + assert ( + get_space_group(mtz_series).short_name() == "C2" + ) # Expected value in test files + + +def test_get_space_group_with_spacegroup_desc( + mtz_series: sl.Series[MTZFileIndex, RawMtz], +) -> None: + assert ( + get_space_group(mtz_series, DEFAULT_SPACE_GROUP_DESC).short_name() == "P212121" + ) + + +@pytest.fixture +def conflicting_mtz_series( + mtz_series: sl.Series[MTZFileIndex, RawMtz], +) -> sl.Series[MTZFileIndex, RawMtz]: + mtz_series[MTZFileIndex(0)].spacegroup = gemmi.SpaceGroup(DEFAULT_SPACE_GROUP_DESC) + # Make sure the space groups are different + assert ( + mtz_series[MTZFileIndex(0)].spacegroup.short_name() + != mtz_series[MTZFileIndex(1)].spacegroup.short_name() + ) + + return mtz_series + + +def test_get_space_group_conflict_raises( + conflicting_mtz_series: sl.Series[MTZFileIndex, RawMtz], +) -> None: + reg = r"Multiple space groups found:.+P 21 21 21.+C 1 2 1" + with pytest.raises(ValueError, match=reg): + get_space_group(conflicting_mtz_series) + + +def test_get_space_conflict_but_desc_provided( + conflicting_mtz_series: sl.Series[MTZFileIndex, RawMtz], +) -> None: + assert ( + get_space_group(conflicting_mtz_series, DEFAULT_SPACE_GROUP_DESC).short_name() + == "P212121" + ) + + +@pytest.fixture +def merged_mtz_dataframe( + mtz_series: sl.Series[MTZFileIndex, RawMtz], +) -> MergedMtzDataFrame: + """Tests if the merged data frame has the expected columns.""" + reduced_mtz_series = sl.Series( + row_dim=MTZFileIndex, + items={i_file: reduce_single_mtz(mtz) for i_file, mtz in mtz_series.items()}, + ) + return merge_mtz_dataframes(reduced_mtz_series) + + +def test_reduce_merged_mtz_dataframe( + mtz_series: sl.Series[MTZFileIndex, RawMtz], + merged_mtz_dataframe: MergedMtzDataFrame, +) -> None: + space_gr = get_space_group(mtz_series) + rapio_asu = get_reciprocal_asu(space_gr) + + nmx_df = reduce_merged_mtz_dataframe( + merged_mtz_df=merged_mtz_dataframe, + rapio_asu=rapio_asu, + sg=space_gr, + ) + assert "hkl_eq" not in merged_mtz_dataframe.columns + assert "hkl_eq" in nmx_df.columns From 87d659d666ef23dc183921e6d71266bef62edecb Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Fri, 12 Apr 2024 16:46:37 +0200 Subject: [PATCH 117/403] Update path type to Path instead of str --- .../docs/examples/scaling_workflow.ipynb | 9 ++--- packages/essnmx/src/ess/nmx/data/__init__.py | 35 +++++++++++-------- packages/essnmx/src/ess/nmx/mtz_io.py | 9 +++-- packages/essnmx/tests/mtz_io_test.py | 16 +++++---- 4 files changed, 38 insertions(+), 31 deletions(-) diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index 72a3cce7..af923a20 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -20,11 +20,12 @@ "metadata": {}, "outputs": [], "source": [ - "from ess.nmx.mtz_io import read_mtz_file, mtz_to_pandas, MTZFilepath\n", + "from ess.nmx.mtz_io import read_mtz_file, mtz_to_pandas, MTZFilePath\n", "from ess.nmx.data import get_small_mtz_samples\n", "\n", + "\n", "small_mtz_sample = get_small_mtz_samples()[0]\n", - "mtz = read_mtz_file(MTZFilepath(small_mtz_sample))\n", + "mtz = read_mtz_file(MTZFilePath(small_mtz_sample))\n", "df = mtz_to_pandas(mtz)\n", "df.head()" ] @@ -53,7 +54,7 @@ "source": [ "import sciline as sl\n", "from ess.nmx.mtz_io import mtz_io_providers\n", - "from ess.nmx.mtz_io import MTZFilepath, MTZFileIndex, NMXMtzDataFrame, SpaceGroupDesc\n", + "from ess.nmx.mtz_io import MTZFileIndex, NMXMtzDataFrame, SpaceGroupDesc\n", "\n", "pl = sl.Pipeline(\n", " providers=mtz_io_providers,\n", @@ -64,7 +65,7 @@ ")\n", "\n", "file_path_table = sl.ParamTable(\n", - " row_dim=MTZFileIndex, columns={MTZFilepath: get_small_mtz_samples()}\n", + " row_dim=MTZFileIndex, columns={MTZFilePath: get_small_mtz_samples()}\n", ")\n", "\n", "pl.set_param_table(file_path_table)\n", diff --git a/packages/essnmx/src/ess/nmx/data/__init__.py b/packages/essnmx/src/ess/nmx/data/__init__.py index 8fec7b37..bdc49063 100644 --- a/packages/essnmx/src/ess/nmx/data/__init__.py +++ b/packages/essnmx/src/ess/nmx/data/__init__.py @@ -1,23 +1,25 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +import pathlib + import pooch -_version = '0' +_version = "0" -__all__ = ['small_mcstas_2_sample', 'small_mcstas_3_sample', 'get_path'] +__all__ = ["small_mcstas_2_sample", "small_mcstas_3_sample", "get_path"] def _make_pooch() -> pooch.Pooch: return pooch.create( - path=pooch.os_cache('essnmx'), - env='ESSNMX_DATA_DIR', + path=pooch.os_cache("essnmx"), + env="ESSNMX_DATA_DIR", retry_if_failed=3, - base_url='https://public.esss.dk/groups/scipp/ess/nmx/', + base_url="https://public.esss.dk/groups/scipp/ess/nmx/", version=_version, registry={ - 'small_mcstas_2_sample.h5': 'md5:c3affe636397f8c9eea1d9c10a2bf487', - 'small_mcstas_3_sample.h5': 'md5:2afaac205d13ee857ee5364e3f1957a7', - 'mtz_samples.tar.gz': 'md5:bed1eaf604bbe8725c1f6a20ca79fcc0', + "small_mcstas_2_sample.h5": "md5:c3affe636397f8c9eea1d9c10a2bf487", + "small_mcstas_3_sample.h5": "md5:2afaac205d13ee857ee5364e3f1957a7", + "mtz_samples.tar.gz": "md5:bed1eaf604bbe8725c1f6a20ca79fcc0", }, ) @@ -31,14 +33,14 @@ def small_mcstas_2_sample(): warnings.warn( DeprecationWarning( - '``essnmx`` will not support loading files ' - 'made by McStas with version less than 3 from ``25.0.0``. ' - 'Use ``small_mcstas_3_sample`` instead.' + "``essnmx`` will not support loading files " + "made by McStas with version less than 3 from ``25.0.0``. " + "Use ``small_mcstas_3_sample`` instead." ), stacklevel=2, ) - return get_path('small_mcstas_2_sample.h5') + return get_path("small_mcstas_2_sample.h5") def small_mcstas_3_sample(): @@ -46,7 +48,7 @@ def small_mcstas_3_sample(): Real McStas 3 file should contain more dataset under ``data`` group. """ - return get_path('small_mcstas_3_sample.h5') + return get_path("small_mcstas_3_sample.h5") def get_path(name: str) -> str: @@ -59,8 +61,11 @@ def get_path(name: str) -> str: return _pooch.fetch(name) -def get_small_mtz_samples() -> list[str]: +def get_small_mtz_samples() -> list[pathlib.Path]: """Return a list of path to MTZ sample files.""" from pooch.processors import Untar - return _pooch.fetch('mtz_samples.tar.gz', processor=Untar()) + return [ + pathlib.Path(file_path) + for file_path in _pooch.fetch("mtz_samples.tar.gz", processor=Untar()) + ] diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index 5c0149e1..8c997f97 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +import pathlib from typing import NewType, Optional import gemmi @@ -12,10 +13,8 @@ """The index of the mtz file when iterating over multiple mtz files.""" # User defined or configurable types -MTZFilepath = NewType("MTZFilepath", str) +MTZFilePath = NewType("MTZFilePath", pathlib.Path) """Path to the mtz file""" -LambdaBinSize = NewType("LambdaBinSize", int) -"""The number of bins to use for binning wavelength(lambda).""" SpaceGroupDesc = NewType("SpaceGroupDesc", str) """The space group description. e.g. 'P 21 21 21'""" DEFAULT_SPACE_GROUP_DESC = SpaceGroupDesc("P 21 21 21") @@ -36,10 +35,10 @@ """The reduced mtz dataframe with derived columns.""" -def read_mtz_file(file_path: MTZFilepath) -> RawMtz: +def read_mtz_file(file_path: MTZFilePath) -> RawMtz: """read mtz file""" - return RawMtz(gemmi.read_mtz_file(file_path)) + return RawMtz(gemmi.read_mtz_file(file_path.as_posix())) def mtz_to_pandas(mtz: gemmi.Mtz) -> pd.DataFrame: diff --git a/packages/essnmx/tests/mtz_io_test.py b/packages/essnmx/tests/mtz_io_test.py index 0f6c9153..f5bdb246 100644 --- a/packages/essnmx/tests/mtz_io_test.py +++ b/packages/essnmx/tests/mtz_io_test.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +import pathlib + import gemmi import pytest import sciline as sl @@ -9,7 +11,7 @@ from ess.nmx.mtz_io import ( MergedMtzDataFrame, MTZFileIndex, - MTZFilepath, + MTZFilePath, RawMtz, get_reciprocal_asu, get_space_group, @@ -22,19 +24,19 @@ @pytest.fixture(params=get_small_mtz_samples()) -def file_path(request) -> str: +def file_path(request) -> pathlib.Path: return request.param -def test_gemmi_mtz(file_path: str) -> None: - mtz = read_mtz_file(MTZFilepath(file_path)) +def test_gemmi_mtz(file_path: pathlib.Path) -> None: + mtz = read_mtz_file(MTZFilePath(file_path)) assert mtz.spacegroup == gemmi.SpaceGroup("C 1 2 1") # Hard-coded value assert len(mtz.columns[0]) == 100 # Number of samples, hard-coded value @pytest.fixture -def gemmi_mtz_object(file_path: str) -> gemmi.Mtz: - return read_mtz_file(MTZFilepath(file_path)) +def gemmi_mtz_object(file_path: pathlib.Path) -> gemmi.Mtz: + return read_mtz_file(MTZFilePath(file_path)) def test_mtz_to_pandas_dataframe(gemmi_mtz_object: gemmi.Mtz) -> None: @@ -67,7 +69,7 @@ def mtz_series() -> sl.Series[MTZFileIndex, RawMtz]: return sl.Series( row_dim=MTZFileIndex, items={ - MTZFileIndex(i_file): read_mtz_file(MTZFilepath(file_path)) + MTZFileIndex(i_file): read_mtz_file(MTZFilePath(file_path)) for i_file, file_path in enumerate(get_small_mtz_samples()) }, ) From 7defdb90a28128500fba617ce0b98104566c3933 Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Tue, 16 Apr 2024 09:22:22 +0200 Subject: [PATCH 118/403] Update src/ess/nmx/mtz_io.py Co-authored-by: Jan-Lukas Wynen --- packages/essnmx/src/ess/nmx/mtz_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index 8c997f97..fae2159c 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -100,7 +100,7 @@ def _calculate_d(row: pd.Series) -> float: return mtz.get_cell().calculate_d(row["hkl"]) mtz_df["d"] = mtz_df.apply(_calculate_d, axis=1) - # $(2d)^{-2} = \sin^2(\theta)/\lambda^2 + # (2d)^{-2} = \sin^2(\theta)/\lambda^2 mtz_df["resolution"] = (1 / mtz_df["d"]) ** 2 / 4 mtz_df["I_div_SIGI"] = orig_df["I"] / orig_df["SIGI"] From 6349427481d188a34a1ad8c77cf1164728fb8922 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 16 Apr 2024 09:30:20 +0200 Subject: [PATCH 119/403] Fix docstring. --- packages/essnmx/docs/api-reference/index.md | 1 + packages/essnmx/src/ess/nmx/mtz_io.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/essnmx/docs/api-reference/index.md b/packages/essnmx/docs/api-reference/index.md index 17ee733c..0233e6e3 100644 --- a/packages/essnmx/docs/api-reference/index.md +++ b/packages/essnmx/docs/api-reference/index.md @@ -37,5 +37,6 @@ data mcstas_loader types + mtz_io ``` diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index fae2159c..26ee1e7b 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -80,8 +80,7 @@ def reduce_single_mtz(mtz: RawMtz) -> RawMtzDataFrame: Notes ----- - :class:`pandas.DataFrame` is the data structure - that the rest of the steps are using, + :class:`pandas.DataFrame` is used from loading to merging, but :class:`gemmi.Mtz` has :func:`gemmi.Mtz:calculate_d` that can derive the ``d`` using ``HKL``. This part of the method must be called on each mtz file separately. @@ -125,7 +124,7 @@ def get_space_group( spacegroup_desc: The space group description to use if not found in the mtz files. - If None, ``DEFAULT_SPACE_GROUP_DESC`` is used. + If None, :attr:`~DEFAULT_SPACE_GROUP_DESC` is used. Returns ------- From c0cedb66f790e9f8a7c7b1966e36ddbf809a8357 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Mon, 15 Apr 2024 15:44:32 +0200 Subject: [PATCH 120/403] Reference scale factor methods. --- .../docs/examples/scaling_workflow.ipynb | 32 +++++-- packages/essnmx/src/ess/nmx/mtz_io.py | 76 +++++++++++++++- packages/essnmx/src/ess/nmx/scaling.py | 91 +++++++++++++++++++ 3 files changed, 189 insertions(+), 10 deletions(-) create mode 100644 packages/essnmx/src/ess/nmx/scaling.py diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index af923a20..810592c7 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -53,14 +53,17 @@ "outputs": [], "source": [ "import sciline as sl\n", - "from ess.nmx.mtz_io import mtz_io_providers\n", - "from ess.nmx.mtz_io import MTZFileIndex, NMXMtzDataFrame, SpaceGroupDesc\n", + "from ess.nmx.mtz_io import mtz_io_providers, mtz_io_params\n", + "from ess.nmx.mtz_io import MTZFileIndex, SpaceGroupDesc\n", + "from ess.nmx.scaling import scaling_providers\n", + "from ess.nmx.scaling import WavelengthBinSize, WavelengthBinned\n", "\n", "pl = sl.Pipeline(\n", - " providers=mtz_io_providers,\n", + " providers=mtz_io_providers+scaling_providers,\n", " params={\n", - " SpaceGroupDesc: \"C 1 2 1\"\n", - " # Replace with the correct space group if needed\n", + " SpaceGroupDesc: \"C 1 2 1\", # Replace with the correct space group if needed\n", + " WavelengthBinSize: 50, # Replace with the correct bin size if needed\n", + " **mtz_io_params\n", " },\n", ")\n", "\n", @@ -85,7 +88,7 @@ "metadata": {}, "outputs": [], "source": [ - "scaling_nmx_workflow = pl.get(NMXMtzDataFrame)\n", + "scaling_nmx_workflow = pl.get(WavelengthBinned)\n", "scaling_nmx_workflow.visualize(graph_attr={\"rankdir\": \"LR\"})" ] }, @@ -102,8 +105,21 @@ "metadata": {}, "outputs": [], "source": [ - "merged_df = scaling_nmx_workflow.compute(NMXMtzDataFrame)\n", - "merged_df.head()" + "binned = scaling_nmx_workflow.compute(WavelengthBinned)\n", + "binned" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import scipp as sc\n", + "from ess.nmx.scaling import get_reference_bin\n", + "\n", + "ref = get_reference_bin(binned)\n", + "ref" ] } ], diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index 26ee1e7b..bfcb77de 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -7,6 +7,7 @@ import numpy as np import pandas as pd import sciline as sl +import scipp as sc # Index types for param table. MTZFileIndex = NewType("MTZFileIndex", int) @@ -20,6 +21,12 @@ DEFAULT_SPACE_GROUP_DESC = SpaceGroupDesc("P 21 21 21") """The default space group description to use if not found in the mtz files.""" +# Custom column names +WavelengthColumnName = NewType("WavelengthColumnName", str) +"""The name of the wavelength column in the mtz file.""" +DEFUAULT_WAVELENGTH_COLUMN_NAME = WavelengthColumnName("LAMBDA") + + # Computed types RawMtz = NewType("RawMtz", gemmi.Mtz) """The mtz file as a gemmi object""" @@ -33,6 +40,7 @@ """The merged mtz dataframe with derived columns.""" NMXMtzDataFrame = NewType("NMXMtzDataFrame", pd.DataFrame) """The reduced mtz dataframe with derived columns.""" +NMXMtzDataArray = NewType("NMXMtzDataArray", sc.DataArray) def read_mtz_file(file_path: MTZFilePath) -> RawMtz: @@ -64,7 +72,10 @@ def mtz_to_pandas(mtz: gemmi.Mtz) -> pd.DataFrame: ) -def reduce_single_mtz(mtz: RawMtz) -> RawMtzDataFrame: +def reduce_single_mtz( + mtz: RawMtz, + lambda_column_name: WavelengthColumnName = DEFUAULT_WAVELENGTH_COLUMN_NAME, +) -> RawMtzDataFrame: """Select and derive columns from the original ``MtzDataFrame``. Parameters @@ -72,6 +83,9 @@ def reduce_single_mtz(mtz: RawMtz) -> RawMtzDataFrame: mtz: The raw mtz dataset. + lambda_column_name: + The name of the wavelength column in the mtz file. + Returns ------- : @@ -101,8 +115,11 @@ def _calculate_d(row: pd.Series) -> float: mtz_df["d"] = mtz_df.apply(_calculate_d, axis=1) # (2d)^{-2} = \sin^2(\theta)/\lambda^2 mtz_df["resolution"] = (1 / mtz_df["d"]) ** 2 / 4 - mtz_df["I_div_SIGI"] = orig_df["I"] / orig_df["SIGI"] + mtz_df[DEFUAULT_WAVELENGTH_COLUMN_NAME] = orig_df[lambda_column_name] + # Keep other columns + for column in [col for col in orig_df.columns if col not in mtz_df]: + mtz_df[column] = orig_df[column] return RawMtzDataFrame(mtz_df) @@ -186,9 +203,59 @@ def _rapio_asu_to_asu(row: pd.Series) -> list[int]: merged_df["hkl_eq"] = merged_df.apply(_rapio_asu_to_asu, axis=1) + def unpack_vector(row: pd.Series, *new_names) -> pd.DataFrame: + return pd.DataFrame( + {name: [val[i] for val in row] for i, name in enumerate(new_names)} + ) + + # Unpack HKL EQ + merged_df[["H_EQ", "K_EQ", "L_EQ"]] = unpack_vector( + merged_df["hkl_eq"], "H_EQ", "K_EQ", "L_EQ" + ) + return NMXMtzDataFrame(merged_df) +def nmx_mtz_dataframe_to_scipp_dataarray( + nmx_mtz_df: NMXMtzDataFrame, +) -> NMXMtzDataArray: + """Converts the reduced mtz dataframe to a scipp dataarray.""" + from scipp.compat.pandas_compat import from_pandas_dataframe, parse_bracket_header + + # Add unit to the name + to_scipp = nmx_mtz_df.copy(deep=False) + to_scipp[DEFUAULT_WAVELENGTH_COLUMN_NAME + " [Å]"] = to_scipp[ + DEFUAULT_WAVELENGTH_COLUMN_NAME + ] + # Add dummy data column + dummy_data_column_name = "DUMMY_DATA" + to_scipp[dummy_data_column_name] = np.ones(len(to_scipp)) + # Pop the vector columns for later + vector_columns = ("hkl", "hkl_eq") + vector_coords = {col: to_scipp.pop(col) for col in vector_columns} + # Add units + to_scipp.rename(columns={"I": "I [dimensionless]"}, inplace=True) + to_scipp.rename(columns={"SIGI": "SIGI [dimensionless]"}, inplace=True) + # Convert to scipp Dataset + nmx_mtz_ds = from_pandas_dataframe( + to_scipp, + data_columns=dummy_data_column_name, + header_parser=parse_bracket_header, + ) + # Add back the vector columns + for col, values in vector_coords.items(): + nmx_mtz_ds.coords[col] = sc.vectors( + dims=nmx_mtz_ds.dims, values=[val for val in values] + ) + # Add HKL EQ hash coordinate for grouping + nmx_mtz_ds.coords["hkl_eq_hash"] = sc.Variable( + dims=nmx_mtz_ds.dims, + values=[hash(tuple(val)) for val in nmx_mtz_ds.coords["hkl_eq"].values], + ) + # Return DataArray + return NMXMtzDataArray(nmx_mtz_ds[dummy_data_column_name].copy(deep=False)) + + mtz_io_providers = ( read_mtz_file, reduce_single_mtz, @@ -196,5 +263,10 @@ def _rapio_asu_to_asu(row: pd.Series) -> list[int]: get_reciprocal_asu, merge_mtz_dataframes, reduce_merged_mtz_dataframe, + nmx_mtz_dataframe_to_scipp_dataarray, ) """The providers related to the MTZ IO.""" +mtz_io_params = { + WavelengthColumnName: DEFUAULT_WAVELENGTH_COLUMN_NAME, +} +"""The parameters related to the MTZ IO.""" diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py new file mode 100644 index 00000000..d8cdbf1e --- /dev/null +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +from typing import NewType + +import scipp as sc + +from .mtz_io import DEFUAULT_WAVELENGTH_COLUMN_NAME, NMXMtzDataArray + +# User defined or configurable types +WavelengthBinSize = NewType("WavelengthBinSize", int) +"""The size of the wavelength(LAMBDA) bins.""" + + +# Computed types +WavelengthBinned = NewType("WavelengthBinned", sc.DataArray) +"""Binned mtz dataframe by wavelength(LAMBDA) with derived columns.""" +ReferenceWavelengthBin = NewType("ReferenceWavelengthBin", sc.DataArray) +"""The reference bin in the binned dataset.""" +ScaleFactorIntensity = NewType("ScaleFactorIntensity", float) +"""The scale factor for intensity.""" +ScaleFactorSigmaIntensity = NewType("ScaleFactorSigmaIntensity", float) +"""The scale factor for the standard uncertainty of intensity.""" +WavelengthScaled = NewType("WavelengthScaled", sc.DataArray) +"""Scaled wavelength by the reference bin.""" + + +def get_lambda_binned( + mtz_da: NMXMtzDataArray, + wavelength_bin_size: WavelengthBinSize, +) -> WavelengthBinned: + """Bin the whole dataset by wavelength(LAMBDA). + + Notes + ----- + Wavelength(LAMBDA) binning should always be done on the merged dataset. + + """ + + return WavelengthBinned( + mtz_da.bin({DEFUAULT_WAVELENGTH_COLUMN_NAME: wavelength_bin_size}) + ) + + +def _is_bin_empty(binned: sc.DataArray, idx: int) -> bool: + return binned[idx].values.size == 0 + + +def get_reference_bin( + binned: WavelengthBinned, +) -> ReferenceWavelengthBin: + """Find the reference group in the binned dataset. + + The reference group is the group in the middle of the binned dataset. + If the middle group is empty, the function will search for the nearest. + + Parameters + ---------- + binned: + The wavelength binned data. + + Raises + ------ + ValueError: + If no reference group is found. + + """ + middle_number, offset = len(binned) // 2, 0 + + while 0 < (cur_idx := middle_number + offset) < len(binned) and _is_bin_empty( + binned, cur_idx + ): + offset = -offset + 1 if offset <= 0 else -offset + + if _is_bin_empty(binned, cur_idx): + raise ValueError("No reference group found.") + + ref: sc.DataArray = binned[cur_idx].values.copy(deep=False) + grouped: sc.DataArray = ref.group("hkl_eq_hash") + scale_factor_coords = ("I", "SIGI") + for coord_name in scale_factor_coords: + grouped.coords[f"scale_factor_{coord_name}"] = sc.concat( + [sc.mean(1 / gr.values.coords[coord_name]) for gr in grouped], + dim=grouped.dim, + ) + + return ReferenceWavelengthBin(grouped) + + +# Providers and default parameters +scaling_providers = (get_lambda_binned,) +"""Providers for scaling data.""" From 8fb5738f387296c970aab94bfafc50607a13e350 Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Tue, 16 Apr 2024 12:30:30 +0200 Subject: [PATCH 121/403] Update src/ess/nmx/mtz_io.py Co-authored-by: Jan-Lukas Wynen --- packages/essnmx/src/ess/nmx/mtz_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index bfcb77de..d16f64e4 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -24,7 +24,7 @@ # Custom column names WavelengthColumnName = NewType("WavelengthColumnName", str) """The name of the wavelength column in the mtz file.""" -DEFUAULT_WAVELENGTH_COLUMN_NAME = WavelengthColumnName("LAMBDA") +DEFAULT_WAVELENGTH_COLUMN_NAME = WavelengthColumnName("LAMBDA") # Computed types From 4df1ac538c55d9a8703e79fdb63482fa2bcd4a48 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 16 Apr 2024 09:34:46 +0200 Subject: [PATCH 122/403] Add scaling module in the api reference. --- packages/essnmx/docs/api-reference/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/essnmx/docs/api-reference/index.md b/packages/essnmx/docs/api-reference/index.md index 0233e6e3..5777419f 100644 --- a/packages/essnmx/docs/api-reference/index.md +++ b/packages/essnmx/docs/api-reference/index.md @@ -38,5 +38,6 @@ mcstas_loader types mtz_io + scaling ``` From 9ec7169b50fbdc55fc84270527ce48052d8302ce Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 16 Apr 2024 12:38:19 +0200 Subject: [PATCH 123/403] Fix typo. --- packages/essnmx/src/ess/nmx/mtz_io.py | 10 +++++----- packages/essnmx/src/ess/nmx/scaling.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index d16f64e4..23059a26 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -74,7 +74,7 @@ def mtz_to_pandas(mtz: gemmi.Mtz) -> pd.DataFrame: def reduce_single_mtz( mtz: RawMtz, - lambda_column_name: WavelengthColumnName = DEFUAULT_WAVELENGTH_COLUMN_NAME, + lambda_column_name: WavelengthColumnName = DEFAULT_WAVELENGTH_COLUMN_NAME, ) -> RawMtzDataFrame: """Select and derive columns from the original ``MtzDataFrame``. @@ -116,7 +116,7 @@ def _calculate_d(row: pd.Series) -> float: # (2d)^{-2} = \sin^2(\theta)/\lambda^2 mtz_df["resolution"] = (1 / mtz_df["d"]) ** 2 / 4 mtz_df["I_div_SIGI"] = orig_df["I"] / orig_df["SIGI"] - mtz_df[DEFUAULT_WAVELENGTH_COLUMN_NAME] = orig_df[lambda_column_name] + mtz_df[DEFAULT_WAVELENGTH_COLUMN_NAME] = orig_df[lambda_column_name] # Keep other columns for column in [col for col in orig_df.columns if col not in mtz_df]: mtz_df[column] = orig_df[column] @@ -224,8 +224,8 @@ def nmx_mtz_dataframe_to_scipp_dataarray( # Add unit to the name to_scipp = nmx_mtz_df.copy(deep=False) - to_scipp[DEFUAULT_WAVELENGTH_COLUMN_NAME + " [Å]"] = to_scipp[ - DEFUAULT_WAVELENGTH_COLUMN_NAME + to_scipp[DEFAULT_WAVELENGTH_COLUMN_NAME + " [Å]"] = to_scipp[ + DEFAULT_WAVELENGTH_COLUMN_NAME ] # Add dummy data column dummy_data_column_name = "DUMMY_DATA" @@ -267,6 +267,6 @@ def nmx_mtz_dataframe_to_scipp_dataarray( ) """The providers related to the MTZ IO.""" mtz_io_params = { - WavelengthColumnName: DEFUAULT_WAVELENGTH_COLUMN_NAME, + WavelengthColumnName: DEFAULT_WAVELENGTH_COLUMN_NAME, } """The parameters related to the MTZ IO.""" diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index d8cdbf1e..dae91367 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -4,7 +4,7 @@ import scipp as sc -from .mtz_io import DEFUAULT_WAVELENGTH_COLUMN_NAME, NMXMtzDataArray +from .mtz_io import DEFAULT_WAVELENGTH_COLUMN_NAME, NMXMtzDataArray # User defined or configurable types WavelengthBinSize = NewType("WavelengthBinSize", int) @@ -37,7 +37,7 @@ def get_lambda_binned( """ return WavelengthBinned( - mtz_da.bin({DEFUAULT_WAVELENGTH_COLUMN_NAME: wavelength_bin_size}) + mtz_da.bin({DEFAULT_WAVELENGTH_COLUMN_NAME: wavelength_bin_size}) ) From 5becc866e16a97c5dd7ca515bb181e2cbdcf363c Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 16 Apr 2024 12:44:58 +0200 Subject: [PATCH 124/403] Remove vector unpacking. --- packages/essnmx/src/ess/nmx/mtz_io.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index 23059a26..f6a21df5 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -203,16 +203,6 @@ def _rapio_asu_to_asu(row: pd.Series) -> list[int]: merged_df["hkl_eq"] = merged_df.apply(_rapio_asu_to_asu, axis=1) - def unpack_vector(row: pd.Series, *new_names) -> pd.DataFrame: - return pd.DataFrame( - {name: [val[i] for val in row] for i, name in enumerate(new_names)} - ) - - # Unpack HKL EQ - merged_df[["H_EQ", "K_EQ", "L_EQ"]] = unpack_vector( - merged_df["hkl_eq"], "H_EQ", "K_EQ", "L_EQ" - ) - return NMXMtzDataFrame(merged_df) @@ -245,7 +235,7 @@ def nmx_mtz_dataframe_to_scipp_dataarray( # Add back the vector columns for col, values in vector_coords.items(): nmx_mtz_ds.coords[col] = sc.vectors( - dims=nmx_mtz_ds.dims, values=[val for val in values] + dims=nmx_mtz_ds.dims, values=values.to_numpy(copy=False) ) # Add HKL EQ hash coordinate for grouping nmx_mtz_ds.coords["hkl_eq_hash"] = sc.Variable( From f9e2ce4effb5a626b3a8ae95061bacf7ee4257f5 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 16 Apr 2024 12:50:17 +0200 Subject: [PATCH 125/403] Add units to scipp object directly. --- packages/essnmx/src/ess/nmx/mtz_io.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index f6a21df5..9d7c657b 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -212,30 +212,28 @@ def nmx_mtz_dataframe_to_scipp_dataarray( """Converts the reduced mtz dataframe to a scipp dataarray.""" from scipp.compat.pandas_compat import from_pandas_dataframe, parse_bracket_header - # Add unit to the name to_scipp = nmx_mtz_df.copy(deep=False) - to_scipp[DEFAULT_WAVELENGTH_COLUMN_NAME + " [Å]"] = to_scipp[ - DEFAULT_WAVELENGTH_COLUMN_NAME - ] # Add dummy data column dummy_data_column_name = "DUMMY_DATA" to_scipp[dummy_data_column_name] = np.ones(len(to_scipp)) # Pop the vector columns for later vector_columns = ("hkl", "hkl_eq") vector_coords = {col: to_scipp.pop(col) for col in vector_columns} - # Add units - to_scipp.rename(columns={"I": "I [dimensionless]"}, inplace=True) - to_scipp.rename(columns={"SIGI": "SIGI [dimensionless]"}, inplace=True) # Convert to scipp Dataset nmx_mtz_ds = from_pandas_dataframe( to_scipp, data_columns=dummy_data_column_name, header_parser=parse_bracket_header, ) + + # Add units + nmx_mtz_ds.coords[DEFAULT_WAVELENGTH_COLUMN_NAME].unit = sc.units.angstrom + nmx_mtz_ds.coords["I"].unit = sc.units.dimensionless + nmx_mtz_ds.coords["SIGI"].unit = sc.units.dimensionless # Add back the vector columns for col, values in vector_coords.items(): nmx_mtz_ds.coords[col] = sc.vectors( - dims=nmx_mtz_ds.dims, values=values.to_numpy(copy=False) + dims=nmx_mtz_ds.dims, values=[val for val in values] ) # Add HKL EQ hash coordinate for grouping nmx_mtz_ds.coords["hkl_eq_hash"] = sc.Variable( From 91598941012f73a7dabd1620a755e909c5551049 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 16 Apr 2024 13:28:45 +0200 Subject: [PATCH 126/403] Hash coordinate only when it's needed. --- packages/essnmx/src/ess/nmx/mtz_io.py | 5 ----- packages/essnmx/src/ess/nmx/scaling.py | 31 +++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index 9d7c657b..e53060c5 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -235,11 +235,6 @@ def nmx_mtz_dataframe_to_scipp_dataarray( nmx_mtz_ds.coords[col] = sc.vectors( dims=nmx_mtz_ds.dims, values=[val for val in values] ) - # Add HKL EQ hash coordinate for grouping - nmx_mtz_ds.coords["hkl_eq_hash"] = sc.Variable( - dims=nmx_mtz_ds.dims, - values=[hash(tuple(val)) for val in nmx_mtz_ds.coords["hkl_eq"].values], - ) # Return DataArray return NMXMtzDataArray(nmx_mtz_ds[dummy_data_column_name].copy(deep=False)) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index dae91367..fb4ff048 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) -from typing import NewType +from typing import Any, Callable, NewType, Sequence import scipp as sc @@ -45,6 +45,34 @@ def _is_bin_empty(binned: sc.DataArray, idx: int) -> bool: return binned[idx].values.size == 0 +def _apply_elem_wise(func: Callable, var: sc.Variable) -> sc.Variable: + """Apply a function element-wise to the variable values. + + This helper is only for vector-dtype variables. + Use ``numpy.vectorize`` for other types. + + """ + + def apply_func(val: Sequence, _cur_depth: int = 0) -> list: + if _cur_depth == len(var.dims): + return func(val) + return [apply_func(v, _cur_depth + 1) for v in val] + + return sc.Variable( + dims=var.dims, + values=apply_func(var.values), + ) + + +def hash_variable(var: sc.Variable) -> sc.Variable: + """Hash the coordinate values.""" + + def _hash_repr(val: Any) -> int: + return hash(str(val)) + + return _apply_elem_wise(_hash_repr, var) + + def get_reference_bin( binned: WavelengthBinned, ) -> ReferenceWavelengthBin: @@ -75,6 +103,7 @@ def get_reference_bin( raise ValueError("No reference group found.") ref: sc.DataArray = binned[cur_idx].values.copy(deep=False) + ref.coords["hkl_eq_hash"] = hash_variable(ref.coords["hkl_eq"]) grouped: sc.DataArray = ref.group("hkl_eq_hash") scale_factor_coords = ("I", "SIGI") for coord_name in scale_factor_coords: From 0c2bf05d30fca40f1cc93d0283b1fc31a93cebcc Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 16 Apr 2024 14:02:19 +0200 Subject: [PATCH 127/403] Elem-wise apply helper test. --- packages/essnmx/tests/scaling_test.py | 36 +++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 packages/essnmx/tests/scaling_test.py diff --git a/packages/essnmx/tests/scaling_test.py b/packages/essnmx/tests/scaling_test.py new file mode 100644 index 00000000..a18093b3 --- /dev/null +++ b/packages/essnmx/tests/scaling_test.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +import scipp as sc + +from ess.nmx.scaling import _apply_elem_wise + + +def test_apply_elem_wise_add() -> None: + var = sc.Variable(dims=["x"], values=[1, 2, 3]) + + assert sc.identical( + _apply_elem_wise(lambda x: x + 1, var), + sc.Variable(dims=["x"], values=var.values + 1), + ) + + +def test_apply_elem_wise_str() -> None: + from ess.nmx.scaling import _apply_elem_wise + + var = sc.Variable(dims=["x"], values=[1, 2, 3]) + + assert sc.identical( + _apply_elem_wise(str, var), + sc.Variable(dims=["x"], values=["1", "2", "3"]), + ) + + +def test_apply_elem_wise_vectors() -> None: + from ess.nmx.scaling import _apply_elem_wise + + var = sc.vectors(dims=["x"], values=[(1, 2, 3), (4, 5, 6), (7, 8, 9)]) + + assert sc.identical( + _apply_elem_wise(sum, var), + sc.array(dims=["x"], values=[6, 15, 24], dtype=float), + ) From 801229cc79976955a110a61207f43b91af791aef Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 16 Apr 2024 14:03:09 +0200 Subject: [PATCH 128/403] Remove unnecessary imports. --- packages/essnmx/tests/scaling_test.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/essnmx/tests/scaling_test.py b/packages/essnmx/tests/scaling_test.py index a18093b3..996953b1 100644 --- a/packages/essnmx/tests/scaling_test.py +++ b/packages/essnmx/tests/scaling_test.py @@ -15,8 +15,6 @@ def test_apply_elem_wise_add() -> None: def test_apply_elem_wise_str() -> None: - from ess.nmx.scaling import _apply_elem_wise - var = sc.Variable(dims=["x"], values=[1, 2, 3]) assert sc.identical( @@ -26,8 +24,6 @@ def test_apply_elem_wise_str() -> None: def test_apply_elem_wise_vectors() -> None: - from ess.nmx.scaling import _apply_elem_wise - var = sc.vectors(dims=["x"], values=[(1, 2, 3), (4, 5, 6), (7, 8, 9)]) assert sc.identical( From 176cd640af76e149446033b1692305f0dd0558e5 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 16 Apr 2024 14:09:15 +0200 Subject: [PATCH 129/403] Hash variable tests. --- packages/essnmx/tests/scaling_test.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/essnmx/tests/scaling_test.py b/packages/essnmx/tests/scaling_test.py index 996953b1..e5517c35 100644 --- a/packages/essnmx/tests/scaling_test.py +++ b/packages/essnmx/tests/scaling_test.py @@ -2,7 +2,7 @@ # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) import scipp as sc -from ess.nmx.scaling import _apply_elem_wise +from ess.nmx.scaling import _apply_elem_wise, hash_variable def test_apply_elem_wise_add() -> None: @@ -30,3 +30,21 @@ def test_apply_elem_wise_vectors() -> None: _apply_elem_wise(sum, var), sc.array(dims=["x"], values=[6, 15, 24], dtype=float), ) + + +def test_hash_variable_unique() -> None: + """Different vector values should have different hashes.""" + from itertools import product + + import numpy as np + + var = sc.vectors(dims=["x"], values=list(product(range(20), repeat=3))) + hash_var = hash_variable(var) + assert len(hash_var.values) == len(np.unique(hash_var.values)) + + +def test_hash_variable_same() -> None: + """Same values should have the same hash.""" + var = sc.vectors(dims=["x"], values=[(1, 2, 3), (1, 2, 3)]) + hash_var = hash_variable(var) + assert hash_var.values[0] == hash_var.values[1] From 2e2fe706c6777d2cdf323f3f0f79fba104c5c239 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 16 Apr 2024 16:04:07 +0200 Subject: [PATCH 130/403] Detour-grouping for reference bin grouping hkl. --- packages/essnmx/src/ess/nmx/scaling.py | 143 ++++++++++++++++++++++--- 1 file changed, 129 insertions(+), 14 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index fb4ff048..443f5064 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -2,6 +2,7 @@ # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) from typing import Any, Callable, NewType, Sequence +import numpy as np import scipp as sc from .mtz_io import DEFAULT_WAVELENGTH_COLUMN_NAME, NMXMtzDataArray @@ -16,6 +17,8 @@ """Binned mtz dataframe by wavelength(LAMBDA) with derived columns.""" ReferenceWavelengthBin = NewType("ReferenceWavelengthBin", sc.DataArray) """The reference bin in the binned dataset.""" +ReferenceScaleFactor = NewType("ReferenceScaleFactor", sc.DataArray) +"""The reference scale factor, grouped by HKL_EQ.""" ScaleFactorIntensity = NewType("ScaleFactorIntensity", float) """The scale factor for intensity.""" ScaleFactorSigmaIntensity = NewType("ScaleFactorSigmaIntensity", float) @@ -45,12 +48,24 @@ def _is_bin_empty(binned: sc.DataArray, idx: int) -> bool: return binned[idx].values.size == 0 -def _apply_elem_wise(func: Callable, var: sc.Variable) -> sc.Variable: +def _apply_elem_wise( + func: Callable, var: sc.Variable, *, result_dtype: Any = None +) -> sc.Variable: """Apply a function element-wise to the variable values. This helper is only for vector-dtype variables. Use ``numpy.vectorize`` for other types. + Parameters + ---------- + func: + The function to apply. + var: + The variable to apply the function to. + result_dtype: + The dtype of the resulting variable. + It is needed especially when the function returns a vector. + """ def apply_func(val: Sequence, _cur_depth: int = 0) -> list: @@ -58,24 +73,31 @@ def apply_func(val: Sequence, _cur_depth: int = 0) -> list: return func(val) return [apply_func(v, _cur_depth + 1) for v in val] + if result_dtype is None: + return sc.Variable( + dims=var.dims, + values=apply_func(var.values), + ) return sc.Variable( dims=var.dims, values=apply_func(var.values), + dtype=result_dtype, ) -def hash_variable(var: sc.Variable) -> sc.Variable: - """Hash the coordinate values.""" +def _hash_repr(val: Any) -> int: + """Hash the string representation of the value.""" + + return hash(str(val)) + - def _hash_repr(val: Any) -> int: - return hash(str(val)) +def hash_variable(var: sc.Variable, hash_func: Callable = hash) -> sc.Variable: + """Hash the coordinate values.""" - return _apply_elem_wise(_hash_repr, var) + return _apply_elem_wise(hash_func, var) -def get_reference_bin( - binned: WavelengthBinned, -) -> ReferenceWavelengthBin: +def get_reference_bin(binned: WavelengthBinned) -> ReferenceWavelengthBin: """Find the reference group in the binned dataset. The reference group is the group in the middle of the binned dataset. @@ -102,9 +124,98 @@ def get_reference_bin( if _is_bin_empty(binned, cur_idx): raise ValueError("No reference group found.") - ref: sc.DataArray = binned[cur_idx].values.copy(deep=False) - ref.coords["hkl_eq_hash"] = hash_variable(ref.coords["hkl_eq"]) - grouped: sc.DataArray = ref.group("hkl_eq_hash") + return binned[cur_idx].values.copy(deep=False) + + +def _detour_group( + da: sc.DataArray, group_name: str, detour_func: Callable +) -> sc.DataArray: + """Group the data array by a hash of a coordinate. + + It uses index of each unique hash value + for grouping instead of hash value itself + to avoid overflow issues. + + """ + from uuid import uuid4 + + copied = da.copy(deep=False) + + # Temporary coords for grouping + detour_idx_coord_name = uuid4().hex + "hash_idx" + + # Create a temporary detoured coordinate + detour_var = _apply_elem_wise(detour_func, da.coords[group_name]) + # Create a temporary hash-index of each unique value + unique_hashes = np.unique(detour_var.values) + hash_to_idx = {hash_val: idx for idx, hash_val in enumerate(unique_hashes)} + copied.coords[detour_idx_coord_name] = _apply_elem_wise( + lambda idx: hash_to_idx[idx], detour_var + ) + + # Group by the hash-index + grouped = copied.group(detour_idx_coord_name) + + # Restore the original values + idx_to_detour = {idx: hash_val for hash_val, idx in hash_to_idx.items()} + detour_to_var = { + hash_val: var + for var, hash_val in zip(da.coords[group_name].values, detour_var.values) + } + idx_to_var = { + idx: detour_to_var[hash_val] for idx, hash_val in idx_to_detour.items() + } + grouped.coords[group_name] = _apply_elem_wise( + lambda idx: idx_to_var[idx], + grouped.coords[detour_idx_coord_name], + result_dtype=da.coords[group_name].dtype, + ) + # Rename dims back to group_name and drop the temporary hash-index coordinate + return grouped.rename_dims({detour_idx_coord_name: group_name}).drop_coords( + [detour_idx_coord_name] + ) + + +def group(da: sc.DataArray, /, *args: str, **group_detour_func_map) -> sc.DataArray: + """Group the data array by the given coordinates. + + Parameters + ---------- + da: + The data array to group. + args: + The coordinates to group by. + group_hash_func_map: + The hash functions for each coordinate. + + Returns + ------- + sc.DataArray + The grouped data array. + + """ + grouped = da + for group_name in args: + if group_name in group_detour_func_map: + grouped = _detour_group( + grouped, group_name, group_detour_func_map[group_name] + ) + else: + try: + grouped = sc.group(grouped, group_name) + except Exception: + grouped = _detour_group( + grouped, group_name, group_detour_func_map.get(group_name, hash) + ) + + return grouped + + +def calculate_scale_factor_per_hkl_eq( + ref_bin: ReferenceWavelengthBin, +) -> ReferenceScaleFactor: + grouped = group(ref_bin, "hkl_eq", hkl_eq=_hash_repr) + scale_factor_coords = ("I", "SIGI") for coord_name in scale_factor_coords: grouped.coords[f"scale_factor_{coord_name}"] = sc.concat( @@ -112,9 +223,13 @@ def get_reference_bin( dim=grouped.dim, ) - return ReferenceWavelengthBin(grouped) + return ReferenceScaleFactor(grouped) # Providers and default parameters -scaling_providers = (get_lambda_binned,) +scaling_providers = ( + get_lambda_binned, + get_reference_bin, + calculate_scale_factor_per_hkl_eq, +) """Providers for scaling data.""" From cb64c509d84f24b4d3a7ba2a1425d0d6ae82363d Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 16 Apr 2024 16:04:55 +0200 Subject: [PATCH 131/403] Update scale factor example. --- packages/essnmx/docs/examples/scaling_workflow.ipynb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index 810592c7..df6428a4 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -115,11 +115,10 @@ "metadata": {}, "outputs": [], "source": [ - "import scipp as sc\n", - "from ess.nmx.scaling import get_reference_bin\n", + "from ess.nmx.scaling import get_reference_bin, calculate_scale_factor_per_hkl_eq\n", "\n", - "ref = get_reference_bin(binned)\n", - "ref" + "scale_factor = calculate_scale_factor_per_hkl_eq(get_reference_bin(binned))\n", + "scale_factor" ] } ], From a9eebd0d177a807e3318dd3a0cc8c8aeb5029452 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 16 Apr 2024 16:10:23 +0200 Subject: [PATCH 132/403] Update tests. --- packages/essnmx/tests/scaling_test.py | 58 +++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/packages/essnmx/tests/scaling_test.py b/packages/essnmx/tests/scaling_test.py index e5517c35..e3fdfdf2 100644 --- a/packages/essnmx/tests/scaling_test.py +++ b/packages/essnmx/tests/scaling_test.py @@ -1,8 +1,15 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +import pytest import scipp as sc -from ess.nmx.scaling import _apply_elem_wise, hash_variable +from ess.nmx.mtz_io import DEFAULT_WAVELENGTH_COLUMN_NAME +from ess.nmx.scaling import ( + _apply_elem_wise, + _hash_repr, + get_reference_bin, + hash_variable, +) def test_apply_elem_wise_add() -> None: @@ -39,12 +46,57 @@ def test_hash_variable_unique() -> None: import numpy as np var = sc.vectors(dims=["x"], values=list(product(range(20), repeat=3))) - hash_var = hash_variable(var) + hash_var = hash_variable(var, hash_func=_hash_repr) assert len(hash_var.values) == len(np.unique(hash_var.values)) def test_hash_variable_same() -> None: """Same values should have the same hash.""" var = sc.vectors(dims=["x"], values=[(1, 2, 3), (1, 2, 3)]) - hash_var = hash_variable(var) + hash_var = hash_variable(var, hash_func=_hash_repr) assert hash_var.values[0] == hash_var.values[1] + + +@pytest.fixture +def nmx_data_array() -> sc.DataArray: + return sc.DataArray( + data=sc.ones(dims=["row"], shape=[7]), + coords={ + DEFAULT_WAVELENGTH_COLUMN_NAME: sc.Variable( + dims=["row"], values=[1, 2, 3, 4, 5, 3, 3] + ), + "hkl_eq": sc.vectors( + dims=["row"], + values=[ + (1, 2, 3), + (4, 5, 6), + (7, 8, 9), + (10, 11, 12), + (13, 14, 15), + (8, 7, 9), + (9, 8, 7), + ], + ), + "I": sc.Variable(dims=["row"], values=[1, 2, 3, 4, 5, 3.1, 3.2]), + "SIGI": sc.Variable( + dims=["row"], values=[0.1, 0.2, 0.3, 0.4, 0.5, 0.31, 0.32] + ), + }, + ) + + +def test_get_reference_bin_middle(nmx_data_array: sc.DataArray) -> None: + """Test the middle bin.""" + + ref_bin = get_reference_bin(nmx_data_array.bin({DEFAULT_WAVELENGTH_COLUMN_NAME: 6})) + selected_idx = (2, 5, 6) + for coord in ("I", "SIGI"): + assert all( + ref_bin.coords[coord].values + == [nmx_data_array.coords[coord].values[idx] for idx in selected_idx] + ) + + +@pytest.fixture +def reference_bin(nmx_data_array: sc.DataArray) -> sc.DataArray: + return get_reference_bin(nmx_data_array.bin({DEFAULT_WAVELENGTH_COLUMN_NAME: 6})) From d88ce3dc1dadf3677c56f36c65a1ec344e3a8f27 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 16 Apr 2024 16:21:36 +0200 Subject: [PATCH 133/403] Add reference scaling factor tests. --- packages/essnmx/tests/scaling_test.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/tests/scaling_test.py b/packages/essnmx/tests/scaling_test.py index e3fdfdf2..7ac1eb60 100644 --- a/packages/essnmx/tests/scaling_test.py +++ b/packages/essnmx/tests/scaling_test.py @@ -5,6 +5,7 @@ from ess.nmx.mtz_io import DEFAULT_WAVELENGTH_COLUMN_NAME from ess.nmx.scaling import ( + ReferenceWavelengthBin, _apply_elem_wise, _hash_repr, get_reference_bin, @@ -73,7 +74,7 @@ def nmx_data_array() -> sc.DataArray: (7, 8, 9), (10, 11, 12), (13, 14, 15), - (8, 7, 9), + (7, 8, 9), (9, 8, 7), ], ), @@ -98,5 +99,22 @@ def test_get_reference_bin_middle(nmx_data_array: sc.DataArray) -> None: @pytest.fixture -def reference_bin(nmx_data_array: sc.DataArray) -> sc.DataArray: +def reference_bin(nmx_data_array: sc.DataArray) -> ReferenceWavelengthBin: return get_reference_bin(nmx_data_array.bin({DEFAULT_WAVELENGTH_COLUMN_NAME: 6})) + + +def test_reference_bin_scale_factor(reference_bin: ReferenceWavelengthBin) -> None: + """Test the scale factor for I.""" + from ess.nmx.scaling import calculate_scale_factor_per_hkl_eq, group + + scale_factor_groups = calculate_scale_factor_per_hkl_eq(reference_bin) + grouped = group(reference_bin, "hkl_eq", hkl_eq=_hash_repr) + + for hkl_eq in grouped.coords["hkl_eq"].values: + calculated_gr = scale_factor_groups["hkl_eq", sc.vector(hkl_eq)] + reference_gr = grouped["hkl_eq", sc.vector(hkl_eq)] + for coord in ("I", "SIGI"): + assert sc.identical( + calculated_gr.coords[f"scale_factor_{coord}"], + sc.mean(1 / reference_gr.values.coords[coord]), + ) From e77af9db0b4fab2ea2ecf5684cd33d3c85a26667 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 16 Apr 2024 16:25:34 +0200 Subject: [PATCH 134/403] Add detour grouping tests. --- packages/essnmx/tests/scaling_test.py | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/packages/essnmx/tests/scaling_test.py b/packages/essnmx/tests/scaling_test.py index 7ac1eb60..37ad149a 100644 --- a/packages/essnmx/tests/scaling_test.py +++ b/packages/essnmx/tests/scaling_test.py @@ -40,6 +40,36 @@ def test_apply_elem_wise_vectors() -> None: ) +def test_detour_group_str() -> None: + from ess.nmx.scaling import group + + da = sc.DataArray( + data=sc.ones(dims=["x"], shape=[3]), + coords={"x": sc.Variable(dims=["x"], values=["a", "b", "a"])}, + ) + + grouped = group(da, "x", x=lambda x: x) + assert sc.identical( + grouped.coords["x"], + sc.Variable(dims=["x"], values=["a", "b"]), + ) + + +def test_detour_group_vector() -> None: + from ess.nmx.scaling import _hash_repr, group + + da = sc.DataArray( + data=sc.ones(dims=["x"], shape=[10]), + coords={"x": sc.vectors(dims=["x"], values=[(1, 2, 3), (4, 5, 6)] * 5)}, + ) + + grouped = group(da, "x", x=_hash_repr) + assert sc.identical( + grouped.coords["x"], + sc.vectors(dims=["x"], values=[(1, 2, 3), (4, 5, 6)]), + ) + + def test_hash_variable_unique() -> None: """Different vector values should have different hashes.""" from itertools import product From 4db1069c3768ed749c18c4fbef9fdb36c28a44dc Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 16 Apr 2024 16:29:21 +0200 Subject: [PATCH 135/403] Remove unnecessary hash helper. --- packages/essnmx/src/ess/nmx/scaling.py | 14 +---------- packages/essnmx/tests/scaling_test.py | 32 ++++---------------------- 2 files changed, 5 insertions(+), 41 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 443f5064..2628de98 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -85,18 +85,6 @@ def apply_func(val: Sequence, _cur_depth: int = 0) -> list: ) -def _hash_repr(val: Any) -> int: - """Hash the string representation of the value.""" - - return hash(str(val)) - - -def hash_variable(var: sc.Variable, hash_func: Callable = hash) -> sc.Variable: - """Hash the coordinate values.""" - - return _apply_elem_wise(hash_func, var) - - def get_reference_bin(binned: WavelengthBinned) -> ReferenceWavelengthBin: """Find the reference group in the binned dataset. @@ -214,7 +202,7 @@ def group(da: sc.DataArray, /, *args: str, **group_detour_func_map) -> sc.DataAr def calculate_scale_factor_per_hkl_eq( ref_bin: ReferenceWavelengthBin, ) -> ReferenceScaleFactor: - grouped = group(ref_bin, "hkl_eq", hkl_eq=_hash_repr) + grouped = group(ref_bin, "hkl_eq", hkl_eq=str) scale_factor_coords = ("I", "SIGI") for coord_name in scale_factor_coords: diff --git a/packages/essnmx/tests/scaling_test.py b/packages/essnmx/tests/scaling_test.py index 37ad149a..ed721028 100644 --- a/packages/essnmx/tests/scaling_test.py +++ b/packages/essnmx/tests/scaling_test.py @@ -4,13 +4,7 @@ import scipp as sc from ess.nmx.mtz_io import DEFAULT_WAVELENGTH_COLUMN_NAME -from ess.nmx.scaling import ( - ReferenceWavelengthBin, - _apply_elem_wise, - _hash_repr, - get_reference_bin, - hash_variable, -) +from ess.nmx.scaling import ReferenceWavelengthBin, _apply_elem_wise, get_reference_bin def test_apply_elem_wise_add() -> None: @@ -56,38 +50,20 @@ def test_detour_group_str() -> None: def test_detour_group_vector() -> None: - from ess.nmx.scaling import _hash_repr, group + from ess.nmx.scaling import group da = sc.DataArray( data=sc.ones(dims=["x"], shape=[10]), coords={"x": sc.vectors(dims=["x"], values=[(1, 2, 3), (4, 5, 6)] * 5)}, ) - grouped = group(da, "x", x=_hash_repr) + grouped = group(da, "x", x=str) assert sc.identical( grouped.coords["x"], sc.vectors(dims=["x"], values=[(1, 2, 3), (4, 5, 6)]), ) -def test_hash_variable_unique() -> None: - """Different vector values should have different hashes.""" - from itertools import product - - import numpy as np - - var = sc.vectors(dims=["x"], values=list(product(range(20), repeat=3))) - hash_var = hash_variable(var, hash_func=_hash_repr) - assert len(hash_var.values) == len(np.unique(hash_var.values)) - - -def test_hash_variable_same() -> None: - """Same values should have the same hash.""" - var = sc.vectors(dims=["x"], values=[(1, 2, 3), (1, 2, 3)]) - hash_var = hash_variable(var, hash_func=_hash_repr) - assert hash_var.values[0] == hash_var.values[1] - - @pytest.fixture def nmx_data_array() -> sc.DataArray: return sc.DataArray( @@ -138,7 +114,7 @@ def test_reference_bin_scale_factor(reference_bin: ReferenceWavelengthBin) -> No from ess.nmx.scaling import calculate_scale_factor_per_hkl_eq, group scale_factor_groups = calculate_scale_factor_per_hkl_eq(reference_bin) - grouped = group(reference_bin, "hkl_eq", hkl_eq=_hash_repr) + grouped = group(reference_bin, "hkl_eq", hkl_eq=str) for hkl_eq in grouped.coords["hkl_eq"].values: calculated_gr = scale_factor_groups["hkl_eq", sc.vector(hkl_eq)] From 621ffd77406cd79c73503fdb62febef792a4b8ea Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 16 Apr 2024 16:31:30 +0200 Subject: [PATCH 136/403] Hide detour grouping method. --- packages/essnmx/src/ess/nmx/scaling.py | 4 ++-- packages/essnmx/tests/scaling_test.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 2628de98..5c3b45dc 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -164,7 +164,7 @@ def _detour_group( ) -def group(da: sc.DataArray, /, *args: str, **group_detour_func_map) -> sc.DataArray: +def _group(da: sc.DataArray, /, *args: str, **group_detour_func_map) -> sc.DataArray: """Group the data array by the given coordinates. Parameters @@ -202,7 +202,7 @@ def group(da: sc.DataArray, /, *args: str, **group_detour_func_map) -> sc.DataAr def calculate_scale_factor_per_hkl_eq( ref_bin: ReferenceWavelengthBin, ) -> ReferenceScaleFactor: - grouped = group(ref_bin, "hkl_eq", hkl_eq=str) + grouped = _group(ref_bin, "hkl_eq", hkl_eq=str) scale_factor_coords = ("I", "SIGI") for coord_name in scale_factor_coords: diff --git a/packages/essnmx/tests/scaling_test.py b/packages/essnmx/tests/scaling_test.py index ed721028..14dc9f82 100644 --- a/packages/essnmx/tests/scaling_test.py +++ b/packages/essnmx/tests/scaling_test.py @@ -35,14 +35,14 @@ def test_apply_elem_wise_vectors() -> None: def test_detour_group_str() -> None: - from ess.nmx.scaling import group + from ess.nmx.scaling import _group da = sc.DataArray( data=sc.ones(dims=["x"], shape=[3]), coords={"x": sc.Variable(dims=["x"], values=["a", "b", "a"])}, ) - grouped = group(da, "x", x=lambda x: x) + grouped = _group(da, "x", x=lambda x: x) assert sc.identical( grouped.coords["x"], sc.Variable(dims=["x"], values=["a", "b"]), @@ -50,14 +50,14 @@ def test_detour_group_str() -> None: def test_detour_group_vector() -> None: - from ess.nmx.scaling import group + from ess.nmx.scaling import _group da = sc.DataArray( data=sc.ones(dims=["x"], shape=[10]), coords={"x": sc.vectors(dims=["x"], values=[(1, 2, 3), (4, 5, 6)] * 5)}, ) - grouped = group(da, "x", x=str) + grouped = _group(da, "x", x=str) assert sc.identical( grouped.coords["x"], sc.vectors(dims=["x"], values=[(1, 2, 3), (4, 5, 6)]), @@ -111,10 +111,10 @@ def reference_bin(nmx_data_array: sc.DataArray) -> ReferenceWavelengthBin: def test_reference_bin_scale_factor(reference_bin: ReferenceWavelengthBin) -> None: """Test the scale factor for I.""" - from ess.nmx.scaling import calculate_scale_factor_per_hkl_eq, group + from ess.nmx.scaling import _group, calculate_scale_factor_per_hkl_eq scale_factor_groups = calculate_scale_factor_per_hkl_eq(reference_bin) - grouped = group(reference_bin, "hkl_eq", hkl_eq=str) + grouped = _group(reference_bin, "hkl_eq", hkl_eq=str) for hkl_eq in grouped.coords["hkl_eq"].values: calculated_gr = scale_factor_groups["hkl_eq", sc.vector(hkl_eq)] From 4438b7ed9c8d0963795866c2cdff7c250759066b Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 16 Apr 2024 16:47:19 +0200 Subject: [PATCH 137/403] Move general helper function to reduction module. --- packages/essnmx/src/ess/nmx/reduction.py | 124 ++++++++++++++++++++- packages/essnmx/src/ess/nmx/scaling.py | 133 +---------------------- packages/essnmx/tests/scaling_test.py | 65 +---------- 3 files changed, 135 insertions(+), 187 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index d0d92a88..e8510f2d 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -2,9 +2,10 @@ # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) import io import pathlib -from typing import Optional, Union +from typing import Any, Callable, Optional, Sequence, Union import h5py +import numpy as np import sciline import scipp as sc @@ -243,3 +244,124 @@ def bin_time_of_arrival( counts=counts, **{**nmx_data, **new_coords}, ) + + +def _apply_elem_wise( + func: Callable, var: sc.Variable, *, result_dtype: Any = None +) -> sc.Variable: + """Apply a function element-wise to the variable values. + + This helper is only for vector-dtype variables. + Use ``numpy.vectorize`` for other types. + + Parameters + ---------- + func: + The function to apply. + var: + The variable to apply the function to. + result_dtype: + The dtype of the resulting variable. + It is needed especially when the function returns a vector. + + """ + + def apply_func(val: Sequence, _cur_depth: int = 0) -> list: + if _cur_depth == len(var.dims): + return func(val) + return [apply_func(v, _cur_depth + 1) for v in val] + + if result_dtype is None: + return sc.Variable( + dims=var.dims, + values=apply_func(var.values), + ) + return sc.Variable( + dims=var.dims, + values=apply_func(var.values), + dtype=result_dtype, + ) + + +def _detour_group( + da: sc.DataArray, group_name: str, detour_func: Callable +) -> sc.DataArray: + """Group the data array by a hash of a coordinate. + + It uses index of each unique hash value + for grouping instead of hash value itself + to avoid overflow issues. + + """ + from uuid import uuid4 + + copied = da.copy(deep=False) + + # Temporary coords for grouping + detour_idx_coord_name = uuid4().hex + "hash_idx" + + # Create a temporary detoured coordinate + detour_var = _apply_elem_wise(detour_func, da.coords[group_name]) + # Create a temporary hash-index of each unique value + unique_hashes = np.unique(detour_var.values) + hash_to_idx = {hash_val: idx for idx, hash_val in enumerate(unique_hashes)} + copied.coords[detour_idx_coord_name] = _apply_elem_wise( + lambda idx: hash_to_idx[idx], detour_var + ) + + # Group by the hash-index + grouped = copied.group(detour_idx_coord_name) + + # Restore the original values + idx_to_detour = {idx: hash_val for hash_val, idx in hash_to_idx.items()} + detour_to_var = { + hash_val: var + for var, hash_val in zip(da.coords[group_name].values, detour_var.values) + } + idx_to_var = { + idx: detour_to_var[hash_val] for idx, hash_val in idx_to_detour.items() + } + grouped.coords[group_name] = _apply_elem_wise( + lambda idx: idx_to_var[idx], + grouped.coords[detour_idx_coord_name], + result_dtype=da.coords[group_name].dtype, + ) + # Rename dims back to group_name and drop the temporary hash-index coordinate + return grouped.rename_dims({detour_idx_coord_name: group_name}).drop_coords( + [detour_idx_coord_name] + ) + + +def _group(da: sc.DataArray, /, *args: str, **group_detour_func_map) -> sc.DataArray: + """Group the data array by the given coordinates. + + Parameters + ---------- + da: + The data array to group. + args: + The coordinates to group by. + group_hash_func_map: + The hash functions for each coordinate. + + Returns + ------- + sc.DataArray + The grouped data array. + + """ + grouped = da + for group_name in args: + if group_name in group_detour_func_map: + grouped = _detour_group( + grouped, group_name, group_detour_func_map[group_name] + ) + else: + try: + grouped = sc.group(grouped, group_name) + except Exception: + grouped = _detour_group( + grouped, group_name, group_detour_func_map.get(group_name, hash) + ) + + return grouped diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 5c3b45dc..944401c8 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -1,11 +1,11 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) -from typing import Any, Callable, NewType, Sequence +from typing import NewType -import numpy as np import scipp as sc from .mtz_io import DEFAULT_WAVELENGTH_COLUMN_NAME, NMXMtzDataArray +from .reduction import _group # User defined or configurable types WavelengthBinSize = NewType("WavelengthBinSize", int) @@ -27,6 +27,10 @@ """Scaled wavelength by the reference bin.""" +def _is_bin_empty(binned: sc.DataArray, idx: int) -> bool: + return binned[idx].values.size == 0 + + def get_lambda_binned( mtz_da: NMXMtzDataArray, wavelength_bin_size: WavelengthBinSize, @@ -44,47 +48,6 @@ def get_lambda_binned( ) -def _is_bin_empty(binned: sc.DataArray, idx: int) -> bool: - return binned[idx].values.size == 0 - - -def _apply_elem_wise( - func: Callable, var: sc.Variable, *, result_dtype: Any = None -) -> sc.Variable: - """Apply a function element-wise to the variable values. - - This helper is only for vector-dtype variables. - Use ``numpy.vectorize`` for other types. - - Parameters - ---------- - func: - The function to apply. - var: - The variable to apply the function to. - result_dtype: - The dtype of the resulting variable. - It is needed especially when the function returns a vector. - - """ - - def apply_func(val: Sequence, _cur_depth: int = 0) -> list: - if _cur_depth == len(var.dims): - return func(val) - return [apply_func(v, _cur_depth + 1) for v in val] - - if result_dtype is None: - return sc.Variable( - dims=var.dims, - values=apply_func(var.values), - ) - return sc.Variable( - dims=var.dims, - values=apply_func(var.values), - dtype=result_dtype, - ) - - def get_reference_bin(binned: WavelengthBinned) -> ReferenceWavelengthBin: """Find the reference group in the binned dataset. @@ -115,90 +78,6 @@ def get_reference_bin(binned: WavelengthBinned) -> ReferenceWavelengthBin: return binned[cur_idx].values.copy(deep=False) -def _detour_group( - da: sc.DataArray, group_name: str, detour_func: Callable -) -> sc.DataArray: - """Group the data array by a hash of a coordinate. - - It uses index of each unique hash value - for grouping instead of hash value itself - to avoid overflow issues. - - """ - from uuid import uuid4 - - copied = da.copy(deep=False) - - # Temporary coords for grouping - detour_idx_coord_name = uuid4().hex + "hash_idx" - - # Create a temporary detoured coordinate - detour_var = _apply_elem_wise(detour_func, da.coords[group_name]) - # Create a temporary hash-index of each unique value - unique_hashes = np.unique(detour_var.values) - hash_to_idx = {hash_val: idx for idx, hash_val in enumerate(unique_hashes)} - copied.coords[detour_idx_coord_name] = _apply_elem_wise( - lambda idx: hash_to_idx[idx], detour_var - ) - - # Group by the hash-index - grouped = copied.group(detour_idx_coord_name) - - # Restore the original values - idx_to_detour = {idx: hash_val for hash_val, idx in hash_to_idx.items()} - detour_to_var = { - hash_val: var - for var, hash_val in zip(da.coords[group_name].values, detour_var.values) - } - idx_to_var = { - idx: detour_to_var[hash_val] for idx, hash_val in idx_to_detour.items() - } - grouped.coords[group_name] = _apply_elem_wise( - lambda idx: idx_to_var[idx], - grouped.coords[detour_idx_coord_name], - result_dtype=da.coords[group_name].dtype, - ) - # Rename dims back to group_name and drop the temporary hash-index coordinate - return grouped.rename_dims({detour_idx_coord_name: group_name}).drop_coords( - [detour_idx_coord_name] - ) - - -def _group(da: sc.DataArray, /, *args: str, **group_detour_func_map) -> sc.DataArray: - """Group the data array by the given coordinates. - - Parameters - ---------- - da: - The data array to group. - args: - The coordinates to group by. - group_hash_func_map: - The hash functions for each coordinate. - - Returns - ------- - sc.DataArray - The grouped data array. - - """ - grouped = da - for group_name in args: - if group_name in group_detour_func_map: - grouped = _detour_group( - grouped, group_name, group_detour_func_map[group_name] - ) - else: - try: - grouped = sc.group(grouped, group_name) - except Exception: - grouped = _detour_group( - grouped, group_name, group_detour_func_map.get(group_name, hash) - ) - - return grouped - - def calculate_scale_factor_per_hkl_eq( ref_bin: ReferenceWavelengthBin, ) -> ReferenceScaleFactor: diff --git a/packages/essnmx/tests/scaling_test.py b/packages/essnmx/tests/scaling_test.py index 14dc9f82..29879924 100644 --- a/packages/essnmx/tests/scaling_test.py +++ b/packages/essnmx/tests/scaling_test.py @@ -4,64 +4,11 @@ import scipp as sc from ess.nmx.mtz_io import DEFAULT_WAVELENGTH_COLUMN_NAME -from ess.nmx.scaling import ReferenceWavelengthBin, _apply_elem_wise, get_reference_bin - - -def test_apply_elem_wise_add() -> None: - var = sc.Variable(dims=["x"], values=[1, 2, 3]) - - assert sc.identical( - _apply_elem_wise(lambda x: x + 1, var), - sc.Variable(dims=["x"], values=var.values + 1), - ) - - -def test_apply_elem_wise_str() -> None: - var = sc.Variable(dims=["x"], values=[1, 2, 3]) - - assert sc.identical( - _apply_elem_wise(str, var), - sc.Variable(dims=["x"], values=["1", "2", "3"]), - ) - - -def test_apply_elem_wise_vectors() -> None: - var = sc.vectors(dims=["x"], values=[(1, 2, 3), (4, 5, 6), (7, 8, 9)]) - - assert sc.identical( - _apply_elem_wise(sum, var), - sc.array(dims=["x"], values=[6, 15, 24], dtype=float), - ) - - -def test_detour_group_str() -> None: - from ess.nmx.scaling import _group - - da = sc.DataArray( - data=sc.ones(dims=["x"], shape=[3]), - coords={"x": sc.Variable(dims=["x"], values=["a", "b", "a"])}, - ) - - grouped = _group(da, "x", x=lambda x: x) - assert sc.identical( - grouped.coords["x"], - sc.Variable(dims=["x"], values=["a", "b"]), - ) - - -def test_detour_group_vector() -> None: - from ess.nmx.scaling import _group - - da = sc.DataArray( - data=sc.ones(dims=["x"], shape=[10]), - coords={"x": sc.vectors(dims=["x"], values=[(1, 2, 3), (4, 5, 6)] * 5)}, - ) - - grouped = _group(da, "x", x=str) - assert sc.identical( - grouped.coords["x"], - sc.vectors(dims=["x"], values=[(1, 2, 3), (4, 5, 6)]), - ) +from ess.nmx.scaling import ( + ReferenceWavelengthBin, + calculate_scale_factor_per_hkl_eq, + get_reference_bin, +) @pytest.fixture @@ -111,7 +58,7 @@ def reference_bin(nmx_data_array: sc.DataArray) -> ReferenceWavelengthBin: def test_reference_bin_scale_factor(reference_bin: ReferenceWavelengthBin) -> None: """Test the scale factor for I.""" - from ess.nmx.scaling import _group, calculate_scale_factor_per_hkl_eq + from ess.nmx.reduction import _group scale_factor_groups = calculate_scale_factor_per_hkl_eq(reference_bin) grouped = _group(reference_bin, "hkl_eq", hkl_eq=str) From a1c0eb9378267499d760f35f67589063217bf71f Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 16 Apr 2024 16:54:15 +0200 Subject: [PATCH 138/403] Add reduction helper tests. --- packages/essnmx/tests/reduction_test.py | 60 +++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 packages/essnmx/tests/reduction_test.py diff --git a/packages/essnmx/tests/reduction_test.py b/packages/essnmx/tests/reduction_test.py new file mode 100644 index 00000000..0adb1d56 --- /dev/null +++ b/packages/essnmx/tests/reduction_test.py @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +import scipp as sc + +from ess.nmx.reduction import _apply_elem_wise, _group + + +def test_apply_elem_wise_add() -> None: + var = sc.Variable(dims=["x"], values=[1, 2, 3]) + + assert sc.identical( + _apply_elem_wise(lambda x: x + 1, var), + sc.Variable(dims=["x"], values=var.values + 1), + ) + + +def test_apply_elem_wise_str() -> None: + var = sc.Variable(dims=["x"], values=[1, 2, 3]) + + assert sc.identical( + _apply_elem_wise(str, var), + sc.Variable(dims=["x"], values=["1", "2", "3"]), + ) + + +def test_apply_elem_wise_vectors() -> None: + var = sc.vectors(dims=["x"], values=[(1, 2, 3), (4, 5, 6), (7, 8, 9)]) + + assert sc.identical( + _apply_elem_wise(sum, var), + sc.array(dims=["x"], values=[6, 15, 24], dtype=float), + ) + + +def test_detour_group_str() -> None: + from ess.nmx.scaling import _group + + da = sc.DataArray( + data=sc.ones(dims=["x"], shape=[3]), + coords={"x": sc.Variable(dims=["x"], values=["a", "b", "a"])}, + ) + + grouped = _group(da, "x", x=lambda x: x) + assert sc.identical( + grouped.coords["x"], + sc.Variable(dims=["x"], values=["a", "b"]), + ) + + +def test_detour_group_vector() -> None: + da = sc.DataArray( + data=sc.ones(dims=["x"], shape=[10]), + coords={"x": sc.vectors(dims=["x"], values=[(1, 2, 3), (4, 5, 6)] * 5)}, + ) + + grouped = _group(da, "x", x=str) + assert sc.identical( + grouped.coords["x"], + sc.vectors(dims=["x"], values=[(1, 2, 3), (4, 5, 6)]), + ) From 0dbefceba4d269eb0b3eb12161399319f35f26b2 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 17 Apr 2024 19:00:14 +0200 Subject: [PATCH 139/403] Move intensity as data and update grouping methods. --- packages/essnmx/src/ess/nmx/mtz_io.py | 56 +++++-- packages/essnmx/src/ess/nmx/reduction.py | 191 ++++++++++------------- packages/essnmx/src/ess/nmx/scaling.py | 14 +- packages/essnmx/tests/reduction_test.py | 91 ++++++----- packages/essnmx/tests/scaling_test.py | 56 +++---- 5 files changed, 199 insertions(+), 209 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index e53060c5..e8be4dbd 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -26,6 +26,13 @@ """The name of the wavelength column in the mtz file.""" DEFAULT_WAVELENGTH_COLUMN_NAME = WavelengthColumnName("LAMBDA") +IntensityColumnName = NewType("IntensityColumnName", str) +"""The name of the intensity column in the mtz file.""" +DEFAULT_INTENSITY_COLUMN_NAME = IntensityColumnName("I") + +SigmaIntensityColumnName = NewType("SigmaIntensityColumnName", str) +"""The name of the standard uncertainty of intensity column in the mtz file.""" +DEFAULT_SIGMA_INTENSITY_COLUMN_NAME = SigmaIntensityColumnName("SIGI") # Computed types RawMtz = NewType("RawMtz", gemmi.Mtz) @@ -202,6 +209,10 @@ def _rapio_asu_to_asu(row: pd.Series) -> list[int]: return rapio_asu.to_asu(row["hkl"], sg.operations())[0] merged_df["hkl_eq"] = merged_df.apply(_rapio_asu_to_asu, axis=1) + # Unpack the indices for later. + merged_df[["H_EQ", "K_EQ", "L_EQ"]] = pd.DataFrame( + merged_df['hkl_eq'].to_list(), index=merged_df.index + ) return NMXMtzDataFrame(merged_df) @@ -209,34 +220,45 @@ def _rapio_asu_to_asu(row: pd.Series) -> list[int]: def nmx_mtz_dataframe_to_scipp_dataarray( nmx_mtz_df: NMXMtzDataFrame, ) -> NMXMtzDataArray: - """Converts the reduced mtz dataframe to a scipp dataarray.""" + """Converts the reduced mtz dataframe to a scipp dataarray. + + The intensity, with column name :attr:`~DEFAULT_INTENSITY_COLUMN_NAME` + becomes the data and the standard uncertainty of intensity, + with column name :attr:`~DEFAULT_SIGMA_INTENSITY_COLUMN_NAME` + becomes the variances of the data. + + """ from scipp.compat.pandas_compat import from_pandas_dataframe, parse_bracket_header to_scipp = nmx_mtz_df.copy(deep=False) - # Add dummy data column - dummy_data_column_name = "DUMMY_DATA" - to_scipp[dummy_data_column_name] = np.ones(len(to_scipp)) - # Pop the vector columns for later - vector_columns = ("hkl", "hkl_eq") - vector_coords = {col: to_scipp.pop(col) for col in vector_columns} + # Pop the indices columns. + # TODO: We can put them back once we support tuple[int] dtype. + # See https://github.com/scipp/scipp/issues/3046 for more details. + # As a temporary solution, we will use individual indices columns. + for col in ("hkl", "hkl_eq"): + to_scipp.pop(col) # Convert to scipp Dataset nmx_mtz_ds = from_pandas_dataframe( to_scipp, - data_columns=dummy_data_column_name, + data_columns=[ + DEFAULT_INTENSITY_COLUMN_NAME, + DEFAULT_SIGMA_INTENSITY_COLUMN_NAME, + ], header_parser=parse_bracket_header, ) - # Add units nmx_mtz_ds.coords[DEFAULT_WAVELENGTH_COLUMN_NAME].unit = sc.units.angstrom - nmx_mtz_ds.coords["I"].unit = sc.units.dimensionless - nmx_mtz_ds.coords["SIGI"].unit = sc.units.dimensionless - # Add back the vector columns - for col, values in vector_coords.items(): - nmx_mtz_ds.coords[col] = sc.vectors( - dims=nmx_mtz_ds.dims, values=[val for val in values] - ) + for key in nmx_mtz_ds.keys(): + nmx_mtz_ds[key].unit = sc.units.dimensionless + for coord in ("H", "K", "L", "H_EQ", "K_EQ", "L_EQ"): + nmx_mtz_ds.coords[coord].unit = sc.units.dimensionless + + # Add variances + nmx_mtz_da = nmx_mtz_ds[DEFAULT_INTENSITY_COLUMN_NAME].copy(deep=False) + nmx_mtz_da.variances = nmx_mtz_ds[DEFAULT_SIGMA_INTENSITY_COLUMN_NAME].data ** 2 + # Return DataArray - return NMXMtzDataArray(nmx_mtz_ds[dummy_data_column_name].copy(deep=False)) + return NMXMtzDataArray(nmx_mtz_da) mtz_io_providers = ( diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index e8510f2d..4dfcaba0 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) import io import pathlib -from typing import Any, Callable, Optional, Sequence, Union +from typing import Optional, Union import h5py import numpy as np @@ -25,58 +25,58 @@ def origin_position(self) -> sc.Variable: """Position of the first pixel (lowest ID) in the detector. Relative position from the sample.""" - return self['origin_position'] + return self["origin_position"] @property def crystal_rotation(self) -> sc.Variable: """Rotation of the crystal.""" - return self['crystal_rotation'] + return self["crystal_rotation"] @property def sample_name(self) -> sc.Variable: - return self['sample_name'] + return self["sample_name"] @property def fast_axis(self) -> sc.Variable: """Fast axis, along where the pixel ID increases by 1.""" - return self['fast_axis'] + return self["fast_axis"] @property def slow_axis(self) -> sc.Variable: """Slow axis, along where the pixel ID increases by > 1. The pixel ID increases by the number of pixels in the fast axis.""" - return self['slow_axis'] + return self["slow_axis"] @property def proton_charge(self) -> sc.Variable: """Accumulated number of protons during the measurement.""" - return self['proton_charge'] + return self["proton_charge"] @property def source_position(self) -> sc.Variable: """Relative position of the source from the sample.""" - return self['source_position'] + return self["source_position"] @property def sample_position(self) -> sc.Variable: """Relative position of the sample from the sample. (0, 0, 0)""" - return self['sample_position'] + return self["sample_position"] class NMXData(_SharedFields, sc.DataGroup): @property def weights(self) -> sc.DataArray: """Event data grouped by pixel id.""" - return self['weights'] + return self["weights"] class NMXReducedData(_SharedFields, sc.DataGroup): @property def counts(self) -> sc.DataArray: """Binned time of arrival data from flattened event data.""" - return self['counts'] + return self["counts"] def _create_dataset_from_var( self, @@ -137,8 +137,8 @@ def _create_sample_group(self, nx_entry: h5py.Group) -> h5py.Group: self._create_dataset_from_var( root_entry=nx_sample, var=self.crystal_rotation, - name='crystal_rotation', - long_name='crystal rotation in Phi (XYZ)', + name="crystal_rotation", + long_name="crystal rotation in Phi (XYZ)", ) return nx_sample @@ -236,9 +236,9 @@ def bin_time_of_arrival( nmx_data = list(nmx_data.values()) nmx_data = sc.concat(nmx_data, DETECTOR_DIM) - counts = nmx_data.pop('weights').hist(t=time_bin_step) + counts = nmx_data.pop("weights").hist(t=time_bin_step) new_coords = instrument.to_coords(*detector_name.values()) - new_coords.pop('pixel_id') + new_coords.pop("pixel_id") return NMXReducedData( counts=counts, @@ -246,103 +246,55 @@ def bin_time_of_arrival( ) -def _apply_elem_wise( - func: Callable, var: sc.Variable, *, result_dtype: Any = None -) -> sc.Variable: - """Apply a function element-wise to the variable values. - - This helper is only for vector-dtype variables. - Use ``numpy.vectorize`` for other types. - - Parameters - ---------- - func: - The function to apply. - var: - The variable to apply the function to. - result_dtype: - The dtype of the resulting variable. - It is needed especially when the function returns a vector. - - """ - - def apply_func(val: Sequence, _cur_depth: int = 0) -> list: - if _cur_depth == len(var.dims): - return func(val) - return [apply_func(v, _cur_depth + 1) for v in val] - - if result_dtype is None: - return sc.Variable( - dims=var.dims, - values=apply_func(var.values), - ) - return sc.Variable( - dims=var.dims, - values=apply_func(var.values), - dtype=result_dtype, +def _join_variables(*vars: sc.Variable, splitter: str = " ") -> sc.Variable: + # Check if all columns are integer + if not all(var.dtype == int for var in vars): + raise ValueError("All columns must be integer type.") + # Check if all columns have the same dimensions + dims = set(var.dim for var in vars) + if len(dims) != 1: + raise ValueError("All columns must have the same dimensions.") + return sc.array( + dims=dims, + values=[ + splitter.join(str(val) for val in row) + for row in zip(*(var.values for var in vars)) + ], ) -def _detour_group( - da: sc.DataArray, group_name: str, detour_func: Callable -) -> sc.DataArray: - """Group the data array by a hash of a coordinate. - - It uses index of each unique hash value - for grouping instead of hash value itself - to avoid overflow issues. +def _split_variable(var: sc.Variable, splitter: str = " ") -> tuple[sc.Variable, ...]: + if var.dtype != str: + raise ValueError("The variable must be string type.") + separated = [val.split(splitter) for val in var.values] + # Check if all rows have the same length + lengths = set(len(row) for row in separated) + if len(lengths) != 1: + raise ValueError("All rows must have the same length.") - """ - from uuid import uuid4 - - copied = da.copy(deep=False) - - # Temporary coords for grouping - detour_idx_coord_name = uuid4().hex + "hash_idx" - - # Create a temporary detoured coordinate - detour_var = _apply_elem_wise(detour_func, da.coords[group_name]) - # Create a temporary hash-index of each unique value - unique_hashes = np.unique(detour_var.values) - hash_to_idx = {hash_val: idx for idx, hash_val in enumerate(unique_hashes)} - copied.coords[detour_idx_coord_name] = _apply_elem_wise( - lambda idx: hash_to_idx[idx], detour_var - ) - - # Group by the hash-index - grouped = copied.group(detour_idx_coord_name) - - # Restore the original values - idx_to_detour = {idx: hash_val for hash_val, idx in hash_to_idx.items()} - detour_to_var = { - hash_val: var - for var, hash_val in zip(da.coords[group_name].values, detour_var.values) - } - idx_to_var = { - idx: detour_to_var[hash_val] for idx, hash_val in idx_to_detour.items() - } - grouped.coords[group_name] = _apply_elem_wise( - lambda idx: idx_to_var[idx], - grouped.coords[detour_idx_coord_name], - result_dtype=da.coords[group_name].dtype, - ) - # Rename dims back to group_name and drop the temporary hash-index coordinate - return grouped.rename_dims({detour_idx_coord_name: group_name}).drop_coords( - [detour_idx_coord_name] + return tuple( + sc.array(dims=var.dims, values=[int(row[i]) for row in separated], dtype=int) + for i in range(lengths.pop()) ) -def _group(da: sc.DataArray, /, *args: str, **group_detour_func_map) -> sc.DataArray: +def _zip_and_group(da: sc.DataArray, /, *args: str | sc.Variable) -> sc.DataArray: """Group the data array by the given coordinates. + It is a temporary solution before we fix the issue in scipp. + It should be replaced with the scipp group function once it's possible + to group by string-type coordinates or ``tuple[int]`` type of coordinates. + See [#3046](https://github.com/scipp/scipp/issues/3046) and + [#3425](https://github.com/scipp/scipp/issues/3425) for more details. + Parameters ---------- da: The data array to group. args: The coordinates to group by. - group_hash_func_map: - The hash functions for each coordinate. + group_detour_func_map: + The conversion functions. Returns ------- @@ -350,18 +302,33 @@ def _group(da: sc.DataArray, /, *args: str, **group_detour_func_map) -> sc.DataA The grouped data array. """ - grouped = da - for group_name in args: - if group_name in group_detour_func_map: - grouped = _detour_group( - grouped, group_name, group_detour_func_map[group_name] - ) - else: - try: - grouped = sc.group(grouped, group_name) - except Exception: - grouped = _detour_group( - grouped, group_name, group_detour_func_map.get(group_name, hash) - ) - - return grouped + copied = da.copy(deep=False) + group_coord_names = tuple(arg if isinstance(arg, str) else arg.dim for arg in args) + tmp_str_coord_name = "".join(group_coord_names) + group_coords = tuple( + copied.coords[name] for name in group_coord_names + ) # Must keep the order + + tmp_coord = _join_variables(*group_coords) + copied.coords[tmp_str_coord_name] = tmp_coord + + if all(isinstance(arg, str) for arg in args): + group_var = sc.array( + dims=[tmp_str_coord_name], + values=np.unique(copied.coords[tmp_str_coord_name].values), + ) + elif all(isinstance(arg, sc.Variable) for arg in args): + group_var = _join_variables( + *[arg.rename_dims({arg.dim: tmp_str_coord_name}) for arg in args] + ) + else: + raise ValueError("All coordinates must be either str or sc.Variable.") + + grouped = copied.group(group_var) + real_coords = _split_variable(grouped.coords[tmp_str_coord_name]) + for i_coord, name in enumerate(group_coord_names): + grouped.coords[name] = real_coords[i_coord].rename_dims( + {real_coords[i_coord].dim: grouped.dim} + ) + + return grouped.drop_coords([tmp_str_coord_name]) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 944401c8..0976548a 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -5,7 +5,7 @@ import scipp as sc from .mtz_io import DEFAULT_WAVELENGTH_COLUMN_NAME, NMXMtzDataArray -from .reduction import _group +from .reduction import _zip_and_group # User defined or configurable types WavelengthBinSize = NewType("WavelengthBinSize", int) @@ -81,16 +81,10 @@ def get_reference_bin(binned: WavelengthBinned) -> ReferenceWavelengthBin: def calculate_scale_factor_per_hkl_eq( ref_bin: ReferenceWavelengthBin, ) -> ReferenceScaleFactor: - grouped = _group(ref_bin, "hkl_eq", hkl_eq=str) + grouped = _zip_and_group(ref_bin, "H_EQ", "K_EQ", "L_EQ") + grouped = grouped.rename_dims({grouped.dim: "HKL_EQ"}) - scale_factor_coords = ("I", "SIGI") - for coord_name in scale_factor_coords: - grouped.coords[f"scale_factor_{coord_name}"] = sc.concat( - [sc.mean(1 / gr.values.coords[coord_name]) for gr in grouped], - dim=grouped.dim, - ) - - return ReferenceScaleFactor(grouped) + return ReferenceScaleFactor((1 / grouped).bins.mean()) # Providers and default parameters diff --git a/packages/essnmx/tests/reduction_test.py b/packages/essnmx/tests/reduction_test.py index 0adb1d56..106e2c2f 100644 --- a/packages/essnmx/tests/reduction_test.py +++ b/packages/essnmx/tests/reduction_test.py @@ -2,59 +2,80 @@ # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) import scipp as sc -from ess.nmx.reduction import _apply_elem_wise, _group +from ess.nmx.reduction import _zip_and_group -def test_apply_elem_wise_add() -> None: - var = sc.Variable(dims=["x"], values=[1, 2, 3]) +def test_zip_and_group_str() -> None: + from ess.nmx.scaling import _zip_and_group - assert sc.identical( - _apply_elem_wise(lambda x: x + 1, var), - sc.Variable(dims=["x"], values=var.values + 1), + da = sc.DataArray( + data=sc.ones(dims=["xy"], shape=[6]), + coords={ + "x": sc.array(dims=["xy"], values=[1, 1, 2, 2, 3, 3]), + "y": sc.array(dims=["xy"], values=[0, 1, 2, 2, 0, 3]), + }, ) - -def test_apply_elem_wise_str() -> None: - var = sc.Variable(dims=["x"], values=[1, 2, 3]) - + grouped = _zip_and_group(da, "x", "y") + assert sc.identical( + grouped.coords["x"], + sc.array(dims=["xy"], values=[1, 1, 2, 3, 3]), + ) assert sc.identical( - _apply_elem_wise(str, var), - sc.Variable(dims=["x"], values=["1", "2", "3"]), + grouped.coords["y"], + sc.array(dims=["xy"], values=[0, 1, 2, 0, 3]), ) -def test_apply_elem_wise_vectors() -> None: - var = sc.vectors(dims=["x"], values=[(1, 2, 3), (4, 5, 6), (7, 8, 9)]) - - assert sc.identical( - _apply_elem_wise(sum, var), - sc.array(dims=["x"], values=[6, 15, 24], dtype=float), +def test_zip_and_group_variables_all_possibilities() -> None: + da = sc.DataArray( + data=sc.ones(dims=["xy"], shape=[6]), + coords={ + "x": sc.array(dims=["xy"], values=[1, 1, 2, 2, 3, 3]), + "y": sc.array(dims=["xy"], values=[0, 1, 2, 2, 0, 3]), + }, ) + var_x = sc.array(dims=["x"], values=[1, 1, 2, 3, 3]) + var_y = sc.array(dims=["y"], values=[0, 1, 2, 0, 3]) -def test_detour_group_str() -> None: - from ess.nmx.scaling import _group + grouped = _zip_and_group(da, var_x, var_y) + assert sc.identical(grouped.coords["x"], var_x.rename_dims({"x": "xy"})) + assert sc.identical(grouped.coords["y"], var_y.rename_dims({"y": "xy"})) + +def test_zip_and_group_variables_less_groups() -> None: da = sc.DataArray( - data=sc.ones(dims=["x"], shape=[3]), - coords={"x": sc.Variable(dims=["x"], values=["a", "b", "a"])}, + data=sc.ones(dims=["xy"], shape=[6]), + coords={ + "x": sc.array(dims=["xy"], values=[1, 1, 2, 2, 3, 3]), + "y": sc.array(dims=["xy"], values=[0, 1, 2, 2, 0, 3]), + }, ) - grouped = _group(da, "x", x=lambda x: x) - assert sc.identical( - grouped.coords["x"], - sc.Variable(dims=["x"], values=["a", "b"]), - ) + var_x = sc.array(dims=["x"], values=[1, 1, 3, 3]) + var_y = sc.array(dims=["y"], values=[0, 2, 0, 3]) + + grouped = _zip_and_group(da, var_x, var_y) + assert sc.identical(grouped.coords["x"], var_x.rename_dims({"x": "xy"})) + assert sc.identical(grouped.coords["y"], var_y.rename_dims({"y": "xy"})) -def test_detour_group_vector() -> None: +def test_zip_and_group_variables_empty_group() -> None: da = sc.DataArray( - data=sc.ones(dims=["x"], shape=[10]), - coords={"x": sc.vectors(dims=["x"], values=[(1, 2, 3), (4, 5, 6)] * 5)}, + data=sc.ones(dims=["xy"], shape=[6]), + coords={ + "x": sc.array(dims=["xy"], values=[1, 1, 2, 2, 3, 3]), + "y": sc.array(dims=["xy"], values=[0, 1, 2, 2, 0, 3]), + }, ) - grouped = _group(da, "x", x=str) - assert sc.identical( - grouped.coords["x"], - sc.vectors(dims=["x"], values=[(1, 2, 3), (4, 5, 6)]), - ) + var_x = sc.array(dims=["x"], values=[1, 1, 3, 3, 4]) + var_y = sc.array(dims=["y"], values=[0, 2, 0, 3, 4]) + + grouped = _zip_and_group(da, var_x, var_y) + assert sc.identical(grouped.coords["x"], var_x.rename_dims({"x": "xy"})) + assert sc.identical(grouped.coords["y"], var_y.rename_dims({"y": "xy"})) + assert grouped[0].values.size == 1 + # last group should be empty + assert grouped[-1].values.size == 0 diff --git a/packages/essnmx/tests/scaling_test.py b/packages/essnmx/tests/scaling_test.py index 29879924..fb89fe67 100644 --- a/packages/essnmx/tests/scaling_test.py +++ b/packages/essnmx/tests/scaling_test.py @@ -13,30 +13,21 @@ @pytest.fixture def nmx_data_array() -> sc.DataArray: - return sc.DataArray( - data=sc.ones(dims=["row"], shape=[7]), + da = sc.DataArray( + data=sc.array(dims=["row"], values=[1, 2, 3, 4, 5, 3.1, 3.2]), coords={ DEFAULT_WAVELENGTH_COLUMN_NAME: sc.Variable( dims=["row"], values=[1, 2, 3, 4, 5, 3, 3] ), - "hkl_eq": sc.vectors( - dims=["row"], - values=[ - (1, 2, 3), - (4, 5, 6), - (7, 8, 9), - (10, 11, 12), - (13, 14, 15), - (7, 8, 9), - (9, 8, 7), - ], - ), - "I": sc.Variable(dims=["row"], values=[1, 2, 3, 4, 5, 3.1, 3.2]), - "SIGI": sc.Variable( - dims=["row"], values=[0.1, 0.2, 0.3, 0.4, 0.5, 0.31, 0.32] - ), + "H_EQ": sc.array(dims=["row"], values=[1, 4, 7, 10, 13, 7, 9]), + "K_EQ": sc.array(dims=["row"], values=[2, 5, 8, 11, 14, 8, 8]), + "L_EQ": sc.array(dims=["row"], values=[3, 6, 9, 12, 15, 9, 7]), }, ) + da.variances = ( + sc.array(dims=["row"], values=[0.1, 0.2, 0.3, 0.4, 0.5, 0.31, 0.32]) ** 2 + ) + return da def test_get_reference_bin_middle(nmx_data_array: sc.DataArray) -> None: @@ -44,11 +35,9 @@ def test_get_reference_bin_middle(nmx_data_array: sc.DataArray) -> None: ref_bin = get_reference_bin(nmx_data_array.bin({DEFAULT_WAVELENGTH_COLUMN_NAME: 6})) selected_idx = (2, 5, 6) - for coord in ("I", "SIGI"): - assert all( - ref_bin.coords[coord].values - == [nmx_data_array.coords[coord].values[idx] for idx in selected_idx] - ) + assert all( + ref_bin.data.values == [nmx_data_array.data.values[idx] for idx in selected_idx] + ) @pytest.fixture @@ -58,16 +47,13 @@ def reference_bin(nmx_data_array: sc.DataArray) -> ReferenceWavelengthBin: def test_reference_bin_scale_factor(reference_bin: ReferenceWavelengthBin) -> None: """Test the scale factor for I.""" - from ess.nmx.reduction import _group - - scale_factor_groups = calculate_scale_factor_per_hkl_eq(reference_bin) - grouped = _group(reference_bin, "hkl_eq", hkl_eq=str) + scale_factor = calculate_scale_factor_per_hkl_eq(reference_bin) + expected_groups = [(7, 8, 9), (9, 8, 7)] - for hkl_eq in grouped.coords["hkl_eq"].values: - calculated_gr = scale_factor_groups["hkl_eq", sc.vector(hkl_eq)] - reference_gr = grouped["hkl_eq", sc.vector(hkl_eq)] - for coord in ("I", "SIGI"): - assert sc.identical( - calculated_gr.coords[f"scale_factor_{coord}"], - sc.mean(1 / reference_gr.values.coords[coord]), - ) + assert len(scale_factor) == len(expected_groups) + for idx, group in enumerate(expected_groups): + hkl = tuple( + scale_factor.coords[coord][idx].value for coord in ("H_EQ", "K_EQ", "L_EQ") + ) + print(hkl) + assert hkl == group From e3c4be2369c680784391d9949187bd983fef889d Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 17 Apr 2024 19:02:33 +0200 Subject: [PATCH 140/403] Revert unnecessary formatting. --- packages/essnmx/src/ess/nmx/reduction.py | 54 ++++++++++++------------ 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 4dfcaba0..8492b6ad 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -25,58 +25,58 @@ def origin_position(self) -> sc.Variable: """Position of the first pixel (lowest ID) in the detector. Relative position from the sample.""" - return self["origin_position"] + return self['origin_position'] @property def crystal_rotation(self) -> sc.Variable: """Rotation of the crystal.""" - return self["crystal_rotation"] + return self['crystal_rotation'] @property def sample_name(self) -> sc.Variable: - return self["sample_name"] + return self['sample_name'] @property def fast_axis(self) -> sc.Variable: """Fast axis, along where the pixel ID increases by 1.""" - return self["fast_axis"] + return self['fast_axis'] @property def slow_axis(self) -> sc.Variable: """Slow axis, along where the pixel ID increases by > 1. The pixel ID increases by the number of pixels in the fast axis.""" - return self["slow_axis"] + return self['slow_axis'] @property def proton_charge(self) -> sc.Variable: """Accumulated number of protons during the measurement.""" - return self["proton_charge"] + return self['proton_charge'] @property def source_position(self) -> sc.Variable: """Relative position of the source from the sample.""" - return self["source_position"] + return self['source_position'] @property def sample_position(self) -> sc.Variable: """Relative position of the sample from the sample. (0, 0, 0)""" - return self["sample_position"] + return self['sample_position'] class NMXData(_SharedFields, sc.DataGroup): @property def weights(self) -> sc.DataArray: """Event data grouped by pixel id.""" - return self["weights"] + return self['weights'] class NMXReducedData(_SharedFields, sc.DataGroup): @property def counts(self) -> sc.DataArray: """Binned time of arrival data from flattened event data.""" - return self["counts"] + return self['counts'] def _create_dataset_from_var( self, @@ -90,18 +90,18 @@ def _create_dataset_from_var( ) -> h5py.Dataset: compression_options = dict() if compression is not None: - compression_options["compression"] = compression + compression_options['compression'] = compression if compression_opts is not None: - compression_options["compression_opts"] = compression_opts + compression_options['compression_opts'] = compression_opts dataset = root_entry.create_dataset( name, data=var.values, **compression_options, ) - dataset.attrs["units"] = str(var.unit) + dataset.attrs['units'] = str(var.unit) if long_name is not None: - dataset.attrs["long_name"] = long_name + dataset.attrs['long_name'] = long_name return dataset def _create_compressed_dataset( @@ -123,16 +123,16 @@ def _create_compressed_dataset( def _create_root_data_entry(self, file_obj: h5py.File) -> h5py.Group: nx_entry = file_obj.create_group("NMX_data") - nx_entry.attrs["NX_class"] = "NXentry" - nx_entry.attrs["default"] = "data" - nx_entry.attrs["name"] = "NMX" - nx_entry["name"] = "NMX" - nx_entry["definition"] = "TOFRAW" + nx_entry.attrs['NX_class'] = "NXentry" + nx_entry.attrs['default'] = "data" + nx_entry.attrs['name'] = "NMX" + nx_entry['name'] = "NMX" + nx_entry['definition'] = "TOFRAW" return nx_entry def _create_sample_group(self, nx_entry: h5py.Group) -> h5py.Group: nx_sample = nx_entry.create_group("NXsample") - nx_sample["name"] = self.sample_name.value + nx_sample['name'] = self.sample_name.value # Crystal rotation self._create_dataset_from_var( root_entry=nx_sample, @@ -190,12 +190,12 @@ def _create_detector_group(self, nx_entry: h5py.Group) -> h5py.Group: def _create_source_group(self, nx_entry: h5py.Group) -> h5py.Group: nx_source = nx_entry.create_group("NXsource") - nx_source["name"] = "European Spallation Source" - nx_source["short_name"] = "ESS" - nx_source["type"] = "Spallation Neutron Source" - nx_source["distance"] = sc.norm(self.source_position).value - nx_source["probe"] = "neutron" - nx_source["target_material"] = "W" + nx_source['name'] = "European Spallation Source" + nx_source['short_name'] = "ESS" + nx_source['type'] = "Spallation Neutron Source" + nx_source['distance'] = sc.norm(self.source_position).value + nx_source['probe'] = "neutron" + nx_source['target_material'] = "W" return nx_source def export_as_nexus( @@ -213,7 +213,7 @@ def export_as_nexus( file_base = output_file_base with h5py.File(file_base, "w") as out_file: - out_file.attrs["default"] = "NMX_data" + out_file.attrs['default'] = "NMX_data" # Root Data Entry nx_entry = self._create_root_data_entry(out_file) # Sample From 77cbe2bb601ce21e507aa68cd0f31f30547788cd Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 17 Apr 2024 19:04:51 +0200 Subject: [PATCH 141/403] Revert unnecessary formatting. --- packages/essnmx/src/ess/nmx/reduction.py | 58 ++++++++++++------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 8492b6ad..963e6d1b 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -25,58 +25,58 @@ def origin_position(self) -> sc.Variable: """Position of the first pixel (lowest ID) in the detector. Relative position from the sample.""" - return self['origin_position'] + return self["origin_position"] @property def crystal_rotation(self) -> sc.Variable: """Rotation of the crystal.""" - return self['crystal_rotation'] + return self["crystal_rotation"] @property def sample_name(self) -> sc.Variable: - return self['sample_name'] + return self["sample_name"] @property def fast_axis(self) -> sc.Variable: """Fast axis, along where the pixel ID increases by 1.""" - return self['fast_axis'] + return self["fast_axis"] @property def slow_axis(self) -> sc.Variable: """Slow axis, along where the pixel ID increases by > 1. The pixel ID increases by the number of pixels in the fast axis.""" - return self['slow_axis'] + return self["slow_axis"] @property def proton_charge(self) -> sc.Variable: """Accumulated number of protons during the measurement.""" - return self['proton_charge'] + return self["proton_charge"] @property def source_position(self) -> sc.Variable: """Relative position of the source from the sample.""" - return self['source_position'] + return self["source_position"] @property def sample_position(self) -> sc.Variable: """Relative position of the sample from the sample. (0, 0, 0)""" - return self['sample_position'] + return self["sample_position"] class NMXData(_SharedFields, sc.DataGroup): @property def weights(self) -> sc.DataArray: """Event data grouped by pixel id.""" - return self['weights'] + return self["weights"] class NMXReducedData(_SharedFields, sc.DataGroup): @property def counts(self) -> sc.DataArray: """Binned time of arrival data from flattened event data.""" - return self['counts'] + return self["counts"] def _create_dataset_from_var( self, @@ -90,18 +90,18 @@ def _create_dataset_from_var( ) -> h5py.Dataset: compression_options = dict() if compression is not None: - compression_options['compression'] = compression + compression_options["compression"] = compression if compression_opts is not None: - compression_options['compression_opts'] = compression_opts + compression_options["compression_opts"] = compression_opts dataset = root_entry.create_dataset( name, data=var.values, **compression_options, ) - dataset.attrs['units'] = str(var.unit) + dataset.attrs["units"] = str(var.unit) if long_name is not None: - dataset.attrs['long_name'] = long_name + dataset.attrs["long_name"] = long_name return dataset def _create_compressed_dataset( @@ -123,22 +123,22 @@ def _create_compressed_dataset( def _create_root_data_entry(self, file_obj: h5py.File) -> h5py.Group: nx_entry = file_obj.create_group("NMX_data") - nx_entry.attrs['NX_class'] = "NXentry" - nx_entry.attrs['default'] = "data" - nx_entry.attrs['name'] = "NMX" - nx_entry['name'] = "NMX" - nx_entry['definition'] = "TOFRAW" + nx_entry.attrs["NX_class"] = "NXentry" + nx_entry.attrs["default"] = "data" + nx_entry.attrs["name"] = "NMX" + nx_entry["name"] = "NMX" + nx_entry["definition"] = "TOFRAW" return nx_entry def _create_sample_group(self, nx_entry: h5py.Group) -> h5py.Group: nx_sample = nx_entry.create_group("NXsample") - nx_sample['name'] = self.sample_name.value + nx_sample["name"] = self.sample_name.value # Crystal rotation self._create_dataset_from_var( root_entry=nx_sample, var=self.crystal_rotation, - name="crystal_rotation", - long_name="crystal rotation in Phi (XYZ)", + name='crystal_rotation', + long_name='crystal rotation in Phi (XYZ)', ) return nx_sample @@ -190,12 +190,12 @@ def _create_detector_group(self, nx_entry: h5py.Group) -> h5py.Group: def _create_source_group(self, nx_entry: h5py.Group) -> h5py.Group: nx_source = nx_entry.create_group("NXsource") - nx_source['name'] = "European Spallation Source" - nx_source['short_name'] = "ESS" - nx_source['type'] = "Spallation Neutron Source" - nx_source['distance'] = sc.norm(self.source_position).value - nx_source['probe'] = "neutron" - nx_source['target_material'] = "W" + nx_source["name"] = "European Spallation Source" + nx_source["short_name"] = "ESS" + nx_source["type"] = "Spallation Neutron Source" + nx_source["distance"] = sc.norm(self.source_position).value + nx_source["probe"] = "neutron" + nx_source["target_material"] = "W" return nx_source def export_as_nexus( @@ -213,7 +213,7 @@ def export_as_nexus( file_base = output_file_base with h5py.File(file_base, "w") as out_file: - out_file.attrs['default'] = "NMX_data" + out_file.attrs["default"] = "NMX_data" # Root Data Entry nx_entry = self._create_root_data_entry(out_file) # Sample From 17f64b2e3332ee2d281e37d292b30d972a0726db Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 17 Apr 2024 19:07:15 +0200 Subject: [PATCH 142/403] Revert unnecessary formatting. --- packages/essnmx/src/ess/nmx/reduction.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 963e6d1b..a87e54c2 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -25,58 +25,58 @@ def origin_position(self) -> sc.Variable: """Position of the first pixel (lowest ID) in the detector. Relative position from the sample.""" - return self["origin_position"] + return self['origin_position'] @property def crystal_rotation(self) -> sc.Variable: """Rotation of the crystal.""" - return self["crystal_rotation"] + return self['crystal_rotation'] @property def sample_name(self) -> sc.Variable: - return self["sample_name"] + return self['sample_name'] @property def fast_axis(self) -> sc.Variable: """Fast axis, along where the pixel ID increases by 1.""" - return self["fast_axis"] + return self['fast_axis'] @property def slow_axis(self) -> sc.Variable: """Slow axis, along where the pixel ID increases by > 1. The pixel ID increases by the number of pixels in the fast axis.""" - return self["slow_axis"] + return self['slow_axis'] @property def proton_charge(self) -> sc.Variable: """Accumulated number of protons during the measurement.""" - return self["proton_charge"] + return self['proton_charge'] @property def source_position(self) -> sc.Variable: """Relative position of the source from the sample.""" - return self["source_position"] + return self['source_position'] @property def sample_position(self) -> sc.Variable: """Relative position of the sample from the sample. (0, 0, 0)""" - return self["sample_position"] + return self['sample_position'] class NMXData(_SharedFields, sc.DataGroup): @property def weights(self) -> sc.DataArray: """Event data grouped by pixel id.""" - return self["weights"] + return self['"weights'] class NMXReducedData(_SharedFields, sc.DataGroup): @property def counts(self) -> sc.DataArray: """Binned time of arrival data from flattened event data.""" - return self["counts"] + return self['counts'] def _create_dataset_from_var( self, @@ -236,9 +236,9 @@ def bin_time_of_arrival( nmx_data = list(nmx_data.values()) nmx_data = sc.concat(nmx_data, DETECTOR_DIM) - counts = nmx_data.pop("weights").hist(t=time_bin_step) + counts = nmx_data.pop('weights').hist(t=time_bin_step) new_coords = instrument.to_coords(*detector_name.values()) - new_coords.pop("pixel_id") + new_coords.pop('pixel_id') return NMXReducedData( counts=counts, From 170a9bd2eb51abe4ac4bce3d8bbef3956a236fe5 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 17 Apr 2024 19:07:51 +0200 Subject: [PATCH 143/403] Fix typo. --- packages/essnmx/src/ess/nmx/reduction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index a87e54c2..f991d6cf 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -69,7 +69,7 @@ class NMXData(_SharedFields, sc.DataGroup): @property def weights(self) -> sc.DataArray: """Event data grouped by pixel id.""" - return self['"weights'] + return self['weights'] class NMXReducedData(_SharedFields, sc.DataGroup): From 051543bf486938dd3fe45157f2a72eff0e065b17 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 18 Apr 2024 14:06:10 +0200 Subject: [PATCH 144/403] Add length check and update comments. --- packages/essnmx/src/ess/nmx/reduction.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index f991d6cf..028e5ac5 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -247,13 +247,18 @@ def bin_time_of_arrival( def _join_variables(*vars: sc.Variable, splitter: str = " ") -> sc.Variable: - # Check if all columns are integer + # Check if all variables are integer if not all(var.dtype == int for var in vars): - raise ValueError("All columns must be integer type.") - # Check if all columns have the same dimensions + raise ValueError("All variables must be integer type.") + # Check if all variables have the same dimensions dims = set(var.dim for var in vars) if len(dims) != 1: - raise ValueError("All columns must have the same dimensions.") + raise ValueError("All variables must have the same dimensions.") + # Check if all variables have the same length + lengths = set(len(var.values) for var in vars) + if len(lengths) != 1: + raise ValueError("All variables must have the same length.") + return sc.array( dims=dims, values=[ From f8c4e9fd5159c4c3410de287c3f7569fe32372a5 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 18 Apr 2024 14:11:49 +0200 Subject: [PATCH 145/403] Add comments for workaround. --- packages/essnmx/src/ess/nmx/reduction.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 028e5ac5..ad99cb46 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -317,6 +317,8 @@ def _zip_and_group(da: sc.DataArray, /, *args: str | sc.Variable) -> sc.DataArra tmp_coord = _join_variables(*group_coords) copied.coords[tmp_str_coord_name] = tmp_coord + # Workaround for scipp issue #3425 + # See https://github.com/scipp/scipp/issues/3425 for more details if all(isinstance(arg, str) for arg in args): group_var = sc.array( dims=[tmp_str_coord_name], From df43c53e20009fb96a3ace1a39c41db8ab7ceffe Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 18 Apr 2024 14:20:22 +0200 Subject: [PATCH 146/403] Update wrong units. --- packages/essnmx/src/ess/nmx/mtz_io.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index e8be4dbd..20588864 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -250,8 +250,6 @@ def nmx_mtz_dataframe_to_scipp_dataarray( nmx_mtz_ds.coords[DEFAULT_WAVELENGTH_COLUMN_NAME].unit = sc.units.angstrom for key in nmx_mtz_ds.keys(): nmx_mtz_ds[key].unit = sc.units.dimensionless - for coord in ("H", "K", "L", "H_EQ", "K_EQ", "L_EQ"): - nmx_mtz_ds.coords[coord].unit = sc.units.dimensionless # Add variances nmx_mtz_da = nmx_mtz_ds[DEFAULT_INTENSITY_COLUMN_NAME].copy(deep=False) From fe2f5e488a17f688c74ca762adf5c0f62e7a8779 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 18 Apr 2024 14:33:01 +0200 Subject: [PATCH 147/403] Use group/flatten instead of zipandgrouping for reference bin. --- packages/essnmx/src/ess/nmx/scaling.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 0976548a..a3eaf905 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -5,7 +5,6 @@ import scipp as sc from .mtz_io import DEFAULT_WAVELENGTH_COLUMN_NAME, NMXMtzDataArray -from .reduction import _zip_and_group # User defined or configurable types WavelengthBinSize = NewType("WavelengthBinSize", int) @@ -81,10 +80,13 @@ def get_reference_bin(binned: WavelengthBinned) -> ReferenceWavelengthBin: def calculate_scale_factor_per_hkl_eq( ref_bin: ReferenceWavelengthBin, ) -> ReferenceScaleFactor: - grouped = _zip_and_group(ref_bin, "H_EQ", "K_EQ", "L_EQ") - grouped = grouped.rename_dims({grouped.dim: "HKL_EQ"}) + # Workaround for https://github.com/scipp/scipp/issues/3046 + grouped = ref_bin.group("H_EQ", "K_EQ", "L_EQ").flatten( + dims=["H_EQ", "K_EQ", "L_EQ"], to="HKL_EQ" + ) + non_empty = grouped[grouped.bins.size().data > sc.scalar(0, unit=None)] - return ReferenceScaleFactor((1 / grouped).bins.mean()) + return ReferenceScaleFactor((1 / non_empty).bins.mean()) # Providers and default parameters From e46b716c93c8f347a0abb6d164f2a87c5bd826ae Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 18 Apr 2024 14:42:45 +0200 Subject: [PATCH 148/403] Remove unnecessary local import.| --- packages/essnmx/tests/reduction_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/tests/reduction_test.py b/packages/essnmx/tests/reduction_test.py index 106e2c2f..bfbb84dd 100644 --- a/packages/essnmx/tests/reduction_test.py +++ b/packages/essnmx/tests/reduction_test.py @@ -6,7 +6,7 @@ def test_zip_and_group_str() -> None: - from ess.nmx.scaling import _zip_and_group + from ess.nmx.reduction import _zip_and_group da = sc.DataArray( data=sc.ones(dims=["xy"], shape=[6]), From 716b2928c0a19bfd509d5e2537ecd97b8e180d7f Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 18 Apr 2024 14:44:34 +0200 Subject: [PATCH 149/403] Remove unnecessary local import.| --- packages/essnmx/tests/reduction_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/essnmx/tests/reduction_test.py b/packages/essnmx/tests/reduction_test.py index bfbb84dd..c16597ca 100644 --- a/packages/essnmx/tests/reduction_test.py +++ b/packages/essnmx/tests/reduction_test.py @@ -6,8 +6,6 @@ def test_zip_and_group_str() -> None: - from ess.nmx.reduction import _zip_and_group - da = sc.DataArray( data=sc.ones(dims=["xy"], shape=[6]), coords={ From f35add1bcadeb464188c616388055ced78dc228f Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 18 Apr 2024 15:02:18 +0200 Subject: [PATCH 150/403] Remove unnecessary coordinate restortion. --- packages/essnmx/src/ess/nmx/reduction.py | 24 +--------------------- packages/essnmx/tests/reduction_test.py | 26 +++++++++--------------- 2 files changed, 11 insertions(+), 39 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index ad99cb46..73c1d830 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -268,21 +268,6 @@ def _join_variables(*vars: sc.Variable, splitter: str = " ") -> sc.Variable: ) -def _split_variable(var: sc.Variable, splitter: str = " ") -> tuple[sc.Variable, ...]: - if var.dtype != str: - raise ValueError("The variable must be string type.") - separated = [val.split(splitter) for val in var.values] - # Check if all rows have the same length - lengths = set(len(row) for row in separated) - if len(lengths) != 1: - raise ValueError("All rows must have the same length.") - - return tuple( - sc.array(dims=var.dims, values=[int(row[i]) for row in separated], dtype=int) - for i in range(lengths.pop()) - ) - - def _zip_and_group(da: sc.DataArray, /, *args: str | sc.Variable) -> sc.DataArray: """Group the data array by the given coordinates. @@ -331,11 +316,4 @@ def _zip_and_group(da: sc.DataArray, /, *args: str | sc.Variable) -> sc.DataArra else: raise ValueError("All coordinates must be either str or sc.Variable.") - grouped = copied.group(group_var) - real_coords = _split_variable(grouped.coords[tmp_str_coord_name]) - for i_coord, name in enumerate(group_coord_names): - grouped.coords[name] = real_coords[i_coord].rename_dims( - {real_coords[i_coord].dim: grouped.dim} - ) - - return grouped.drop_coords([tmp_str_coord_name]) + return copied.group(group_var) diff --git a/packages/essnmx/tests/reduction_test.py b/packages/essnmx/tests/reduction_test.py index c16597ca..d1e386ab 100644 --- a/packages/essnmx/tests/reduction_test.py +++ b/packages/essnmx/tests/reduction_test.py @@ -13,16 +13,10 @@ def test_zip_and_group_str() -> None: "y": sc.array(dims=["xy"], values=[0, 1, 2, 2, 0, 3]), }, ) + var_xy = sc.array(dims=["xy"], values=["1 0", "1 1", "2 2", "3 0", "3 3"]) grouped = _zip_and_group(da, "x", "y") - assert sc.identical( - grouped.coords["x"], - sc.array(dims=["xy"], values=[1, 1, 2, 3, 3]), - ) - assert sc.identical( - grouped.coords["y"], - sc.array(dims=["xy"], values=[0, 1, 2, 0, 3]), - ) + assert sc.identical(grouped.coords["xy"], var_xy) def test_zip_and_group_variables_all_possibilities() -> None: @@ -36,10 +30,10 @@ def test_zip_and_group_variables_all_possibilities() -> None: var_x = sc.array(dims=["x"], values=[1, 1, 2, 3, 3]) var_y = sc.array(dims=["y"], values=[0, 1, 2, 0, 3]) + var_xy = sc.array(dims=["xy"], values=["1 0", "1 1", "2 2", "3 0", "3 3"]) grouped = _zip_and_group(da, var_x, var_y) - assert sc.identical(grouped.coords["x"], var_x.rename_dims({"x": "xy"})) - assert sc.identical(grouped.coords["y"], var_y.rename_dims({"y": "xy"})) + assert sc.identical(grouped.coords["xy"], var_xy) def test_zip_and_group_variables_less_groups() -> None: @@ -51,12 +45,12 @@ def test_zip_and_group_variables_less_groups() -> None: }, ) - var_x = sc.array(dims=["x"], values=[1, 1, 3, 3]) - var_y = sc.array(dims=["y"], values=[0, 2, 0, 3]) + var_x = sc.array(dims=["x"], values=[1, 1, 3, 3, 4]) + var_y = sc.array(dims=["y"], values=[0, 2, 0, 3, 4]) + var_xy = sc.array(dims=["xy"], values=["1 0", "1 2", "3 0", "3 3", "4 4"]) grouped = _zip_and_group(da, var_x, var_y) - assert sc.identical(grouped.coords["x"], var_x.rename_dims({"x": "xy"})) - assert sc.identical(grouped.coords["y"], var_y.rename_dims({"y": "xy"})) + assert sc.identical(grouped.coords["xy"], var_xy) def test_zip_and_group_variables_empty_group() -> None: @@ -70,10 +64,10 @@ def test_zip_and_group_variables_empty_group() -> None: var_x = sc.array(dims=["x"], values=[1, 1, 3, 3, 4]) var_y = sc.array(dims=["y"], values=[0, 2, 0, 3, 4]) + var_xy = sc.array(dims=["xy"], values=["1 0", "1 2", "3 0", "3 3", "4 4"]) grouped = _zip_and_group(da, var_x, var_y) - assert sc.identical(grouped.coords["x"], var_x.rename_dims({"x": "xy"})) - assert sc.identical(grouped.coords["y"], var_y.rename_dims({"y": "xy"})) + assert sc.identical(grouped.coords["xy"], var_xy) assert grouped[0].values.size == 1 # last group should be empty assert grouped[-1].values.size == 0 From 279f7e4ddbee987c4bf747de7c307daf7d5e11d5 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 23 Apr 2024 09:35:05 +0200 Subject: [PATCH 151/403] Remove unused helper functions. --- packages/essnmx/src/ess/nmx/reduction.py | 74 ------------------------ packages/essnmx/tests/reduction_test.py | 73 ----------------------- 2 files changed, 147 deletions(-) delete mode 100644 packages/essnmx/tests/reduction_test.py diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 73c1d830..d0d92a88 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -5,7 +5,6 @@ from typing import Optional, Union import h5py -import numpy as np import sciline import scipp as sc @@ -244,76 +243,3 @@ def bin_time_of_arrival( counts=counts, **{**nmx_data, **new_coords}, ) - - -def _join_variables(*vars: sc.Variable, splitter: str = " ") -> sc.Variable: - # Check if all variables are integer - if not all(var.dtype == int for var in vars): - raise ValueError("All variables must be integer type.") - # Check if all variables have the same dimensions - dims = set(var.dim for var in vars) - if len(dims) != 1: - raise ValueError("All variables must have the same dimensions.") - # Check if all variables have the same length - lengths = set(len(var.values) for var in vars) - if len(lengths) != 1: - raise ValueError("All variables must have the same length.") - - return sc.array( - dims=dims, - values=[ - splitter.join(str(val) for val in row) - for row in zip(*(var.values for var in vars)) - ], - ) - - -def _zip_and_group(da: sc.DataArray, /, *args: str | sc.Variable) -> sc.DataArray: - """Group the data array by the given coordinates. - - It is a temporary solution before we fix the issue in scipp. - It should be replaced with the scipp group function once it's possible - to group by string-type coordinates or ``tuple[int]`` type of coordinates. - See [#3046](https://github.com/scipp/scipp/issues/3046) and - [#3425](https://github.com/scipp/scipp/issues/3425) for more details. - - Parameters - ---------- - da: - The data array to group. - args: - The coordinates to group by. - group_detour_func_map: - The conversion functions. - - Returns - ------- - sc.DataArray - The grouped data array. - - """ - copied = da.copy(deep=False) - group_coord_names = tuple(arg if isinstance(arg, str) else arg.dim for arg in args) - tmp_str_coord_name = "".join(group_coord_names) - group_coords = tuple( - copied.coords[name] for name in group_coord_names - ) # Must keep the order - - tmp_coord = _join_variables(*group_coords) - copied.coords[tmp_str_coord_name] = tmp_coord - - # Workaround for scipp issue #3425 - # See https://github.com/scipp/scipp/issues/3425 for more details - if all(isinstance(arg, str) for arg in args): - group_var = sc.array( - dims=[tmp_str_coord_name], - values=np.unique(copied.coords[tmp_str_coord_name].values), - ) - elif all(isinstance(arg, sc.Variable) for arg in args): - group_var = _join_variables( - *[arg.rename_dims({arg.dim: tmp_str_coord_name}) for arg in args] - ) - else: - raise ValueError("All coordinates must be either str or sc.Variable.") - - return copied.group(group_var) diff --git a/packages/essnmx/tests/reduction_test.py b/packages/essnmx/tests/reduction_test.py deleted file mode 100644 index d1e386ab..00000000 --- a/packages/essnmx/tests/reduction_test.py +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) -import scipp as sc - -from ess.nmx.reduction import _zip_and_group - - -def test_zip_and_group_str() -> None: - da = sc.DataArray( - data=sc.ones(dims=["xy"], shape=[6]), - coords={ - "x": sc.array(dims=["xy"], values=[1, 1, 2, 2, 3, 3]), - "y": sc.array(dims=["xy"], values=[0, 1, 2, 2, 0, 3]), - }, - ) - var_xy = sc.array(dims=["xy"], values=["1 0", "1 1", "2 2", "3 0", "3 3"]) - - grouped = _zip_and_group(da, "x", "y") - assert sc.identical(grouped.coords["xy"], var_xy) - - -def test_zip_and_group_variables_all_possibilities() -> None: - da = sc.DataArray( - data=sc.ones(dims=["xy"], shape=[6]), - coords={ - "x": sc.array(dims=["xy"], values=[1, 1, 2, 2, 3, 3]), - "y": sc.array(dims=["xy"], values=[0, 1, 2, 2, 0, 3]), - }, - ) - - var_x = sc.array(dims=["x"], values=[1, 1, 2, 3, 3]) - var_y = sc.array(dims=["y"], values=[0, 1, 2, 0, 3]) - var_xy = sc.array(dims=["xy"], values=["1 0", "1 1", "2 2", "3 0", "3 3"]) - - grouped = _zip_and_group(da, var_x, var_y) - assert sc.identical(grouped.coords["xy"], var_xy) - - -def test_zip_and_group_variables_less_groups() -> None: - da = sc.DataArray( - data=sc.ones(dims=["xy"], shape=[6]), - coords={ - "x": sc.array(dims=["xy"], values=[1, 1, 2, 2, 3, 3]), - "y": sc.array(dims=["xy"], values=[0, 1, 2, 2, 0, 3]), - }, - ) - - var_x = sc.array(dims=["x"], values=[1, 1, 3, 3, 4]) - var_y = sc.array(dims=["y"], values=[0, 2, 0, 3, 4]) - var_xy = sc.array(dims=["xy"], values=["1 0", "1 2", "3 0", "3 3", "4 4"]) - - grouped = _zip_and_group(da, var_x, var_y) - assert sc.identical(grouped.coords["xy"], var_xy) - - -def test_zip_and_group_variables_empty_group() -> None: - da = sc.DataArray( - data=sc.ones(dims=["xy"], shape=[6]), - coords={ - "x": sc.array(dims=["xy"], values=[1, 1, 2, 2, 3, 3]), - "y": sc.array(dims=["xy"], values=[0, 1, 2, 2, 0, 3]), - }, - ) - - var_x = sc.array(dims=["x"], values=[1, 1, 3, 3, 4]) - var_y = sc.array(dims=["y"], values=[0, 2, 0, 3, 4]) - var_xy = sc.array(dims=["xy"], values=["1 0", "1 2", "3 0", "3 3", "4 4"]) - - grouped = _zip_and_group(da, var_x, var_y) - assert sc.identical(grouped.coords["xy"], var_xy) - assert grouped[0].values.size == 1 - # last group should be empty - assert grouped[-1].values.size == 0 From e19ce43cf5a0b34401d202d6510f93f0c653b34c Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 23 Apr 2024 15:45:30 +0200 Subject: [PATCH 152/403] Rename types and functions with more clear meaning and update dosctrings. --- .../docs/about/data_workflow_overview.md | 2 +- .../docs/examples/scaling_workflow.ipynb | 4 +- packages/essnmx/src/ess/nmx/mtz_io.py | 167 ++++++++++++++---- packages/essnmx/tests/mtz_io_test.py | 32 +++- 4 files changed, 161 insertions(+), 44 deletions(-) diff --git a/packages/essnmx/docs/about/data_workflow_overview.md b/packages/essnmx/docs/about/data_workflow_overview.md index ca0de56a..29389c1d 100644 --- a/packages/essnmx/docs/about/data_workflow_overview.md +++ b/packages/essnmx/docs/about/data_workflow_overview.md @@ -21,7 +21,7 @@ Then the single events get binned into pixels and then histogramed in the TOF di This result can be exported to an HDF5 file along with additional metadata and instrument coordinates (pixel IDs). -See [workflow example](../examples/workflow_example) for more details. +See [workflow example](../examples/workflow) for more details. ### Spot finding and integration (DIALS) For the next five steps of the data reduction from spot finding to spot integration, diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index df6428a4..6cdc0bf7 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -115,9 +115,9 @@ "metadata": {}, "outputs": [], "source": [ - "from ess.nmx.scaling import get_reference_bin, calculate_scale_factor_per_hkl_eq\n", + "from ess.nmx.scaling import get_reference_bin, estimate_scale_factor_per_hkl_asu_from_reference_bin\n", "\n", - "scale_factor = calculate_scale_factor_per_hkl_eq(get_reference_bin(binned))\n", + "scale_factor = estimate_scale_factor_per_hkl_asu_from_reference_bin(get_reference_bin(binned))\n", "scale_factor" ] } diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index 20588864..4acda05a 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -25,14 +25,15 @@ WavelengthColumnName = NewType("WavelengthColumnName", str) """The name of the wavelength column in the mtz file.""" DEFAULT_WAVELENGTH_COLUMN_NAME = WavelengthColumnName("LAMBDA") +DEFAULT_WAVELENGTH_COORD_NAME = "wavelength" IntensityColumnName = NewType("IntensityColumnName", str) """The name of the intensity column in the mtz file.""" DEFAULT_INTENSITY_COLUMN_NAME = IntensityColumnName("I") -SigmaIntensityColumnName = NewType("SigmaIntensityColumnName", str) +IntensitySigColumnName = NewType("IntensitySigColumnName", str) """The name of the standard uncertainty of intensity column in the mtz file.""" -DEFAULT_SIGMA_INTENSITY_COLUMN_NAME = SigmaIntensityColumnName("SIGI") +DEFAULT_INTENSITY_SIG_COLUMN_NAME = IntensitySigColumnName("SIGI") # Computed types RawMtz = NewType("RawMtz", gemmi.Mtz) @@ -60,14 +61,16 @@ def mtz_to_pandas(mtz: gemmi.Mtz) -> pd.DataFrame: """Converts the mtz file to a pandas dataframe. It is equivalent to the following code: - ```python - import numpy as np - import pandas as pd - - data = np.array(mtz, copy=False) - columns = mtz.column_labels() - return pd.DataFrame(data, columns=columns) - ``` + + .. code-block:: python + + import numpy as np + import pandas as pd + + data = np.array(mtz, copy=False) + columns = mtz.column_labels() + return pd.DataFrame(data, columns=columns) + It is recommended in the gemmi documentation. """ @@ -79,9 +82,11 @@ def mtz_to_pandas(mtz: gemmi.Mtz) -> pd.DataFrame: ) -def reduce_single_mtz( +def process_single_mtz_to_dataframe( mtz: RawMtz, - lambda_column_name: WavelengthColumnName = DEFAULT_WAVELENGTH_COLUMN_NAME, + wavelength_column_name: WavelengthColumnName = DEFAULT_WAVELENGTH_COLUMN_NAME, + intensity_column_name: IntensityColumnName = DEFAULT_INTENSITY_COLUMN_NAME, + intensity_sig_col_name: IntensitySigColumnName = DEFAULT_INTENSITY_SIG_COLUMN_NAME, ) -> RawMtzDataFrame: """Select and derive columns from the original ``MtzDataFrame``. @@ -90,15 +95,35 @@ def reduce_single_mtz( mtz: The raw mtz dataset. - lambda_column_name: + wavelength_column_name: The name of the wavelength column in the mtz file. + intensity_column_name: + The name of the intensity column in the mtz file. + + sigma_intensity_column_name: + The name of the standard uncertainty of intensity column in the mtz file. + Returns ------- : - The new mtz dataframe with derived columns. + The new mtz dataframe with derived and renamed columns. + The derived columns are: + - ``hkl``: The miller indices as a list of integers. + - ``d``: The d-spacing calculated from the miller indices. + :math:``\\dfrac{2}{d^{2}} = \\dfrac{\\sin^2(\\theta)}{\\lambda^2}`` + - ``resolution``: The resolution calculated from the d-spacing. + + For consistent names of columns/coordinates, the following columns are renamed: + + - ``wavelength_column_name`` -> ``'wavelength'`` + - ``intensity_column_name`` -> ``'I'`` + - ``sigma_intensity_column_name`` -> ``'SIGI'`` + + Other columns are kept as they are. + Notes ----- :class:`pandas.DataFrame` is used from loading to merging, @@ -120,10 +145,10 @@ def _calculate_d(row: pd.Series) -> float: return mtz.get_cell().calculate_d(row["hkl"]) mtz_df["d"] = mtz_df.apply(_calculate_d, axis=1) - # (2d)^{-2} = \sin^2(\theta)/\lambda^2 mtz_df["resolution"] = (1 / mtz_df["d"]) ** 2 / 4 - mtz_df["I_div_SIGI"] = orig_df["I"] / orig_df["SIGI"] - mtz_df[DEFAULT_WAVELENGTH_COLUMN_NAME] = orig_df[lambda_column_name] + mtz_df[DEFAULT_WAVELENGTH_COORD_NAME] = orig_df[wavelength_column_name] + mtz_df[DEFAULT_INTENSITY_COLUMN_NAME] = orig_df[intensity_column_name] + mtz_df[DEFAULT_INTENSITY_SIG_COLUMN_NAME] = orig_df[intensity_sig_col_name] # Keep other columns for column in [col for col in orig_df.columns if col not in mtz_df]: mtz_df[column] = orig_df[column] @@ -193,7 +218,7 @@ def merge_mtz_dataframes( return MergedMtzDataFrame(pd.concat(mtz_dfs.values(), ignore_index=True)) -def reduce_merged_mtz_dataframe( +def process_merged_mtz_dataframe( *, merged_mtz_df: MergedMtzDataFrame, rapio_asu: RapioAsu, @@ -206,17 +231,62 @@ def reduce_merged_mtz_dataframe( merged_df = merged_mtz_df.copy(deep=False) def _rapio_asu_to_asu(row: pd.Series) -> list[int]: + """Converts miller indices(HKL) to ASU indices.""" return rapio_asu.to_asu(row["hkl"], sg.operations())[0] - merged_df["hkl_eq"] = merged_df.apply(_rapio_asu_to_asu, axis=1) + merged_df["hkl_asu"] = merged_df.apply(_rapio_asu_to_asu, axis=1) # Unpack the indices for later. - merged_df[["H_EQ", "K_EQ", "L_EQ"]] = pd.DataFrame( - merged_df['hkl_eq'].to_list(), index=merged_df.index + merged_df[["H_ASU", "K_ASU", "L_ASU"]] = pd.DataFrame( + merged_df["hkl_asu"].to_list(), index=merged_df.index ) return NMXMtzDataFrame(merged_df) +def _join_variables(*vars: sc.Variable, splitter: str = " ") -> sc.Variable: + """Joins multiple integer dtype variables into a single string dtype variable. + + Parameters + ---------- + vars: + The integer dtype variables to join with same dimensions and length. + + splitter: + The string to join the variables. + + Returns + ------- + : + The joined variable. It keeps the dimensions of the input variables. + But it drops the units since the output is a string. + + Raises + ------ + ValueError + If the input variables have different dimensions or lengths. + + """ + # Check if all variables are integer + if not all(var.dtype == int for var in vars): + raise ValueError("All variables must be integer type.") + # Check if all variables have the same dimensions + dims = set(var.dim for var in vars) + if len(dims) != 1: + raise ValueError("All variables must have the same dimensions.") + # Check if all variables have the same length + lengths = set(len(var.values) for var in vars) + if len(lengths) != 1: + raise ValueError("All variables must have the same length.") + + return sc.array( + dims=dims, + values=[ + splitter.join(str(val) for val in row) + for row in zip(*(var.values for var in vars)) + ], + ) + + def nmx_mtz_dataframe_to_scipp_dataarray( nmx_mtz_df: NMXMtzDataFrame, ) -> NMXMtzDataArray: @@ -227,33 +297,63 @@ def nmx_mtz_dataframe_to_scipp_dataarray( with column name :attr:`~DEFAULT_SIGMA_INTENSITY_COLUMN_NAME` becomes the variances of the data. + Parameters + ---------- + nmx_mtz_df: + The merged and processed mtz dataframe. + + Returns + ------- + : + The scipp dataarray with the intensity and variances. + The ``I`` column becomes the data and the + squared ``SIGI`` column becomes the variances. + Therefore they are not in the coordinates. + + Following coordinates are dropped from the dataframe: + + - ``hkl``: The miller indices as a list of integers. + There is no dtype that can represent this in scipp. + + Following coordinates are modified: + + - ``hkl_asu``: The miller indices as a string. + This coordinate will be used to derive estimated scale factors. + It is modified to have a string dtype + as the same reason as why ``hkl`` coordinate is dropped. + """ from scipp.compat.pandas_compat import from_pandas_dataframe, parse_bracket_header to_scipp = nmx_mtz_df.copy(deep=False) - # Pop the indices columns. - # TODO: We can put them back once we support tuple[int] dtype. - # See https://github.com/scipp/scipp/issues/3046 for more details. - # As a temporary solution, we will use individual indices columns. - for col in ("hkl", "hkl_eq"): - to_scipp.pop(col) # Convert to scipp Dataset nmx_mtz_ds = from_pandas_dataframe( to_scipp, data_columns=[ DEFAULT_INTENSITY_COLUMN_NAME, - DEFAULT_SIGMA_INTENSITY_COLUMN_NAME, + DEFAULT_INTENSITY_SIG_COLUMN_NAME, ], header_parser=parse_bracket_header, ) + # Pop the indices columns. + # TODO: We can put them back once we support tuple[int] dtype. + # See https://github.com/scipp/scipp/issues/3046 for more details. + # Temporarily, we will join them into a single string. + # It is done on the scipp variable instead of the dataframe + # since columns with string dtype are converted to PyObject dtype + # instead of string by `from_pandas_dataframe`. + nmx_mtz_ds = nmx_mtz_ds.drop_coords(["hkl", "hkl_asu"]) + nmx_mtz_ds.coords["hkl_asu"] = _join_variables( + *(nmx_mtz_ds.coords[f"{idx_desc}_ASU"] for idx_desc in "HKL") + ) # Add units - nmx_mtz_ds.coords[DEFAULT_WAVELENGTH_COLUMN_NAME].unit = sc.units.angstrom + nmx_mtz_ds.coords[DEFAULT_WAVELENGTH_COORD_NAME].unit = sc.units.angstrom for key in nmx_mtz_ds.keys(): nmx_mtz_ds[key].unit = sc.units.dimensionless # Add variances nmx_mtz_da = nmx_mtz_ds[DEFAULT_INTENSITY_COLUMN_NAME].copy(deep=False) - nmx_mtz_da.variances = nmx_mtz_ds[DEFAULT_SIGMA_INTENSITY_COLUMN_NAME].data ** 2 + nmx_mtz_da.variances = nmx_mtz_ds[DEFAULT_INTENSITY_SIG_COLUMN_NAME].data ** 2 # Return DataArray return NMXMtzDataArray(nmx_mtz_da) @@ -261,15 +361,18 @@ def nmx_mtz_dataframe_to_scipp_dataarray( mtz_io_providers = ( read_mtz_file, - reduce_single_mtz, + process_single_mtz_to_dataframe, get_space_group, get_reciprocal_asu, merge_mtz_dataframes, - reduce_merged_mtz_dataframe, + process_merged_mtz_dataframe, nmx_mtz_dataframe_to_scipp_dataarray, ) """The providers related to the MTZ IO.""" + mtz_io_params = { WavelengthColumnName: DEFAULT_WAVELENGTH_COLUMN_NAME, + IntensityColumnName: DEFAULT_INTENSITY_COLUMN_NAME, + IntensitySigColumnName: DEFAULT_INTENSITY_SIG_COLUMN_NAME, } """The parameters related to the MTZ IO.""" diff --git a/packages/essnmx/tests/mtz_io_test.py b/packages/essnmx/tests/mtz_io_test.py index f5bdb246..5c8f6057 100644 --- a/packages/essnmx/tests/mtz_io_test.py +++ b/packages/essnmx/tests/mtz_io_test.py @@ -5,6 +5,7 @@ import gemmi import pytest import sciline as sl +import scipp as sc from ess.nmx.data import get_small_mtz_samples from ess.nmx.mtz_io import DEFAULT_SPACE_GROUP_DESC # P 21 21 21 @@ -13,13 +14,14 @@ MTZFileIndex, MTZFilePath, RawMtz, + _join_variables, get_reciprocal_asu, get_space_group, merge_mtz_dataframes, mtz_to_pandas, + process_merged_mtz_dataframe, + process_single_mtz_to_dataframe, read_mtz_file, - reduce_merged_mtz_dataframe, - reduce_single_mtz, ) @@ -53,15 +55,15 @@ def test_mtz_to_pandas_dataframe(gemmi_mtz_object: gemmi.Mtz) -> None: def test_mtz_to_reduced_pandas_dataframe(gemmi_mtz_object: gemmi.Mtz) -> None: - df = reduce_single_mtz(RawMtz(gemmi_mtz_object)) - for expected_colum in ["hkl", "d", "resolution", "I_div_SIGI", *"HKL"]: + df = process_single_mtz_to_dataframe(RawMtz(gemmi_mtz_object)) + for expected_colum in ["hkl", "d", "resolution", *"HKL", "wavelength", "I", "SIGI"]: assert expected_colum in df.columns for hkl_column in "HKL": assert hkl_column in df.columns assert df[hkl_column].dtype == int - assert "hkl_eq" not in df.columns # It should be done on merged dataframes + assert "hkl_asu" not in df.columns # It should be done on merged dataframes @pytest.fixture @@ -127,7 +129,10 @@ def merged_mtz_dataframe( """Tests if the merged data frame has the expected columns.""" reduced_mtz_series = sl.Series( row_dim=MTZFileIndex, - items={i_file: reduce_single_mtz(mtz) for i_file, mtz in mtz_series.items()}, + items={ + i_file: process_single_mtz_to_dataframe(mtz) + for i_file, mtz in mtz_series.items() + }, ) return merge_mtz_dataframes(reduced_mtz_series) @@ -139,10 +144,19 @@ def test_reduce_merged_mtz_dataframe( space_gr = get_space_group(mtz_series) rapio_asu = get_reciprocal_asu(space_gr) - nmx_df = reduce_merged_mtz_dataframe( + nmx_df = process_merged_mtz_dataframe( merged_mtz_df=merged_mtz_dataframe, rapio_asu=rapio_asu, sg=space_gr, ) - assert "hkl_eq" not in merged_mtz_dataframe.columns - assert "hkl_eq" in nmx_df.columns + assert "hkl_asu" not in merged_mtz_dataframe.columns + assert "hkl_asu" in nmx_df.columns + + +def test_join_variables() -> None: + var_x = sc.array(dims=["xy"], values=[1, 1, 2, 3, 3], unit=None) + var_y = sc.array(dims=["xy"], values=[0, 1, 2, 0, 3], unit=None) + var_xy = sc.array(dims=["xy"], values=["1 0", "1 1", "2 2", "3 0", "3 3"]) + + joined = _join_variables(var_x, var_y, splitter=" ") + assert sc.identical(joined, var_xy) From be7d1fec933546bd6f494cec23bce015ec3fe235 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 23 Apr 2024 16:10:51 +0200 Subject: [PATCH 153/403] Update names and docstrings. --- .../docs/examples/scaling_workflow.ipynb | 16 +- packages/essnmx/src/ess/nmx/scaling.py | 139 +++++++++++++----- packages/essnmx/tests/scaling_test.py | 31 ++-- 3 files changed, 131 insertions(+), 55 deletions(-) diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index 6cdc0bf7..ad5e53d5 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -115,11 +115,23 @@ "metadata": {}, "outputs": [], "source": [ - "from ess.nmx.scaling import get_reference_bin, estimate_scale_factor_per_hkl_asu_from_reference_bin\n", + "from ess.nmx.scaling import get_reference_intensities, estimate_scale_factor_per_hkl_asu_from_reference\n", "\n", - "scale_factor = estimate_scale_factor_per_hkl_asu_from_reference_bin(get_reference_bin(binned))\n", + "scale_factor = estimate_scale_factor_per_hkl_asu_from_reference(get_reference_intensities(binned))\n", "scale_factor" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import scipp as sc\n", + "from ess.nmx.scaling import get_reference_intensities, ReferenceWavelength\n", + "\n", + "get_reference_intensities(binned, ReferenceWavelength(sc.scalar(3, unit=sc.units.angstrom)))" + ] } ], "metadata": { diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index a3eaf905..6ad63004 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -1,36 +1,29 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) -from typing import NewType +from typing import NewType, Optional import scipp as sc -from .mtz_io import DEFAULT_WAVELENGTH_COLUMN_NAME, NMXMtzDataArray +from .mtz_io import DEFAULT_WAVELENGTH_COORD_NAME, NMXMtzDataArray # User defined or configurable types WavelengthBinSize = NewType("WavelengthBinSize", int) """The size of the wavelength(LAMBDA) bins.""" - +ReferenceWavelength = NewType("ReferenceWavelength", sc.Variable) +"""The wavelength to select reference intensities.""" # Computed types WavelengthBinned = NewType("WavelengthBinned", sc.DataArray) """Binned mtz dataframe by wavelength(LAMBDA) with derived columns.""" -ReferenceWavelengthBin = NewType("ReferenceWavelengthBin", sc.DataArray) -"""The reference bin in the binned dataset.""" -ReferenceScaleFactor = NewType("ReferenceScaleFactor", sc.DataArray) -"""The reference scale factor, grouped by HKL_EQ.""" -ScaleFactorIntensity = NewType("ScaleFactorIntensity", float) -"""The scale factor for intensity.""" -ScaleFactorSigmaIntensity = NewType("ScaleFactorSigmaIntensity", float) -"""The scale factor for the standard uncertainty of intensity.""" -WavelengthScaled = NewType("WavelengthScaled", sc.DataArray) -"""Scaled wavelength by the reference bin.""" - - -def _is_bin_empty(binned: sc.DataArray, idx: int) -> bool: - return binned[idx].values.size == 0 +ReferenceIntensities = NewType("ReferenceIntensities", sc.DataArray) +"""Reference intensities selected by the wavelength.""" +EstimatedScaleFactor = NewType("EstimatedScaleFactor", sc.DataArray) +"""The estimated scale factor from the reference intensities per ``hkl_asu``.""" +EstimatedScaledIntensities = NewType("EstimatedScaledIntensities", float) +"""Scaled intensities by the estimated scale factor.""" -def get_lambda_binned( +def get_wavelength_binned( mtz_da: NMXMtzDataArray, wavelength_bin_size: WavelengthBinSize, ) -> WavelengthBinned: @@ -43,56 +36,122 @@ def get_lambda_binned( """ return WavelengthBinned( - mtz_da.bin({DEFAULT_WAVELENGTH_COLUMN_NAME: wavelength_bin_size}) + mtz_da.bin({DEFAULT_WAVELENGTH_COORD_NAME: wavelength_bin_size}) ) -def get_reference_bin(binned: WavelengthBinned) -> ReferenceWavelengthBin: - """Find the reference group in the binned dataset. +def _is_bin_empty(binned: sc.DataArray, idx: int) -> bool: + """Check if the bin is empty.""" + return binned[idx].values.size == 0 + + +def _get_middle_bin_idx(binned: WavelengthBinned) -> int: + """Find the middle bin index. + + If the middle one is empty, the function will search for the nearest. + """ + middle_number, offset = len(binned) // 2, 0 + + while 0 < (cur_idx := middle_number + offset) < len(binned) and _is_bin_empty( + binned, cur_idx + ): + offset = -offset + 1 if offset <= 0 else -offset + + if _is_bin_empty(binned, cur_idx): + raise ValueError("No reference group found.") - The reference group is the group in the middle of the binned dataset. - If the middle group is empty, the function will search for the nearest. + return cur_idx + + +def get_reference_intensities( + binned: WavelengthBinned, + reference_wavelength: Optional[ReferenceWavelength] = None, +) -> ReferenceIntensities: + """Find the reference intensities by the wavelength. Parameters ---------- binned: The wavelength binned data. + reference_wavelength: + The reference wavelength to select the intensities. + If ``None``, the middle group is selected. + It should be a scalar variable as it is selecting one of bins. + Raises ------ ValueError: If no reference group is found. """ - middle_number, offset = len(binned) // 2, 0 + if reference_wavelength is None: + ref_idx = _get_middle_bin_idx(binned) + return binned[ref_idx].values.copy(deep=False) + else: + if reference_wavelength.dims: + raise ValueError("Reference wavelength should be a scalar.") + try: + return binned['wavelength', reference_wavelength].values.copy(deep=False) + except IndexError: + raise IndexError(f"{reference_wavelength} out of range.") - while 0 < (cur_idx := middle_number + offset) < len(binned) and _is_bin_empty( - binned, cur_idx - ): - offset = -offset + 1 if offset <= 0 else -offset - if _is_bin_empty(binned, cur_idx): - raise ValueError("No reference group found.") +def estimate_scale_factor_per_hkl_asu_from_reference( + reference_intensities: ReferenceIntensities, +) -> EstimatedScaleFactor: + """Calculate the estimated scale factor per ``hkl_asu``. - return binned[cur_idx].values.copy(deep=False) + The estimated scale factor is calculatd as the average + of the inverse of the non-empty reference intensities. + It is part of the calculation of estimated scaled intensities + for fitting the scaling model. -def calculate_scale_factor_per_hkl_eq( - ref_bin: ReferenceWavelengthBin, -) -> ReferenceScaleFactor: + .. math:: + + EstimatedScaleFactor_{(hkl)} = \\dfrac{ + \\sum_{i=1}^{N_{(hkl)}} \\dfrac{1}{I_{i}} + }{ + N_{(hkl)} + } + = average( \\dfrac{1}{I_{(hkl)}} ) + + Estimated scale factor is calculated per ``hkl_asu``. + + Parameters + ---------- + reference_intensities: + The reference intensities selected by wavelength. + + Returns + ------- + : + The estimated scale factor per ``hkl_asu``. + The result should have a dimension of ``hkl_asu``. + + It does not have a dimension of ``wavelength`` since + it is calculated from the reference intensities, + which is selected by one ``wavelength``. + + """ # Workaround for https://github.com/scipp/scipp/issues/3046 - grouped = ref_bin.group("H_EQ", "K_EQ", "L_EQ").flatten( - dims=["H_EQ", "K_EQ", "L_EQ"], to="HKL_EQ" + # and https://github.com/scipp/scipp/issues/3425 + # This workaround is implemented with an assumption that + # the size of all combinations of (H_ASU, K_ASU, L_ASU) is small enough + # to be handled in memory. + grouped = reference_intensities.group("H_ASU", "K_ASU", "L_ASU").flatten( + dims=["H_ASU", "K_ASU", "L_ASU"], to="hkl_asu" ) non_empty = grouped[grouped.bins.size().data > sc.scalar(0, unit=None)] - return ReferenceScaleFactor((1 / non_empty).bins.mean()) + return EstimatedScaleFactor((1 / non_empty).bins.mean()) # Providers and default parameters scaling_providers = ( - get_lambda_binned, - get_reference_bin, - calculate_scale_factor_per_hkl_eq, + get_wavelength_binned, + get_reference_intensities, + estimate_scale_factor_per_hkl_asu_from_reference, ) """Providers for scaling data.""" diff --git a/packages/essnmx/tests/scaling_test.py b/packages/essnmx/tests/scaling_test.py index fb89fe67..1ce1dcb2 100644 --- a/packages/essnmx/tests/scaling_test.py +++ b/packages/essnmx/tests/scaling_test.py @@ -5,9 +5,9 @@ from ess.nmx.mtz_io import DEFAULT_WAVELENGTH_COLUMN_NAME from ess.nmx.scaling import ( - ReferenceWavelengthBin, - calculate_scale_factor_per_hkl_eq, - get_reference_bin, + ReferenceIntensities, + estimate_scale_factor_per_hkl_asu_from_reference, + get_reference_intensities, ) @@ -19,9 +19,9 @@ def nmx_data_array() -> sc.DataArray: DEFAULT_WAVELENGTH_COLUMN_NAME: sc.Variable( dims=["row"], values=[1, 2, 3, 4, 5, 3, 3] ), - "H_EQ": sc.array(dims=["row"], values=[1, 4, 7, 10, 13, 7, 9]), - "K_EQ": sc.array(dims=["row"], values=[2, 5, 8, 11, 14, 8, 8]), - "L_EQ": sc.array(dims=["row"], values=[3, 6, 9, 12, 15, 9, 7]), + "H_ASU": sc.array(dims=["row"], values=[1, 4, 7, 10, 13, 7, 9]), + "K_ASU": sc.array(dims=["row"], values=[2, 5, 8, 11, 14, 8, 8]), + "L_ASU": sc.array(dims=["row"], values=[3, 6, 9, 12, 15, 9, 7]), }, ) da.variances = ( @@ -33,7 +33,9 @@ def nmx_data_array() -> sc.DataArray: def test_get_reference_bin_middle(nmx_data_array: sc.DataArray) -> None: """Test the middle bin.""" - ref_bin = get_reference_bin(nmx_data_array.bin({DEFAULT_WAVELENGTH_COLUMN_NAME: 6})) + ref_bin = get_reference_intensities( + nmx_data_array.bin({DEFAULT_WAVELENGTH_COLUMN_NAME: 6}) + ) selected_idx = (2, 5, 6) assert all( ref_bin.data.values == [nmx_data_array.data.values[idx] for idx in selected_idx] @@ -41,19 +43,22 @@ def test_get_reference_bin_middle(nmx_data_array: sc.DataArray) -> None: @pytest.fixture -def reference_bin(nmx_data_array: sc.DataArray) -> ReferenceWavelengthBin: - return get_reference_bin(nmx_data_array.bin({DEFAULT_WAVELENGTH_COLUMN_NAME: 6})) +def reference_bin(nmx_data_array: sc.DataArray) -> ReferenceIntensities: + return get_reference_intensities( + nmx_data_array.bin({DEFAULT_WAVELENGTH_COLUMN_NAME: 6}) + ) -def test_reference_bin_scale_factor(reference_bin: ReferenceWavelengthBin) -> None: +def test_reference_bin_scale_factor(reference_bin: ReferenceIntensities) -> None: """Test the scale factor for I.""" - scale_factor = calculate_scale_factor_per_hkl_eq(reference_bin) + scale_factor = estimate_scale_factor_per_hkl_asu_from_reference(reference_bin) expected_groups = [(7, 8, 9), (9, 8, 7)] assert len(scale_factor) == len(expected_groups) + assert scale_factor.dim == "hkl_asu" for idx, group in enumerate(expected_groups): hkl = tuple( - scale_factor.coords[coord][idx].value for coord in ("H_EQ", "K_EQ", "L_EQ") + scale_factor.coords[coord][idx].value + for coord in (f"{idx}_ASU" for idx in "HKL") ) - print(hkl) assert hkl == group From 4e021225acd498092db2072f71219c817a7d443f Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 24 Apr 2024 14:43:14 +0200 Subject: [PATCH 154/403] Fix typo in docstrings. --- packages/essnmx/src/ess/nmx/mtz_io.py | 39 ++++++++++++++------------- packages/essnmx/tests/mtz_io_test.py | 4 +-- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index 4acda05a..7a8738ea 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -31,9 +31,9 @@ """The name of the intensity column in the mtz file.""" DEFAULT_INTENSITY_COLUMN_NAME = IntensityColumnName("I") -IntensitySigColumnName = NewType("IntensitySigColumnName", str) +StdDevColumnName = NewType("StdDevColumnName", str) """The name of the standard uncertainty of intensity column in the mtz file.""" -DEFAULT_INTENSITY_SIG_COLUMN_NAME = IntensitySigColumnName("SIGI") +DEFAULT_STD_DEV_COLUMN_NAME = StdDevColumnName("SIGI") # Computed types RawMtz = NewType("RawMtz", gemmi.Mtz) @@ -42,12 +42,12 @@ """The raw mtz dataframe.""" SpaceGroup = NewType("SpaceGroup", gemmi.SpaceGroup) """The space group.""" -RapioAsu = NewType("RapioAsu", gemmi.ReciprocalAsu) +ReciprocalAsymmetricUnit = NewType("ReciprocalAsymmetricUnit", gemmi.ReciprocalAsu) """The reciprocal asymmetric unit.""" MergedMtzDataFrame = NewType("MergedMtzDataFrame", pd.DataFrame) """The merged mtz dataframe with derived columns.""" NMXMtzDataFrame = NewType("NMXMtzDataFrame", pd.DataFrame) -"""The reduced mtz dataframe with derived columns.""" +"""The processed mtz dataframe with derived columns.""" NMXMtzDataArray = NewType("NMXMtzDataArray", sc.DataArray) @@ -86,7 +86,7 @@ def process_single_mtz_to_dataframe( mtz: RawMtz, wavelength_column_name: WavelengthColumnName = DEFAULT_WAVELENGTH_COLUMN_NAME, intensity_column_name: IntensityColumnName = DEFAULT_INTENSITY_COLUMN_NAME, - intensity_sig_col_name: IntensitySigColumnName = DEFAULT_INTENSITY_SIG_COLUMN_NAME, + intensity_sig_col_name: StdDevColumnName = DEFAULT_STD_DEV_COLUMN_NAME, ) -> RawMtzDataFrame: """Select and derive columns from the original ``MtzDataFrame``. @@ -101,7 +101,7 @@ def process_single_mtz_to_dataframe( intensity_column_name: The name of the intensity column in the mtz file. - sigma_intensity_column_name: + intensity_sig_col_name: The name of the standard uncertainty of intensity column in the mtz file. Returns @@ -120,7 +120,7 @@ def process_single_mtz_to_dataframe( - ``wavelength_column_name`` -> ``'wavelength'`` - ``intensity_column_name`` -> ``'I'`` - - ``sigma_intensity_column_name`` -> ``'SIGI'`` + - ``intensity_sig_col_name`` -> ``'SIGI'`` Other columns are kept as they are. @@ -148,7 +148,7 @@ def _calculate_d(row: pd.Series) -> float: mtz_df["resolution"] = (1 / mtz_df["d"]) ** 2 / 4 mtz_df[DEFAULT_WAVELENGTH_COORD_NAME] = orig_df[wavelength_column_name] mtz_df[DEFAULT_INTENSITY_COLUMN_NAME] = orig_df[intensity_column_name] - mtz_df[DEFAULT_INTENSITY_SIG_COLUMN_NAME] = orig_df[intensity_sig_col_name] + mtz_df[DEFAULT_STD_DEV_COLUMN_NAME] = orig_df[intensity_sig_col_name] # Keep other columns for column in [col for col in orig_df.columns if col not in mtz_df]: mtz_df[column] = orig_df[column] @@ -204,10 +204,10 @@ def get_space_group( ) -def get_reciprocal_asu(spacegroup: SpaceGroup) -> RapioAsu: +def get_reciprocal_asu(spacegroup: SpaceGroup) -> ReciprocalAsymmetricUnit: """Returns the reciprocal asymmetric unit from the space group.""" - return RapioAsu(gemmi.ReciprocalAsu(spacegroup)) + return ReciprocalAsymmetricUnit(gemmi.ReciprocalAsu(spacegroup)) def merge_mtz_dataframes( @@ -221,20 +221,21 @@ def merge_mtz_dataframes( def process_merged_mtz_dataframe( *, merged_mtz_df: MergedMtzDataFrame, - rapio_asu: RapioAsu, + reciprocal_asu: ReciprocalAsymmetricUnit, sg: SpaceGroup, ) -> NMXMtzDataFrame: - """Reduces the shallow copy of a merged mtz dataframes. + """Modify/Add columns of the shallow copy of a merged mtz dataframes. This method must be called after merging multiple mtz dataframes. """ merged_df = merged_mtz_df.copy(deep=False) - def _rapio_asu_to_asu(row: pd.Series) -> list[int]: + def _reciprocal_asu(row: pd.Series) -> list[int]: """Converts miller indices(HKL) to ASU indices.""" - return rapio_asu.to_asu(row["hkl"], sg.operations())[0] - merged_df["hkl_asu"] = merged_df.apply(_rapio_asu_to_asu, axis=1) + return reciprocal_asu.to_asu(row["hkl"], sg.operations())[0] + + merged_df["hkl_asu"] = merged_df.apply(_reciprocal_asu, axis=1) # Unpack the indices for later. merged_df[["H_ASU", "K_ASU", "L_ASU"]] = pd.DataFrame( merged_df["hkl_asu"].to_list(), index=merged_df.index @@ -290,7 +291,7 @@ def _join_variables(*vars: sc.Variable, splitter: str = " ") -> sc.Variable: def nmx_mtz_dataframe_to_scipp_dataarray( nmx_mtz_df: NMXMtzDataFrame, ) -> NMXMtzDataArray: - """Converts the reduced mtz dataframe to a scipp dataarray. + """Converts the processed mtz dataframe to a scipp dataarray. The intensity, with column name :attr:`~DEFAULT_INTENSITY_COLUMN_NAME` becomes the data and the standard uncertainty of intensity, @@ -331,7 +332,7 @@ def nmx_mtz_dataframe_to_scipp_dataarray( to_scipp, data_columns=[ DEFAULT_INTENSITY_COLUMN_NAME, - DEFAULT_INTENSITY_SIG_COLUMN_NAME, + DEFAULT_STD_DEV_COLUMN_NAME, ], header_parser=parse_bracket_header, ) @@ -353,7 +354,7 @@ def nmx_mtz_dataframe_to_scipp_dataarray( # Add variances nmx_mtz_da = nmx_mtz_ds[DEFAULT_INTENSITY_COLUMN_NAME].copy(deep=False) - nmx_mtz_da.variances = nmx_mtz_ds[DEFAULT_INTENSITY_SIG_COLUMN_NAME].data ** 2 + nmx_mtz_da.variances = nmx_mtz_ds[DEFAULT_STD_DEV_COLUMN_NAME].data ** 2 # Return DataArray return NMXMtzDataArray(nmx_mtz_da) @@ -373,6 +374,6 @@ def nmx_mtz_dataframe_to_scipp_dataarray( mtz_io_params = { WavelengthColumnName: DEFAULT_WAVELENGTH_COLUMN_NAME, IntensityColumnName: DEFAULT_INTENSITY_COLUMN_NAME, - IntensitySigColumnName: DEFAULT_INTENSITY_SIG_COLUMN_NAME, + StdDevColumnName: DEFAULT_STD_DEV_COLUMN_NAME, } """The parameters related to the MTZ IO.""" diff --git a/packages/essnmx/tests/mtz_io_test.py b/packages/essnmx/tests/mtz_io_test.py index 5c8f6057..963a9d5d 100644 --- a/packages/essnmx/tests/mtz_io_test.py +++ b/packages/essnmx/tests/mtz_io_test.py @@ -142,11 +142,11 @@ def test_reduce_merged_mtz_dataframe( merged_mtz_dataframe: MergedMtzDataFrame, ) -> None: space_gr = get_space_group(mtz_series) - rapio_asu = get_reciprocal_asu(space_gr) + reciprocal_asu = get_reciprocal_asu(space_gr) nmx_df = process_merged_mtz_dataframe( merged_mtz_df=merged_mtz_dataframe, - rapio_asu=rapio_asu, + reciprocal_asu=reciprocal_asu, sg=space_gr, ) assert "hkl_asu" not in merged_mtz_dataframe.columns From a4e9cf7dafd2e50de23b3d62a81286e39ba5c989 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 24 Apr 2024 14:49:08 +0200 Subject: [PATCH 155/403] Update docs with new user-defined parameter. --- .../docs/examples/scaling_workflow.ipynb | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index ad5e53d5..7f9f69bc 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -53,16 +53,19 @@ "outputs": [], "source": [ "import sciline as sl\n", + "import scipp as sc\n", + "\n", "from ess.nmx.mtz_io import mtz_io_providers, mtz_io_params\n", "from ess.nmx.mtz_io import MTZFileIndex, SpaceGroupDesc\n", "from ess.nmx.scaling import scaling_providers\n", - "from ess.nmx.scaling import WavelengthBinSize, WavelengthBinned\n", + "from ess.nmx.scaling import WavelengthBinSize, WavelengthBinned, ReferenceWavelength\n", "\n", "pl = sl.Pipeline(\n", " providers=mtz_io_providers+scaling_providers,\n", " params={\n", " SpaceGroupDesc: \"C 1 2 1\", # Replace with the correct space group if needed\n", " WavelengthBinSize: 50, # Replace with the correct bin size if needed\n", + " ReferenceWavelength: sc.scalar(3, unit=sc.units.angstrom), # Remove it if you want to use the middle of the bin\n", " **mtz_io_params\n", " },\n", ")\n", @@ -117,20 +120,7 @@ "source": [ "from ess.nmx.scaling import get_reference_intensities, estimate_scale_factor_per_hkl_asu_from_reference\n", "\n", - "scale_factor = estimate_scale_factor_per_hkl_asu_from_reference(get_reference_intensities(binned))\n", - "scale_factor" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import scipp as sc\n", - "from ess.nmx.scaling import get_reference_intensities, ReferenceWavelength\n", - "\n", - "get_reference_intensities(binned, ReferenceWavelength(sc.scalar(3, unit=sc.units.angstrom)))" + "estimate_scale_factor_per_hkl_asu_from_reference(get_reference_intensities(binned))" ] } ], From e99b505c6be31911d5bcea97a7568011abe912dd Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 24 Apr 2024 15:22:56 +0200 Subject: [PATCH 156/403] Convert indices column directly to string. --- packages/essnmx/src/ess/nmx/mtz_io.py | 57 ++++----------------------- packages/essnmx/tests/mtz_io_test.py | 11 ------ 2 files changed, 8 insertions(+), 60 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index 7a8738ea..a05e28c2 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -244,50 +244,6 @@ def _reciprocal_asu(row: pd.Series) -> list[int]: return NMXMtzDataFrame(merged_df) -def _join_variables(*vars: sc.Variable, splitter: str = " ") -> sc.Variable: - """Joins multiple integer dtype variables into a single string dtype variable. - - Parameters - ---------- - vars: - The integer dtype variables to join with same dimensions and length. - - splitter: - The string to join the variables. - - Returns - ------- - : - The joined variable. It keeps the dimensions of the input variables. - But it drops the units since the output is a string. - - Raises - ------ - ValueError - If the input variables have different dimensions or lengths. - - """ - # Check if all variables are integer - if not all(var.dtype == int for var in vars): - raise ValueError("All variables must be integer type.") - # Check if all variables have the same dimensions - dims = set(var.dim for var in vars) - if len(dims) != 1: - raise ValueError("All variables must have the same dimensions.") - # Check if all variables have the same length - lengths = set(len(var.values) for var in vars) - if len(lengths) != 1: - raise ValueError("All variables must have the same length.") - - return sc.array( - dims=dims, - values=[ - splitter.join(str(val) for val in row) - for row in zip(*(var.values for var in vars)) - ], - ) - - def nmx_mtz_dataframe_to_scipp_dataarray( nmx_mtz_df: NMXMtzDataFrame, ) -> NMXMtzDataArray: @@ -339,14 +295,17 @@ def nmx_mtz_dataframe_to_scipp_dataarray( # Pop the indices columns. # TODO: We can put them back once we support tuple[int] dtype. # See https://github.com/scipp/scipp/issues/3046 for more details. - # Temporarily, we will join them into a single string. + # Temporarily, we will manually convert them to a string. # It is done on the scipp variable instead of the dataframe # since columns with string dtype are converted to PyObject dtype # instead of string by `from_pandas_dataframe`. - nmx_mtz_ds = nmx_mtz_ds.drop_coords(["hkl", "hkl_asu"]) - nmx_mtz_ds.coords["hkl_asu"] = _join_variables( - *(nmx_mtz_ds.coords[f"{idx_desc}_ASU"] for idx_desc in "HKL") - ) + for indices_name in ("hkl", "hkl_asu"): + nmx_mtz_ds.coords[indices_name] = sc.array( + dims=nmx_mtz_ds.coords[indices_name].dims, + values=nmx_mtz_df[indices_name].astype(str).tolist() + # `astype`` is not enough to convert the dtype to string. + # The result of `astype` will have `PyObject` as a dtype. + ) # Add units nmx_mtz_ds.coords[DEFAULT_WAVELENGTH_COORD_NAME].unit = sc.units.angstrom for key in nmx_mtz_ds.keys(): diff --git a/packages/essnmx/tests/mtz_io_test.py b/packages/essnmx/tests/mtz_io_test.py index 963a9d5d..bf623507 100644 --- a/packages/essnmx/tests/mtz_io_test.py +++ b/packages/essnmx/tests/mtz_io_test.py @@ -5,7 +5,6 @@ import gemmi import pytest import sciline as sl -import scipp as sc from ess.nmx.data import get_small_mtz_samples from ess.nmx.mtz_io import DEFAULT_SPACE_GROUP_DESC # P 21 21 21 @@ -14,7 +13,6 @@ MTZFileIndex, MTZFilePath, RawMtz, - _join_variables, get_reciprocal_asu, get_space_group, merge_mtz_dataframes, @@ -151,12 +149,3 @@ def test_reduce_merged_mtz_dataframe( ) assert "hkl_asu" not in merged_mtz_dataframe.columns assert "hkl_asu" in nmx_df.columns - - -def test_join_variables() -> None: - var_x = sc.array(dims=["xy"], values=[1, 1, 2, 3, 3], unit=None) - var_y = sc.array(dims=["xy"], values=[0, 1, 2, 0, 3], unit=None) - var_xy = sc.array(dims=["xy"], values=["1 0", "1 1", "2 2", "3 0", "3 3"]) - - joined = _join_variables(var_x, var_y, splitter=" ") - assert sc.identical(joined, var_xy) From 44cac2786444154232e605d96d84fdd7a0ca3f6c Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 24 Apr 2024 15:31:13 +0200 Subject: [PATCH 157/403] Update grouping. --- packages/essnmx/src/ess/nmx/scaling.py | 14 ++++++-------- packages/essnmx/tests/scaling_test.py | 23 ++++++++++++++--------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 6ad63004..251c92ef 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -137,15 +137,13 @@ def estimate_scale_factor_per_hkl_asu_from_reference( """ # Workaround for https://github.com/scipp/scipp/issues/3046 # and https://github.com/scipp/scipp/issues/3425 - # This workaround is implemented with an assumption that - # the size of all combinations of (H_ASU, K_ASU, L_ASU) is small enough - # to be handled in memory. - grouped = reference_intensities.group("H_ASU", "K_ASU", "L_ASU").flatten( - dims=["H_ASU", "K_ASU", "L_ASU"], to="hkl_asu" - ) - non_empty = grouped[grouped.bins.size().data > sc.scalar(0, unit=None)] + import numpy as np + + unique_hkl = np.unique(reference_intensities.coords["hkl_asu"].values) + group_var = sc.array(dims=["hkl_asu"], values=unique_hkl) + grouped = reference_intensities.group(group_var) - return EstimatedScaleFactor((1 / non_empty).bins.mean()) + return EstimatedScaleFactor((1 / grouped).bins.mean()) # Providers and default parameters diff --git a/packages/essnmx/tests/scaling_test.py b/packages/essnmx/tests/scaling_test.py index 1ce1dcb2..2228dffb 100644 --- a/packages/essnmx/tests/scaling_test.py +++ b/packages/essnmx/tests/scaling_test.py @@ -19,9 +19,18 @@ def nmx_data_array() -> sc.DataArray: DEFAULT_WAVELENGTH_COLUMN_NAME: sc.Variable( dims=["row"], values=[1, 2, 3, 4, 5, 3, 3] ), - "H_ASU": sc.array(dims=["row"], values=[1, 4, 7, 10, 13, 7, 9]), - "K_ASU": sc.array(dims=["row"], values=[2, 5, 8, 11, 14, 8, 8]), - "L_ASU": sc.array(dims=["row"], values=[3, 6, 9, 12, 15, 9, 7]), + "hkl_asu": sc.array( + dims=["row"], + values=[ + "[1, 2, 3]", + "[4, 5, 6]", + "[7, 8, 9]", + "[10, 11, 12]", + "[13, 14, 15]", + "[7, 8, 9]", + "[9, 8, 7]", + ], + ), }, ) da.variances = ( @@ -52,13 +61,9 @@ def reference_bin(nmx_data_array: sc.DataArray) -> ReferenceIntensities: def test_reference_bin_scale_factor(reference_bin: ReferenceIntensities) -> None: """Test the scale factor for I.""" scale_factor = estimate_scale_factor_per_hkl_asu_from_reference(reference_bin) - expected_groups = [(7, 8, 9), (9, 8, 7)] + expected_groups = [[7, 8, 9], [9, 8, 7]] assert len(scale_factor) == len(expected_groups) assert scale_factor.dim == "hkl_asu" for idx, group in enumerate(expected_groups): - hkl = tuple( - scale_factor.coords[coord][idx].value - for coord in (f"{idx}_ASU" for idx in "HKL") - ) - assert hkl == group + assert scale_factor.coords['hkl_asu'][idx].value == str(group) From 0b22e174e7dc770199869d8bb9e36991505634ae Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 24 Apr 2024 17:09:54 +0200 Subject: [PATCH 158/403] Update docstring and add tests. --- packages/essnmx/src/ess/nmx/mtz_io.py | 23 +++++++++-------- packages/essnmx/tests/mtz_io_test.py | 36 +++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index a05e28c2..7e41d458 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -267,17 +267,20 @@ def nmx_mtz_dataframe_to_scipp_dataarray( squared ``SIGI`` column becomes the variances. Therefore they are not in the coordinates. - Following coordinates are dropped from the dataframe: - - - ``hkl``: The miller indices as a list of integers. - There is no dtype that can represent this in scipp. - Following coordinates are modified: - - ``hkl_asu``: The miller indices as a string. + - ``hkl``: The miller indices as a string. + It is modified to have a string dtype + since is no dtype that can represent this in scipp. + + - ``hkl_asu``: The asymmetric unit of miller indices as a string. This coordinate will be used to derive estimated scale factors. It is modified to have a string dtype - as the same reason as why ``hkl`` coordinate is dropped. + as the same reason as why ``hkl`` coordinate is modified. + + Zero or negative intensities are removed from the dataarray. + It can happen due to the post-processing of the data, + e.g. background subtraction. """ from scipp.compat.pandas_compat import from_pandas_dataframe, parse_bracket_header @@ -313,10 +316,10 @@ def nmx_mtz_dataframe_to_scipp_dataarray( # Add variances nmx_mtz_da = nmx_mtz_ds[DEFAULT_INTENSITY_COLUMN_NAME].copy(deep=False) - nmx_mtz_da.variances = nmx_mtz_ds[DEFAULT_STD_DEV_COLUMN_NAME].data ** 2 + nmx_mtz_da.variances = (nmx_mtz_ds[DEFAULT_STD_DEV_COLUMN_NAME].data ** 2).values - # Return DataArray - return NMXMtzDataArray(nmx_mtz_da) + # Return DataArray without negative intensities + return NMXMtzDataArray(nmx_mtz_da[nmx_mtz_da.data > 0]) mtz_io_providers = ( diff --git a/packages/essnmx/tests/mtz_io_test.py b/packages/essnmx/tests/mtz_io_test.py index bf623507..7c0487d3 100644 --- a/packages/essnmx/tests/mtz_io_test.py +++ b/packages/essnmx/tests/mtz_io_test.py @@ -5,6 +5,7 @@ import gemmi import pytest import sciline as sl +import scipp as sc from ess.nmx.data import get_small_mtz_samples from ess.nmx.mtz_io import DEFAULT_SPACE_GROUP_DESC # P 21 21 21 @@ -12,11 +13,14 @@ MergedMtzDataFrame, MTZFileIndex, MTZFilePath, + NMXMtzDataArray, + NMXMtzDataFrame, RawMtz, get_reciprocal_asu, get_space_group, merge_mtz_dataframes, mtz_to_pandas, + nmx_mtz_dataframe_to_scipp_dataarray, process_merged_mtz_dataframe, process_single_mtz_to_dataframe, read_mtz_file, @@ -52,7 +56,7 @@ def test_mtz_to_pandas_dataframe(gemmi_mtz_object: gemmi.Mtz) -> None: assert all(df[column.label] == column.array) -def test_mtz_to_reduced_pandas_dataframe(gemmi_mtz_object: gemmi.Mtz) -> None: +def test_mtz_to_process_pandas_dataframe(gemmi_mtz_object: gemmi.Mtz) -> None: df = process_single_mtz_to_dataframe(RawMtz(gemmi_mtz_object)) for expected_colum in ["hkl", "d", "resolution", *"HKL", "wavelength", "I", "SIGI"]: assert expected_colum in df.columns @@ -135,17 +139,39 @@ def merged_mtz_dataframe( return merge_mtz_dataframes(reduced_mtz_series) -def test_reduce_merged_mtz_dataframe( +@pytest.fixture +def nmx_data_frame( mtz_series: sl.Series[MTZFileIndex, RawMtz], merged_mtz_dataframe: MergedMtzDataFrame, -) -> None: +) -> NMXMtzDataFrame: space_gr = get_space_group(mtz_series) reciprocal_asu = get_reciprocal_asu(space_gr) - nmx_df = process_merged_mtz_dataframe( + return process_merged_mtz_dataframe( merged_mtz_df=merged_mtz_dataframe, reciprocal_asu=reciprocal_asu, sg=space_gr, ) + + +def test_process_merged_mtz_dataframe( + merged_mtz_dataframe: MergedMtzDataFrame, + nmx_data_frame: NMXMtzDataFrame, +) -> None: assert "hkl_asu" not in merged_mtz_dataframe.columns - assert "hkl_asu" in nmx_df.columns + assert "hkl_asu" in nmx_data_frame.columns + + +@pytest.fixture +def nmx_data_array(nmx_data_frame: NMXMtzDataFrame) -> NMXMtzDataArray: + return nmx_mtz_dataframe_to_scipp_dataarray(nmx_data_frame) + + +def test_to_scipp_dataarray( + nmx_data_array: NMXMtzDataArray, +) -> None: + # Check the intended modification + for indices_coord_name in ("hkl", "hkl_asu"): + assert nmx_data_array.coords[indices_coord_name].dtype == str + + assert sc.all(nmx_data_array.data > 0) From 8971af88f82b83d3551d4b7fb59261e73e13ebf1 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 24 Apr 2024 16:09:49 +0200 Subject: [PATCH 159/403] Add average estimated scaled intensities method. --- packages/essnmx/src/ess/nmx/scaling.py | 79 +++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 251c92ef..33f923d2 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -92,7 +92,7 @@ def get_reference_intensities( if reference_wavelength.dims: raise ValueError("Reference wavelength should be a scalar.") try: - return binned['wavelength', reference_wavelength].values.copy(deep=False) + return binned["wavelength", reference_wavelength].values.copy(deep=False) except IndexError: raise IndexError(f"{reference_wavelength} out of range.") @@ -118,6 +118,9 @@ def estimate_scale_factor_per_hkl_asu_from_reference( = average( \\dfrac{1}{I_{(hkl)}} ) Estimated scale factor is calculated per ``hkl_asu``. + This is part of the calculation of roughly-scaled-intensities + for fitting the scaling model. + The whole procedure is described in :func:`average_roughly_scaled_intensities`. Parameters ---------- @@ -146,10 +149,84 @@ def estimate_scale_factor_per_hkl_asu_from_reference( return EstimatedScaleFactor((1 / grouped).bins.mean()) +def average_roughly_scaled_intensities( + binned: WavelengthBinned, + scale_factor: EstimatedScaleFactor, +) -> EstimatedScaledIntensities: + """Scale the intensities by the estimated scale factor. + + Parameters + ---------- + binned: + Binned data by wavelength(LAMBDA) to be grouped and scaled. + + scale_factor: + The estimated scale factor. + + Returns + ------- + : + Average scaled intensties on ``hkl(asu)`` indices per wavelength. + + Notes + ----- + The average of roughly scaled intensities are calculated by the following formula: + + .. math:: + + EstimatedScaledI_{\\lambda} + = \\dfrac{ + \\sum_{k=1}^{N_{\\lambda, (hkl)}} EstimatedScaledI_{\\lambda, (hkl)} + }{ + N_{\\lambda, (hkl)} + } + + And scaled intensities on each ``hkl(asu)`` indices per wavelength + are calculated by the following formula: + + .. math:: + :nowrap: + + \\begin{eqnarray} + EstimatedScaledI_{\\lambda, (hkl)} \\\\ + = \\dfrac{ + \\sum_{i=1}^{N_{reference, (hkl)}} + \\sum_{j=1}^{N_{\\lambda, (hkl)}} + \\dfrac{I_{j}}{I_{i}} + }{ + N_{reference, (hkl)}*N_{\\lambda, (hkl)} + } \\\\ + = \\dfrac{ + \\sum_{i=1}^{N_{reference, (hkl)}} \\dfrac{1}{I_{i}} + }{ + N_{reference, (hkl)} + } * \\dfrac{ + \\sum_{j=1}^{N_{\\lambda, (hkl)}} I_{j} + }{ + N_{\\lambda, (hkl)} + } \\\\ + = average( \\dfrac{1}{I_{ref, (hkl)}} ) * average( I_{\\lambda, (hkl)} ) + \\end{eqnarray} + + Therefore the ``binned(wavelength dimension)`` should be + grouped along the ``hkl(asu)`` coordinate in the calculation. + + """ + # Group by HKL_EQ of the estimated scale factor from reference intensities + grouped = binned.group(scale_factor.coords['hkl_asu']) + + # Drop variances of the scale factor + # Scale each group each bin by the scale factor + return EstimatedScaledIntensities( + sc.mean(grouped.bins.nanmean() * sc.values(scale_factor), dim="HKL_EQ") + ) + + # Providers and default parameters scaling_providers = ( get_wavelength_binned, get_reference_intensities, estimate_scale_factor_per_hkl_asu_from_reference, + average_roughly_scaled_intensities, ) """Providers for scaling data.""" From cf13675f411e5e7fba0cc474fe80a1e93ab3006e Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 25 Apr 2024 13:43:39 +0200 Subject: [PATCH 160/403] Update docstring of filter/cut functions. --- .../docs/examples/scaling_workflow.ipynb | 43 ++-- packages/essnmx/src/ess/nmx/scaling.py | 188 +++++++++++++++++- 2 files changed, 201 insertions(+), 30 deletions(-) diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index 7f9f69bc..f261a4c3 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -57,16 +57,29 @@ "\n", "from ess.nmx.mtz_io import mtz_io_providers, mtz_io_params\n", "from ess.nmx.mtz_io import MTZFileIndex, SpaceGroupDesc\n", - "from ess.nmx.scaling import scaling_providers\n", - "from ess.nmx.scaling import WavelengthBinSize, WavelengthBinned, ReferenceWavelength\n", + "from ess.nmx.scaling import scaling_providers, scaling_params\n", + "from ess.nmx.scaling import (\n", + " WavelengthBinSize,\n", + " FilteredEstimatedScaledIntensities,\n", + " ReferenceWavelength,\n", + " WavelengthBinCutProportion,\n", + " NRoot,\n", + " NRootStdDevCut,\n", + ")\n", "\n", "pl = sl.Pipeline(\n", - " providers=mtz_io_providers+scaling_providers,\n", + " providers=mtz_io_providers + scaling_providers,\n", " params={\n", - " SpaceGroupDesc: \"C 1 2 1\", # Replace with the correct space group if needed\n", - " WavelengthBinSize: 50, # Replace with the correct bin size if needed\n", - " ReferenceWavelength: sc.scalar(3, unit=sc.units.angstrom), # Remove it if you want to use the middle of the bin\n", - " **mtz_io_params\n", + " SpaceGroupDesc: \"C 1 2 1\",\n", + " WavelengthBinSize: 500,\n", + " ReferenceWavelength: sc.scalar(\n", + " 3, unit=sc.units.angstrom\n", + " ), # Remove it if you want to use the middle of the bin\n", + " WavelengthBinCutProportion: 0.25, # 0 < proportion < 0.5\n", + " NRoot: 4, # Increase this value to effectively remove more outliers on the right tail\n", + " NRootStdDevCut: 1.0, # Lower this value to remove more outliers\n", + " **mtz_io_params,\n", + " **scaling_params,\n", " },\n", ")\n", "\n", @@ -91,7 +104,7 @@ "metadata": {}, "outputs": [], "source": [ - "scaling_nmx_workflow = pl.get(WavelengthBinned)\n", + "scaling_nmx_workflow = pl.get(FilteredEstimatedScaledIntensities)\n", "scaling_nmx_workflow.visualize(graph_attr={\"rankdir\": \"LR\"})" ] }, @@ -108,19 +121,7 @@ "metadata": {}, "outputs": [], "source": [ - "binned = scaling_nmx_workflow.compute(WavelengthBinned)\n", - "binned" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from ess.nmx.scaling import get_reference_intensities, estimate_scale_factor_per_hkl_asu_from_reference\n", - "\n", - "estimate_scale_factor_per_hkl_asu_from_reference(get_reference_intensities(binned))" + "scaling_nmx_workflow.compute(FilteredEstimatedScaledIntensities)" ] } ], diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 33f923d2..35cd972b 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -9,23 +9,40 @@ # User defined or configurable types WavelengthBinSize = NewType("WavelengthBinSize", int) """The size of the wavelength(LAMBDA) bins.""" +WavelengthRange = NewType("WavelengthRange", tuple[float, float]) +"""The range of the wavelength(LAMBDA) bins.""" +WavelengthBinCutProportion = NewType("WavelengthBinCutProportion", float) +"""The proportion of the wavelength(LAMBDA) bins to be cut off on both sides.""" +DEFAULT_WAVELENGTH_CUT_PROPORTION = WavelengthBinCutProportion(0.25) +"""Default proportion of the wavelength(LAMBDA) bins to be cut off from both sides.""" ReferenceWavelength = NewType("ReferenceWavelength", sc.Variable) """The wavelength to select reference intensities.""" +NRoot = NewType("NRoot", int) +"""The n-th root to be taken for the standard deviation.""" +NRootStdDevCut = NewType("NRootStdDevCut", float) +"""The number of standard deviations to be cut from the n-th root data.""" # Computed types +"""Filtered mtz dataframe by the quad root of the sample standard deviation.""" WavelengthBinned = NewType("WavelengthBinned", sc.DataArray) """Binned mtz dataframe by wavelength(LAMBDA) with derived columns.""" +FilteredWavelengthBinned = NewType("FilteredWavelengthBinned", sc.DataArray) +"""Filtered binned data.""" ReferenceIntensities = NewType("ReferenceIntensities", sc.DataArray) """Reference intensities selected by the wavelength.""" EstimatedScaleFactor = NewType("EstimatedScaleFactor", sc.DataArray) """The estimated scale factor from the reference intensities per ``hkl_asu``.""" -EstimatedScaledIntensities = NewType("EstimatedScaledIntensities", float) +EstimatedScaledIntensities = NewType("EstimatedScaledIntensities", sc.DataArray) """Scaled intensities by the estimated scale factor.""" +FilteredEstimatedScaledIntensities = NewType( + "FilteredEstimatedScaledIntensities", sc.DataArray +) def get_wavelength_binned( mtz_da: NMXMtzDataArray, wavelength_bin_size: WavelengthBinSize, + wavelength_range: Optional[WavelengthRange] = None, ) -> WavelengthBinned: """Bin the whole dataset by wavelength(LAMBDA). @@ -34,10 +51,51 @@ def get_wavelength_binned( Wavelength(LAMBDA) binning should always be done on the merged dataset. """ + if wavelength_range is None: + binning_var = wavelength_bin_size + else: + binning_var = sc.linspace( + dim=DEFAULT_WAVELENGTH_COORD_NAME, + start=wavelength_range[0], + stop=wavelength_range[1], + num=wavelength_bin_size, + unit=mtz_da.coords[DEFAULT_WAVELENGTH_COORD_NAME].unit, + ) - return WavelengthBinned( - mtz_da.bin({DEFAULT_WAVELENGTH_COORD_NAME: wavelength_bin_size}) - ) + binned = mtz_da.bin({DEFAULT_WAVELENGTH_COORD_NAME: binning_var}) + + return WavelengthBinned(binned) + + +def filter_wavelegnth_binned( + binned: WavelengthBinned, + cut_proportion: WavelengthBinCutProportion = DEFAULT_WAVELENGTH_CUT_PROPORTION, +) -> FilteredWavelengthBinned: + """Filter the binned data by cutting off the edges. + + Parameters + ---------- + binned: + The binned data by wavelength(LAMBDA). + + cut_proportion: + The proportion of the wavelength(LAMBDA) bins to be cut off on both sides. + The default value is :attr:`~DEFAULT_WAVELENGTH_CUT_PROPORTION`. + + Returns + ------- + : + The filtered binned data. + + """ + + if cut_proportion < 0 or cut_proportion >= 0.5: + raise ValueError( + "The cut proportion should be in the range of 0 < proportion < 0.5." + ) + + cut_size = int(len(binned) * cut_proportion) + return FilteredWavelengthBinned(binned[cut_size:-cut_size]) def _is_bin_empty(binned: sc.DataArray, idx: int) -> bool: @@ -45,7 +103,7 @@ def _is_bin_empty(binned: sc.DataArray, idx: int) -> bool: return binned[idx].values.size == 0 -def _get_middle_bin_idx(binned: WavelengthBinned) -> int: +def _get_middle_bin_idx(binned: sc.DataArray) -> int: """Find the middle bin index. If the middle one is empty, the function will search for the nearest. @@ -64,7 +122,7 @@ def _get_middle_bin_idx(binned: WavelengthBinned) -> int: def get_reference_intensities( - binned: WavelengthBinned, + binned: FilteredWavelengthBinned, reference_wavelength: Optional[ReferenceWavelength] = None, ) -> ReferenceIntensities: """Find the reference intensities by the wavelength. @@ -150,7 +208,7 @@ def estimate_scale_factor_per_hkl_asu_from_reference( def average_roughly_scaled_intensities( - binned: WavelengthBinned, + binned: FilteredWavelengthBinned, scale_factor: EstimatedScaleFactor, ) -> EstimatedScaledIntensities: """Scale the intensities by the estimated scale factor. @@ -213,20 +271,132 @@ def average_roughly_scaled_intensities( """ # Group by HKL_EQ of the estimated scale factor from reference intensities - grouped = binned.group(scale_factor.coords['hkl_asu']) + grouped = binned.group(scale_factor.coords["hkl_asu"]) # Drop variances of the scale factor # Scale each group each bin by the scale factor return EstimatedScaledIntensities( - sc.mean(grouped.bins.nanmean() * sc.values(scale_factor), dim="HKL_EQ") + sc.nanmean(grouped.bins.nanmean() * sc.values(scale_factor), dim="hkl_asu") + ) + + +def _calculate_sample_standard_deviation(var: sc.Variable) -> sc.Variable: + """Calculate the sample variation of the data. + + This helper function is a temporary solution before + we release new scipp version with the statistics helper. + """ + import numpy as np + + return sc.scalar(np.nanstd(var.values)) + + +def cut_estimated_scaled_intensities_by_n_root_std_dev( + scaled_intensities: EstimatedScaledIntensities, + n_root: NRoot, + n_root_std_dev_cut: NRootStdDevCut, +) -> FilteredEstimatedScaledIntensities: + """Filter the mtz data array by the quad root of the sample standard deviation. + + Parameters + ---------- + scaled_intensities: + The scaled intensities to be filtered. + + n_root: + The n-th root to be taken for the standard deviation. + Higher n-th root means cutting is more effective on the right tail. + More explanation can be found in the notes. + + n_root_std_dev_cut: + The number of standard deviations to be cut from the n-th root data. + + Returns + ------- + : + The filtered scaled intensities. + + Notes + ----- + *Reason for taking the n-th root of the intensities:* + The scaled intensities are expected to follow gaussian distribution + with the mean of 1.0 (since it is scaled by the reference intensities). + However, the data will have a long tail on the right side ( > 1.0). + Since the negative intensities are already filtered out, + the left tail should be preserved and the right tail should be cut off. + + Therefore, the data is transformed by the n-th root to make the data + more symmetric and the standard deviation cut is applied to filter out. + + Let :math:`m` be the minimum value of the data and :math:`M` be the maximum value. + The size of the left tail from the mean is :math:`1 - m, (1 > m > 0)` + and the size of the right tail is :math:`M - 1 ( M > 1)`. + + As we take the n-th root of the data, + the left tail will be more stretched as + + .. math:: + + 1 - m^{1/n} > 1 - m \\\\ + \\because m < m^{1/n} text{where } 0 < m < 1 \\text{ and } n > 1 + + and the right tail will be more compressed as + + .. math:: + + M^{1/n} - 1 < M - 1 \\\\ + \\because M > M^{1/n} \\text{where } M > 1 \\text{ and } n > 1 + + Comparing how much the tails are stretched or compressed, + + .. math:: + + stretched = 1 - m^{1/n} - (1 - m) = m - m^{1/n} \\\\ + compressed = M - 1 - (M^{1/n} - 1) = M^{1/n} - M + + However, we are assuming the :math:`M >> 1` and :math:`m ~ 0` + in the scaled intensities. + The right tail will be more compressed than the left tail is stretched. + In this way the right tail will be cut off more effectively on the n-th root. + + """ + # Check the range of the n-th root + if n_root < 1: + raise ValueError("The n-th root should be equal to or greater than 1.") + + copied = scaled_intensities.copy(deep=False) + # Take the midpoints of the wavelength bin coordinates + # to represent the average wavelength of the bin + # It is because the bin-edges are dropped while flattening the data + copied.coords[DEFAULT_WAVELENGTH_COORD_NAME] = sc.midpoints( + copied.coords[DEFAULT_WAVELENGTH_COORD_NAME], + ) + nth_root = copied.data ** (1 / n_root) + # Calculate the mean + nth_root_mean = nth_root.mean() + # Calculate the sample standard deviation + nth_root_std_dev = _calculate_sample_standard_deviation(nth_root) + # Calculate the cut value + half_window = n_root_std_dev_cut * nth_root_std_dev + keep_range = (nth_root_mean - half_window, nth_root_mean + half_window) + + # Filter the data + return FilteredEstimatedScaledIntensities( + copied[(nth_root > keep_range[0]) & (nth_root < keep_range[1])] ) # Providers and default parameters scaling_providers = ( + cut_estimated_scaled_intensities_by_n_root_std_dev, get_wavelength_binned, + filter_wavelegnth_binned, get_reference_intensities, estimate_scale_factor_per_hkl_asu_from_reference, average_roughly_scaled_intensities, ) """Providers for scaling data.""" + +scaling_params = { + WavelengthBinCutProportion: DEFAULT_WAVELENGTH_CUT_PROPORTION, +} From 8ec9cd73f0439d5b7d0d790da6da3a6dada5275a Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 2 May 2024 11:38:18 +0200 Subject: [PATCH 161/403] Update docstring. --- packages/essnmx/src/ess/nmx/scaling.py | 43 -------------------------- 1 file changed, 43 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 35cd972b..18d46c6e 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -316,49 +316,6 @@ def cut_estimated_scaled_intensities_by_n_root_std_dev( : The filtered scaled intensities. - Notes - ----- - *Reason for taking the n-th root of the intensities:* - The scaled intensities are expected to follow gaussian distribution - with the mean of 1.0 (since it is scaled by the reference intensities). - However, the data will have a long tail on the right side ( > 1.0). - Since the negative intensities are already filtered out, - the left tail should be preserved and the right tail should be cut off. - - Therefore, the data is transformed by the n-th root to make the data - more symmetric and the standard deviation cut is applied to filter out. - - Let :math:`m` be the minimum value of the data and :math:`M` be the maximum value. - The size of the left tail from the mean is :math:`1 - m, (1 > m > 0)` - and the size of the right tail is :math:`M - 1 ( M > 1)`. - - As we take the n-th root of the data, - the left tail will be more stretched as - - .. math:: - - 1 - m^{1/n} > 1 - m \\\\ - \\because m < m^{1/n} text{where } 0 < m < 1 \\text{ and } n > 1 - - and the right tail will be more compressed as - - .. math:: - - M^{1/n} - 1 < M - 1 \\\\ - \\because M > M^{1/n} \\text{where } M > 1 \\text{ and } n > 1 - - Comparing how much the tails are stretched or compressed, - - .. math:: - - stretched = 1 - m^{1/n} - (1 - m) = m - m^{1/n} \\\\ - compressed = M - 1 - (M^{1/n} - 1) = M^{1/n} - M - - However, we are assuming the :math:`M >> 1` and :math:`m ~ 0` - in the scaled intensities. - The right tail will be more compressed than the left tail is stretched. - In this way the right tail will be cut off more effectively on the n-th root. - """ # Check the range of the n-th root if n_root < 1: From 675034c78aeb479fa4f30f250e5bb8015a1ef78f Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 2 May 2024 14:12:31 +0200 Subject: [PATCH 162/403] Insert reference wavelength selection step to use it for later. --- packages/essnmx/src/ess/nmx/scaling.py | 36 ++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 18d46c6e..4c726d4e 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -28,6 +28,8 @@ """Binned mtz dataframe by wavelength(LAMBDA) with derived columns.""" FilteredWavelengthBinned = NewType("FilteredWavelengthBinned", sc.DataArray) """Filtered binned data.""" +SelectedReferenceWavelength = NewType("SelectedReferenceWavelength", sc.Variable) +"""The wavelength to select reference intensities.""" ReferenceIntensities = NewType("ReferenceIntensities", sc.DataArray) """Reference intensities selected by the wavelength.""" EstimatedScaleFactor = NewType("EstimatedScaleFactor", sc.DataArray) @@ -94,8 +96,10 @@ def filter_wavelegnth_binned( "The cut proportion should be in the range of 0 < proportion < 0.5." ) - cut_size = int(len(binned) * cut_proportion) - return FilteredWavelengthBinned(binned[cut_size:-cut_size]) + cut_size = int(binned.sizes[DEFAULT_WAVELENGTH_COORD_NAME] * cut_proportion) + return FilteredWavelengthBinned( + binned[DEFAULT_WAVELENGTH_COORD_NAME, cut_size:-cut_size] + ) def _is_bin_empty(binned: sc.DataArray, idx: int) -> bool: @@ -121,9 +125,32 @@ def _get_middle_bin_idx(binned: sc.DataArray) -> int: return cur_idx -def get_reference_intensities( +def get_reference_wavelength( binned: FilteredWavelengthBinned, reference_wavelength: Optional[ReferenceWavelength] = None, +) -> SelectedReferenceWavelength: + """Select the reference wavelength. + + Parameters + ---------- + reference_wavelength: + The reference wavelength to select the intensities. + If ``None``, the middle group is selected. + It should be a scalar variable as it is selecting one of bins. + + """ + if reference_wavelength is None: + ref_idx = _get_middle_bin_idx(binned) + return SelectedReferenceWavelength( + binned.coords[DEFAULT_WAVELENGTH_COORD_NAME][ref_idx] + ) + else: + return SelectedReferenceWavelength(reference_wavelength) + + +def get_reference_intensities( + binned: FilteredWavelengthBinned, + reference_wavelength: SelectedReferenceWavelength, ) -> ReferenceIntensities: """Find the reference intensities by the wavelength. @@ -134,8 +161,6 @@ def get_reference_intensities( reference_wavelength: The reference wavelength to select the intensities. - If ``None``, the middle group is selected. - It should be a scalar variable as it is selecting one of bins. Raises ------ @@ -348,6 +373,7 @@ def cut_estimated_scaled_intensities_by_n_root_std_dev( cut_estimated_scaled_intensities_by_n_root_std_dev, get_wavelength_binned, filter_wavelegnth_binned, + get_reference_wavelength, get_reference_intensities, estimate_scale_factor_per_hkl_asu_from_reference, average_roughly_scaled_intensities, From 88a89d6539674b76a00d44e65e618d856097b2fe Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 2 May 2024 14:25:53 +0200 Subject: [PATCH 163/403] Fix tests. --- packages/essnmx/tests/scaling_test.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/essnmx/tests/scaling_test.py b/packages/essnmx/tests/scaling_test.py index 2228dffb..c8f1e130 100644 --- a/packages/essnmx/tests/scaling_test.py +++ b/packages/essnmx/tests/scaling_test.py @@ -3,11 +3,12 @@ import pytest import scipp as sc -from ess.nmx.mtz_io import DEFAULT_WAVELENGTH_COLUMN_NAME +from ess.nmx.mtz_io import DEFAULT_WAVELENGTH_COORD_NAME from ess.nmx.scaling import ( ReferenceIntensities, estimate_scale_factor_per_hkl_asu_from_reference, get_reference_intensities, + get_reference_wavelength, ) @@ -16,7 +17,7 @@ def nmx_data_array() -> sc.DataArray: da = sc.DataArray( data=sc.array(dims=["row"], values=[1, 2, 3, 4, 5, 3.1, 3.2]), coords={ - DEFAULT_WAVELENGTH_COLUMN_NAME: sc.Variable( + DEFAULT_WAVELENGTH_COORD_NAME: sc.Variable( dims=["row"], values=[1, 2, 3, 4, 5, 3, 3] ), "hkl_asu": sc.array( @@ -42,8 +43,12 @@ def nmx_data_array() -> sc.DataArray: def test_get_reference_bin_middle(nmx_data_array: sc.DataArray) -> None: """Test the middle bin.""" + binned = nmx_data_array.bin({DEFAULT_WAVELENGTH_COORD_NAME: 6}) + reference_wavelength = get_reference_wavelength(binned) + ref_bin = get_reference_intensities( - nmx_data_array.bin({DEFAULT_WAVELENGTH_COLUMN_NAME: 6}) + nmx_data_array.bin({DEFAULT_WAVELENGTH_COORD_NAME: 6}), + reference_wavelength, ) selected_idx = (2, 5, 6) assert all( @@ -53,8 +58,12 @@ def test_get_reference_bin_middle(nmx_data_array: sc.DataArray) -> None: @pytest.fixture def reference_bin(nmx_data_array: sc.DataArray) -> ReferenceIntensities: + binned = nmx_data_array.bin({DEFAULT_WAVELENGTH_COORD_NAME: 6}) + reference_wavelength = get_reference_wavelength(binned) + return get_reference_intensities( - nmx_data_array.bin({DEFAULT_WAVELENGTH_COLUMN_NAME: 6}) + binned, + reference_wavelength, ) From 0137bcae868671106ae85a402c83b8f7e5338b13 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 14 May 2024 09:58:18 +0200 Subject: [PATCH 164/403] Add random sample dataset for documentation and testing. --- packages/essnmx/docs/developer/index.md | 1 + .../essnmx/docs/developer/test-dataset.ipynb | 151 ++++++++++++++++++ packages/essnmx/src/ess/nmx/data/__init__.py | 22 ++- 3 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 packages/essnmx/docs/developer/test-dataset.ipynb diff --git a/packages/essnmx/docs/developer/index.md b/packages/essnmx/docs/developer/index.md index 23b55441..4910390a 100644 --- a/packages/essnmx/docs/developer/index.md +++ b/packages/essnmx/docs/developer/index.md @@ -13,4 +13,5 @@ maxdepth: 2 getting-started coding-conventions dependency-management +test-dataset ``` diff --git a/packages/essnmx/docs/developer/test-dataset.ipynb b/packages/essnmx/docs/developer/test-dataset.ipynb new file mode 100644 index 00000000..3811f7fd --- /dev/null +++ b/packages/essnmx/docs/developer/test-dataset.ipynb @@ -0,0 +1,151 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Test Dataset\n", + "\n", + "This page has the instruction of how the test-datasets were generated and how they are used in the tests.\n", + "\n", + "## Scaling workflow - MTZ files\n", + "\n", + "MTZ test datasets are create with ``gemmi`` and random generator.\n", + "\n", + "We have multiple test MTZ files since multiple files are expected in usual cases.\n", + "\n", + "These files do not have any physical meaning and they are meant to be useful for testing the workflow.\n", + "\n", + "Here is the code cell to create the test files." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import gemmi\n", + "import pandas as pd\n", + "import numpy as np\n", + "from ess.nmx.mtz_io import (\n", + " DEFAULT_INTENSITY_COLUMN_NAME,\n", + " DEFAULT_WAVELENGTH_COLUMN_NAME,\n", + " DEFAULT_STD_DEV_COLUMN_NAME,\n", + " DEFAULT_SPACE_GROUP_DESC,\n", + ")\n", + "\n", + "# Negative intensities will happen due to corrections\n", + "# and high intensities are also expected in some cases\n", + "INTENSITY_RANGE = (-20.0, 200.0)\n", + "HKL_RANGE = (-100, 100)\n", + "MANDATORY_FIELDS = (\n", + " \"H\",\n", + " \"K\",\n", + " \"L\",\n", + " DEFAULT_WAVELENGTH_COLUMN_NAME, # LAMBDA\n", + " DEFAULT_INTENSITY_COLUMN_NAME, # I\n", + " DEFAULT_STD_DEV_COLUMN_NAME, # SIGI\n", + ")\n", + "global_rng = np.random.default_rng(0)\n", + "HKL_CANDIDATES = tuple(zip(*[global_rng.integers(*HKL_RANGE, size=100) for _ in range(3)]))\n", + "\n", + "def create_mtz_data_frame(random_seed: int) -> pd.DataFrame:\n", + " rng = np.random.default_rng(random_seed)\n", + " intensities = np.sort(rng.normal(50, 20, size=10_000))[::-1] + (rng.uniform(\n", + " *INTENSITY_RANGE, size=10_000\n", + " )* rng.choice([0]*99 + [1], size=10_000))\n", + " std_devs = np.multiply(intensities, rng.uniform(0.1, 0.15, size=10_000))\n", + " wavelengths = np.sort(rng.uniform(2.8, 3.2, size=10_000))[::-1]\n", + "\n", + " df = pd.DataFrame(\n", + " {\n", + " DEFAULT_INTENSITY_COLUMN_NAME: intensities,\n", + " DEFAULT_STD_DEV_COLUMN_NAME: std_devs,\n", + " DEFAULT_WAVELENGTH_COLUMN_NAME: wavelengths,\n", + " }\n", + " )\n", + " \n", + " df[[\"H\", \"K\", \"L\"]] = pd.Series(\n", + " rng.choice(HKL_CANDIDATES, size=10_000).tolist()\n", + " ).to_list()\n", + "\n", + " return df\n", + "\n", + "\n", + "def dataframe_to_mtz(df: pd.DataFrame) -> gemmi.Mtz:\n", + " \"\"\"Create a random MTZ file with a single dataset.\n", + "\n", + " Columns:\n", + " - H, K, L: Miller indices\n", + " - LAMBDA: Wavelength\n", + " - I: Intensity\n", + " - SIGI: Standard deviation of intensity\n", + "\n", + " \"\"\"\n", + " assert set(df.columns) == set(MANDATORY_FIELDS)\n", + "\n", + " mtz = gemmi.Mtz()\n", + " mtz.add_dataset(\"HKL\")\n", + " column_type_map = { # Column types: https://www.ccp4.ac.uk/html/mtzformat.html#coltypes\n", + " \"H\": \"H\",\n", + " \"K\": \"H\",\n", + " \"L\": \"H\",\n", + " DEFAULT_WAVELENGTH_COLUMN_NAME: \"R\",\n", + " DEFAULT_INTENSITY_COLUMN_NAME: \"J\",\n", + " DEFAULT_STD_DEV_COLUMN_NAME: \"Q\",\n", + " }\n", + "\n", + " for col_name in df.columns:\n", + " mtz.add_column(col_name, type=column_type_map[col_name], expand_data=True)\n", + "\n", + " mtz.spacegroup = gemmi.SpaceGroup(DEFAULT_SPACE_GROUP_DESC)\n", + " mtz.set_data(df.values)\n", + " return mtz\n", + "\n", + "\n", + "for seed in range(1, 6):\n", + " sample_df = create_mtz_data_frame(seed)\n", + " sample_mtz = dataframe_to_mtz(sample_df)\n", + " # sample_mtz.write_to_file(f\"sample_{seed}.mtz\") # Uncomment to save the MTZ file\n", + "\n", + "sample_df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Once the files were created, they were compressed into one file\n", + "and uploaded in the server where pooch can access to.\n", + "\n", + "Here is the script for compressing the files.\n", + "\n", + "```bash\n", + "tar -czvf mtz_random_samples.tar.gz sample_*.mtz\n", + "```" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nmx-dev-310", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/packages/essnmx/src/ess/nmx/data/__init__.py b/packages/essnmx/src/ess/nmx/data/__init__.py index bdc49063..12cf4889 100644 --- a/packages/essnmx/src/ess/nmx/data/__init__.py +++ b/packages/essnmx/src/ess/nmx/data/__init__.py @@ -20,6 +20,7 @@ def _make_pooch() -> pooch.Pooch: "small_mcstas_2_sample.h5": "md5:c3affe636397f8c9eea1d9c10a2bf487", "small_mcstas_3_sample.h5": "md5:2afaac205d13ee857ee5364e3f1957a7", "mtz_samples.tar.gz": "md5:bed1eaf604bbe8725c1f6a20ca79fcc0", + "mtz_random_samples.tar.gz": "md5:c8259ae2e605560ab88959e7109613b6", }, ) @@ -62,10 +63,29 @@ def get_path(name: str) -> str: def get_small_mtz_samples() -> list[pathlib.Path]: - """Return a list of path to MTZ sample files.""" + """Return a list of path to MTZ sample files randomly chosen from real dataset. + + This samples also contain optional columns. + """ from pooch.processors import Untar return [ pathlib.Path(file_path) for file_path in _pooch.fetch("mtz_samples.tar.gz", processor=Untar()) ] + + +def get_small_random_mtz_samples() -> list[pathlib.Path]: + """Return a list of path to MTZ sample files filled with random values + + This sample only contains mandatory columns for the workflow examples. + They are made for documentation, not necessarily for testing. + Use ``get_small_mtz_samples`` for testing since they are + more representative of real data. + """ + from pooch.processors import Untar + + return [ + pathlib.Path(file_path) + for file_path in _pooch.fetch("mtz_random_samples.tar.gz", processor=Untar()) + ] From 4c71f0a39b6489ff3d7ad74dfff37e2fd5f52865 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 14 May 2024 09:59:32 +0200 Subject: [PATCH 165/403] Remove bin-cutting step and add swapping filtering function in the example. --- .../docs/examples/scaling_workflow.ipynb | 139 +++++++++++-- packages/essnmx/src/ess/nmx/scaling.py | 196 ++++++++---------- 2 files changed, 214 insertions(+), 121 deletions(-) diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index f261a4c3..fa50d242 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -21,10 +21,10 @@ "outputs": [], "source": [ "from ess.nmx.mtz_io import read_mtz_file, mtz_to_pandas, MTZFilePath\n", - "from ess.nmx.data import get_small_mtz_samples\n", + "from ess.nmx.data import get_small_random_mtz_samples\n", "\n", "\n", - "small_mtz_sample = get_small_mtz_samples()[0]\n", + "small_mtz_sample = get_small_random_mtz_samples()[0]\n", "mtz = read_mtz_file(MTZFilePath(small_mtz_sample))\n", "df = mtz_to_pandas(mtz)\n", "df.head()" @@ -62,29 +62,36 @@ " WavelengthBinSize,\n", " FilteredEstimatedScaledIntensities,\n", " ReferenceWavelength,\n", - " WavelengthBinCutProportion,\n", - " NRoot,\n", - " NRootStdDevCut,\n", + " ScaledIntensityLeftTailThreshold,\n", + " ScaledIntensityRightTailThreshold,\n", ")\n", "\n", "pl = sl.Pipeline(\n", " providers=mtz_io_providers + scaling_providers,\n", " params={\n", " SpaceGroupDesc: \"C 1 2 1\",\n", - " WavelengthBinSize: 500,\n", + " WavelengthBinSize: 250,\n", " ReferenceWavelength: sc.scalar(\n", " 3, unit=sc.units.angstrom\n", " ), # Remove it if you want to use the middle of the bin\n", - " WavelengthBinCutProportion: 0.25, # 0 < proportion < 0.5\n", - " NRoot: 4, # Increase this value to effectively remove more outliers on the right tail\n", - " NRootStdDevCut: 1.0, # Lower this value to remove more outliers\n", + " ScaledIntensityLeftTailThreshold: sc.scalar(\n", + " 0.1, # Increase it to remove more outliers\n", + " ),\n", + " ScaledIntensityRightTailThreshold: sc.scalar(\n", + " 4.0, # Decrease it to remove more outliers\n", + " ),\n", " **mtz_io_params,\n", " **scaling_params,\n", " },\n", ")\n", "\n", + "\n", "file_path_table = sl.ParamTable(\n", - " row_dim=MTZFileIndex, columns={MTZFilePath: get_small_mtz_samples()}\n", + " # row_dim=MTZFileIndex,\n", + " # columns={\n", + " # MTZFilePath: [pathlib.Path(f\"../developer/sample_{i}.mtz\") for i in range(1, 6)]\n", + " # },\n", + " row_dim=MTZFileIndex, columns={MTZFilePath: get_small_random_mtz_samples()}\n", ")\n", "\n", "pl.set_param_table(file_path_table)\n", @@ -123,11 +130,119 @@ "source": [ "scaling_nmx_workflow.compute(FilteredEstimatedScaledIntensities)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "scaled = scaling_nmx_workflow.compute(FilteredEstimatedScaledIntensities)\n", + "\n", + "sc.values(scaled.data).hist(intensities=30).plot(\n", + " grid=True, linewidth=3, title=\"Density Plot of Estimated Scaled Intensities\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Change Provider\n", + "Here is an example of how to insert different filter function.\n", + "\n", + "In this example, we will swap a provider that filters ``EstimatedScaledIntensities`` and provide ``FilteredEstimatedScaledIntensities``." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from typing import NewType\n", + "import scipp as sc\n", + "from ess.nmx.scaling import (\n", + " EstimatedScaledIntensities,\n", + " FilteredEstimatedScaledIntensities,\n", + ")\n", + "\n", + "# Define the new types for the filtering function\n", + "NRoot = NewType(\"NRoot\", int)\n", + "\"\"\"The n-th root to be taken for the standard deviation.\"\"\"\n", + "NRootStdDevCut = NewType(\"NRootStdDevCut\", float)\n", + "\"\"\"The number of standard deviations to be cut from the n-th root data.\"\"\"\n", + "\n", + "\n", + "def _calculate_sample_standard_deviation(var: sc.Variable) -> sc.Variable:\n", + " \"\"\"Calculate the sample variation of the data.\n", + "\n", + " This helper function is a temporary solution before\n", + " we release new scipp version with the statistics helper.\n", + " \"\"\"\n", + " import numpy as np\n", + "\n", + " return sc.scalar(np.nanstd(var.values))\n", + "\n", + "\n", + "# Define the filtering function with right argument types and return type\n", + "def cut_estimated_scaled_intensities_by_n_root_std_dev(\n", + " scaled_intensities: EstimatedScaledIntensities,\n", + " n_root: NRoot,\n", + " n_root_std_dev_cut: NRootStdDevCut,\n", + ") -> FilteredEstimatedScaledIntensities:\n", + " \"\"\"Filter the mtz data array by the quad root of the sample standard deviation.\n", + "\n", + " Parameters\n", + " ----------\n", + " scaled_intensities:\n", + " The scaled intensities to be filtered.\n", + "\n", + " n_root:\n", + " The n-th root to be taken for the standard deviation.\n", + " Higher n-th root means cutting is more effective on the right tail.\n", + " More explanation can be found in the notes.\n", + "\n", + " n_root_std_dev_cut:\n", + " The number of standard deviations to be cut from the n-th root data.\n", + "\n", + " Returns\n", + " -------\n", + " :\n", + " The filtered scaled intensities.\n", + "\n", + " \"\"\"\n", + " # Check the range of the n-th root\n", + " if n_root < 1:\n", + " raise ValueError(\"The n-th root should be equal to or greater than 1.\")\n", + "\n", + " copied = scaled_intensities.copy(deep=False)\n", + " nth_root = copied.data ** (1 / n_root)\n", + " # Calculate the mean\n", + " nth_root_mean = nth_root.nanmean()\n", + " # Calculate the sample standard deviation\n", + " nth_root_std_dev = _calculate_sample_standard_deviation(nth_root)\n", + " # Calculate the cut value\n", + " half_window = n_root_std_dev_cut * nth_root_std_dev\n", + " keep_range = (nth_root_mean - half_window, nth_root_mean + half_window)\n", + "\n", + " # Filter the data\n", + " return FilteredEstimatedScaledIntensities(\n", + " copied[(nth_root > keep_range[0]) & (nth_root < keep_range[1])]\n", + " )\n", + "\n", + "\n", + "pl.insert(cut_estimated_scaled_intensities_by_n_root_std_dev)\n", + "pl[NRoot] = 4\n", + "pl[NRootStdDevCut] = 1.0\n", + "\n", + "pl.compute(FilteredEstimatedScaledIntensities)" + ] } ], "metadata": { "kernelspec": { - "display_name": "nmx-dev-310", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -145,5 +260,5 @@ } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 4c726d4e..aeae635d 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) -from typing import NewType, Optional +from typing import NewType, Optional, TypeVar import scipp as sc @@ -9,25 +9,21 @@ # User defined or configurable types WavelengthBinSize = NewType("WavelengthBinSize", int) """The size of the wavelength(LAMBDA) bins.""" -WavelengthRange = NewType("WavelengthRange", tuple[float, float]) -"""The range of the wavelength(LAMBDA) bins.""" -WavelengthBinCutProportion = NewType("WavelengthBinCutProportion", float) -"""The proportion of the wavelength(LAMBDA) bins to be cut off on both sides.""" -DEFAULT_WAVELENGTH_CUT_PROPORTION = WavelengthBinCutProportion(0.25) -"""Default proportion of the wavelength(LAMBDA) bins to be cut off from both sides.""" +MinWavelengthBinEdge = NewType("MinWavelengthBinEdge", sc.Variable) +"""The minimum edge of the wavelength(LAMBDA) bins.""" +DEFAULT_MIN_WAVELENGTH_BIN_EDGE = MinWavelengthBinEdge(sc.scalar(2.6, unit="angstrom")) +"""Default minimum edge of the wavelength(LAMBDA) bins.""" +MaxWavelengthBinEdge = NewType("MaxWavelengthBinEdge", sc.Variable) +"""The maximum edge of the wavelength(LAMBDA) bins.""" +DEFAULT_MAX_WAVELENGTH_BIN_EDGE = MaxWavelengthBinEdge(sc.scalar(3.6, unit="angstrom")) +"""Default maximum edge of the wavelength(LAMBDA) bins.""" ReferenceWavelength = NewType("ReferenceWavelength", sc.Variable) """The wavelength to select reference intensities.""" -NRoot = NewType("NRoot", int) -"""The n-th root to be taken for the standard deviation.""" -NRootStdDevCut = NewType("NRootStdDevCut", float) -"""The number of standard deviations to be cut from the n-th root data.""" # Computed types """Filtered mtz dataframe by the quad root of the sample standard deviation.""" WavelengthBinned = NewType("WavelengthBinned", sc.DataArray) """Binned mtz dataframe by wavelength(LAMBDA) with derived columns.""" -FilteredWavelengthBinned = NewType("FilteredWavelengthBinned", sc.DataArray) -"""Filtered binned data.""" SelectedReferenceWavelength = NewType("SelectedReferenceWavelength", sc.Variable) """The wavelength to select reference intensities.""" ReferenceIntensities = NewType("ReferenceIntensities", sc.DataArray) @@ -40,67 +36,56 @@ "FilteredEstimatedScaledIntensities", sc.DataArray ) +T = TypeVar("T") + + +def _if_not_none_else(x: T | None, default: T) -> T: + """Ternary operation helper for optional arguments.""" + return x if x is not None else default + def get_wavelength_binned( mtz_da: NMXMtzDataArray, wavelength_bin_size: WavelengthBinSize, - wavelength_range: Optional[WavelengthRange] = None, + min_wavelength_bin_edge: Optional[MinWavelengthBinEdge] = None, + max_wavelength_bin_edge: Optional[MaxWavelengthBinEdge] = None, ) -> WavelengthBinned: """Bin the whole dataset by wavelength(LAMBDA). - Notes - ----- - Wavelength(LAMBDA) binning should always be done on the merged dataset. - - """ - if wavelength_range is None: - binning_var = wavelength_bin_size - else: - binning_var = sc.linspace( - dim=DEFAULT_WAVELENGTH_COORD_NAME, - start=wavelength_range[0], - stop=wavelength_range[1], - num=wavelength_bin_size, - unit=mtz_da.coords[DEFAULT_WAVELENGTH_COORD_NAME].unit, - ) - - binned = mtz_da.bin({DEFAULT_WAVELENGTH_COORD_NAME: binning_var}) - - return WavelengthBinned(binned) - - -def filter_wavelegnth_binned( - binned: WavelengthBinned, - cut_proportion: WavelengthBinCutProportion = DEFAULT_WAVELENGTH_CUT_PROPORTION, -) -> FilteredWavelengthBinned: - """Filter the binned data by cutting off the edges. - Parameters ---------- - binned: - The binned data by wavelength(LAMBDA). + mtz_da: + The merged dataset. - cut_proportion: - The proportion of the wavelength(LAMBDA) bins to be cut off on both sides. - The default value is :attr:`~DEFAULT_WAVELENGTH_CUT_PROPORTION`. + wavelength_bin_size: + The size of the wavelength(LAMBDA) bins. - Returns - ------- - : - The filtered binned data. + min_wavelength_bin_edge: + The minimum edge of the wavelength(LAMBDA) bins. + Minimum value of the wavelength(LAMBDA) coordinate will be used if ``None``. - """ + max_wavelength_bin_edge: + The maximum edge of the wavelength(LAMBDA) bins. + Maximum value of the wavelength(LAMBDA) coordinate will be used if ``None``. - if cut_proportion < 0 or cut_proportion >= 0.5: - raise ValueError( - "The cut proportion should be in the range of 0 < proportion < 0.5." - ) + Notes + ----- + Wavelength(LAMBDA) binning should always be done on the merged dataset. - cut_size = int(binned.sizes[DEFAULT_WAVELENGTH_COORD_NAME] * cut_proportion) - return FilteredWavelengthBinned( - binned[DEFAULT_WAVELENGTH_COORD_NAME, cut_size:-cut_size] + """ + wavelength_coord = mtz_da.coords[DEFAULT_WAVELENGTH_COORD_NAME] + start = _if_not_none_else(min_wavelength_bin_edge, wavelength_coord.min()) + stop = _if_not_none_else(max_wavelength_bin_edge, wavelength_coord.max()) + binning_var = sc.linspace( + dim=DEFAULT_WAVELENGTH_COORD_NAME, + start=start, + stop=stop, + num=wavelength_bin_size, + unit=wavelength_coord.unit, ) + return WavelengthBinned(mtz_da.bin({DEFAULT_WAVELENGTH_COORD_NAME: binning_var})) + def _is_bin_empty(binned: sc.DataArray, idx: int) -> bool: """Check if the bin is empty.""" @@ -126,13 +111,16 @@ def _get_middle_bin_idx(binned: sc.DataArray) -> int: def get_reference_wavelength( - binned: FilteredWavelengthBinned, + binned: WavelengthBinned, reference_wavelength: Optional[ReferenceWavelength] = None, ) -> SelectedReferenceWavelength: """Select the reference wavelength. Parameters ---------- + binned: + The wavelength binned data. + reference_wavelength: The reference wavelength to select the intensities. If ``None``, the middle group is selected. @@ -149,7 +137,7 @@ def get_reference_wavelength( def get_reference_intensities( - binned: FilteredWavelengthBinned, + binned: WavelengthBinned, reference_wavelength: SelectedReferenceWavelength, ) -> ReferenceIntensities: """Find the reference intensities by the wavelength. @@ -233,7 +221,7 @@ def estimate_scale_factor_per_hkl_asu_from_reference( def average_roughly_scaled_intensities( - binned: FilteredWavelengthBinned, + binned: WavelengthBinned, scale_factor: EstimatedScaleFactor, ) -> EstimatedScaledIntensities: """Scale the intensities by the estimated scale factor. @@ -241,7 +229,7 @@ def average_roughly_scaled_intensities( Parameters ---------- binned: - Binned data by wavelength(LAMBDA) to be grouped and scaled. + The wavelength binned data. scale_factor: The estimated scale factor. @@ -300,79 +288,65 @@ def average_roughly_scaled_intensities( # Drop variances of the scale factor # Scale each group each bin by the scale factor - return EstimatedScaledIntensities( - sc.nanmean(grouped.bins.nanmean() * sc.values(scale_factor), dim="hkl_asu") + intensities = sc.nanmean( + grouped.bins.nanmean() * sc.values(scale_factor), dim="hkl_asu" ) + # Take the midpoints of the wavelength bin coordinates + # to represent the average wavelength of the bin + # It is because the bin-edges are dropped while flattening the data + # and the data is expected to be filtered after this step. + intensities.coords[DEFAULT_WAVELENGTH_COORD_NAME] = sc.midpoints( + intensities.coords[DEFAULT_WAVELENGTH_COORD_NAME], + ) + return EstimatedScaledIntensities(intensities) -def _calculate_sample_standard_deviation(var: sc.Variable) -> sc.Variable: - """Calculate the sample variation of the data. - - This helper function is a temporary solution before - we release new scipp version with the statistics helper. - """ - import numpy as np - - return sc.scalar(np.nanstd(var.values)) +ScaledIntensityLeftTailThreshold = NewType( + "ScaledIntensityLeftTailThreshold", sc.Variable +) +DEFAULT_LEFT_TAIL_THRESHOLD = ScaledIntensityLeftTailThreshold(sc.scalar(0.1)) +ScaledIntensityRightTailThreshold = NewType( + "ScaledIntensityRightTailThreshold", sc.Variable +) +DEFAULT_RIGHT_TAIL_THRESHOLD = ScaledIntensityRightTailThreshold(sc.scalar(2.0)) -def cut_estimated_scaled_intensities_by_n_root_std_dev( +def cut_tails( scaled_intensities: EstimatedScaledIntensities, - n_root: NRoot, - n_root_std_dev_cut: NRootStdDevCut, + left_threashold: ScaledIntensityLeftTailThreshold = DEFAULT_LEFT_TAIL_THRESHOLD, + right_threshold: ScaledIntensityRightTailThreshold = DEFAULT_RIGHT_TAIL_THRESHOLD, ) -> FilteredEstimatedScaledIntensities: - """Filter the mtz data array by the quad root of the sample standard deviation. + """Cut the right tail of the estimated scaled intensities by the threshold. Parameters ---------- scaled_intensities: The scaled intensities to be filtered. - n_root: - The n-th root to be taken for the standard deviation. - Higher n-th root means cutting is more effective on the right tail. - More explanation can be found in the notes. + left_threashold: + The threshold to be cut from the left tail. - n_root_std_dev_cut: - The number of standard deviations to be cut from the n-th root data. + right_threshold: + The threshold to be cut from the right tail. Returns ------- : - The filtered scaled intensities. + The filtered scaled intensities with the tails cut. """ - # Check the range of the n-th root - if n_root < 1: - raise ValueError("The n-th root should be equal to or greater than 1.") - - copied = scaled_intensities.copy(deep=False) - # Take the midpoints of the wavelength bin coordinates - # to represent the average wavelength of the bin - # It is because the bin-edges are dropped while flattening the data - copied.coords[DEFAULT_WAVELENGTH_COORD_NAME] = sc.midpoints( - copied.coords[DEFAULT_WAVELENGTH_COORD_NAME], - ) - nth_root = copied.data ** (1 / n_root) - # Calculate the mean - nth_root_mean = nth_root.mean() - # Calculate the sample standard deviation - nth_root_std_dev = _calculate_sample_standard_deviation(nth_root) - # Calculate the cut value - half_window = n_root_std_dev_cut * nth_root_std_dev - keep_range = (nth_root_mean - half_window, nth_root_mean + half_window) - - # Filter the data return FilteredEstimatedScaledIntensities( - copied[(nth_root > keep_range[0]) & (nth_root < keep_range[1])] + scaled_intensities[ + (scaled_intensities.data > left_threashold) + & (scaled_intensities.data < right_threshold) + ].copy(deep=False) ) # Providers and default parameters scaling_providers = ( - cut_estimated_scaled_intensities_by_n_root_std_dev, + cut_tails, get_wavelength_binned, - filter_wavelegnth_binned, get_reference_wavelength, get_reference_intensities, estimate_scale_factor_per_hkl_asu_from_reference, @@ -381,5 +355,9 @@ def cut_estimated_scaled_intensities_by_n_root_std_dev( """Providers for scaling data.""" scaling_params = { - WavelengthBinCutProportion: DEFAULT_WAVELENGTH_CUT_PROPORTION, + MinWavelengthBinEdge: DEFAULT_MIN_WAVELENGTH_BIN_EDGE, + MaxWavelengthBinEdge: DEFAULT_MAX_WAVELENGTH_BIN_EDGE, + ScaledIntensityLeftTailThreshold: DEFAULT_LEFT_TAIL_THRESHOLD, + ScaledIntensityRightTailThreshold: DEFAULT_RIGHT_TAIL_THRESHOLD, } +"""Default parameters for scaling data.""" From 218f313fc0e31bf8960c0a00c160b516bc5aefc7 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 16 May 2024 14:01:54 +0200 Subject: [PATCH 166/403] Fix index annotation in the docstring. --- packages/essnmx/src/ess/nmx/scaling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index aeae635d..f308afe4 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -247,7 +247,7 @@ def average_roughly_scaled_intensities( EstimatedScaledI_{\\lambda} = \\dfrac{ - \\sum_{k=1}^{N_{\\lambda, (hkl)}} EstimatedScaledI_{\\lambda, (hkl)} + \\sum_{i=1}^{N_{\\lambda, (hkl)}} EstimatedScaledI_{\\lambda, (hkl)} }{ N_{\\lambda, (hkl)} } From 05046ec7ee30324811405a30ad1a9ba8519b6249 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 16 May 2024 14:03:21 +0200 Subject: [PATCH 167/403] Add docstring of the domain types. --- packages/essnmx/src/ess/nmx/scaling.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index f308afe4..5d81dd43 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -304,10 +304,12 @@ def average_roughly_scaled_intensities( ScaledIntensityLeftTailThreshold = NewType( "ScaledIntensityLeftTailThreshold", sc.Variable ) +"""The threshold to cut the left tail of the estimated scaled intensities.""" DEFAULT_LEFT_TAIL_THRESHOLD = ScaledIntensityLeftTailThreshold(sc.scalar(0.1)) ScaledIntensityRightTailThreshold = NewType( "ScaledIntensityRightTailThreshold", sc.Variable ) +"""The threshold to cut the right tail of the estimated scaled intensities.""" DEFAULT_RIGHT_TAIL_THRESHOLD = ScaledIntensityRightTailThreshold(sc.scalar(2.0)) From 216b016a5706faba54ddedaf33c3267db0a6ef56 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 16 May 2024 14:32:13 +0200 Subject: [PATCH 168/403] Update docstring. --- packages/essnmx/src/ess/nmx/scaling.py | 28 +++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 5d81dd43..90052615 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -229,15 +229,23 @@ def average_roughly_scaled_intensities( Parameters ---------- binned: - The wavelength binned data. + Intensities binned in the wavelength dimension. + It will be grouped by reflection (hkl) in the process. scale_factor: - The estimated scale factor. + The estimated scale factor per reflection(hkl) of the reference wavelength bin. + See :func:`estimate_scale_factor_per_hkl_asu_from_reference` + for the calculation of the estimated scale factor. + + .. math:: + + EstimatedScaleFactor_{(hkl)} = + average( \\dfrac{1}{I_{\\lambda=reference, (hkl)}} ) Returns ------- : - Average scaled intensties on ``hkl(asu)`` indices per wavelength. + Average scaled intensities on ``hkl(asu)`` indices per wavelength. Notes ----- @@ -247,7 +255,8 @@ def average_roughly_scaled_intensities( EstimatedScaledI_{\\lambda} = \\dfrac{ - \\sum_{i=1}^{N_{\\lambda, (hkl)}} EstimatedScaledI_{\\lambda, (hkl)} + \\sum_{i=1}^{N_{\\lambda, (hkl)}} + EstimatedScaledI_{\\lambda, (hkl)} }{ N_{\\lambda, (hkl)} } @@ -261,22 +270,23 @@ def average_roughly_scaled_intensities( \\begin{eqnarray} EstimatedScaledI_{\\lambda, (hkl)} \\\\ = \\dfrac{ - \\sum_{i=1}^{N_{reference, (hkl)}} + \\sum_{i=1}^{N_{\\lambda=reference, (hkl)}} \\sum_{j=1}^{N_{\\lambda, (hkl)}} \\dfrac{I_{j}}{I_{i}} }{ - N_{reference, (hkl)}*N_{\\lambda, (hkl)} + N_{\\lambda=reference, (hkl)}*N_{\\lambda, (hkl)} } \\\\ = \\dfrac{ - \\sum_{i=1}^{N_{reference, (hkl)}} \\dfrac{1}{I_{i}} + \\sum_{i=1}^{N_{\\lambda=reference, (hkl)}} \\dfrac{1}{I_{i}} }{ - N_{reference, (hkl)} + N_{\\lambda=reference, (hkl)} } * \\dfrac{ \\sum_{j=1}^{N_{\\lambda, (hkl)}} I_{j} }{ N_{\\lambda, (hkl)} } \\\\ - = average( \\dfrac{1}{I_{ref, (hkl)}} ) * average( I_{\\lambda, (hkl)} ) + = average( \\dfrac{1}{I_{\\lambda=reference, (hkl)}} ) + * average( I_{\\lambda, (hkl)} ) \\end{eqnarray} Therefore the ``binned(wavelength dimension)`` should be From 7d3141d89d2c20b99410366265d0a11ca16f1035 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 16 May 2024 16:12:40 +0200 Subject: [PATCH 169/403] Remove unecessary tenary helper :) --- packages/essnmx/src/ess/nmx/scaling.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 90052615..d2c70c81 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -39,11 +39,6 @@ T = TypeVar("T") -def _if_not_none_else(x: T | None, default: T) -> T: - """Ternary operation helper for optional arguments.""" - return x if x is not None else default - - def get_wavelength_binned( mtz_da: NMXMtzDataArray, wavelength_bin_size: WavelengthBinSize, @@ -74,8 +69,16 @@ def get_wavelength_binned( """ wavelength_coord = mtz_da.coords[DEFAULT_WAVELENGTH_COORD_NAME] - start = _if_not_none_else(min_wavelength_bin_edge, wavelength_coord.min()) - stop = _if_not_none_else(max_wavelength_bin_edge, wavelength_coord.max()) + start = ( + min_wavelength_bin_edge + if min_wavelength_bin_edge is not None + else wavelength_coord.min() + ) + stop = ( + max_wavelength_bin_edge + if max_wavelength_bin_edge is not None + else wavelength_coord.max() + ) binning_var = sc.linspace( dim=DEFAULT_WAVELENGTH_COORD_NAME, start=start, From 2e714716ad493cb242e01df914e078ec08005d93 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 May 2024 17:34:38 +0000 Subject: [PATCH 170/403] Bump scipp from 24.2.0 to 24.5.0 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 24.2.0 to 24.5.0. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/24.02.0...24.05.0) --- updated-dependencies: - dependency-name: scipp dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 16fa8066..15fa8425 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -35,8 +35,6 @@ idna==3.6 # via requests importlib-metadata==7.0.2 # via dask -importlib-resources==6.1.3 - # via matplotlib kiwisolver==1.4.5 # via matplotlib locket==1.0.0 @@ -83,7 +81,7 @@ requests==2.31.0 # via pooch sciline==24.2.1 # via -r base.in -scipp==24.2.0 +scipp==24.5.0 # via # -r base.in # scippnexus @@ -102,6 +100,4 @@ tzdata==2024.1 urllib3==2.2.1 # via requests zipp==3.17.0 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata From c018b821c7348b3e58dbefb87761066d0c207ae4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 31 May 2024 18:03:39 +0000 Subject: [PATCH 171/403] Bump scipp from 24.5.0 to 24.5.1 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 24.5.0 to 24.5.1. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/24.05.0...24.05.1) --- updated-dependencies: - dependency-name: scipp dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 15fa8425..f60ca9d4 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -81,7 +81,7 @@ requests==2.31.0 # via pooch sciline==24.2.1 # via -r base.in -scipp==24.5.0 +scipp==24.5.1 # via # -r base.in # scippnexus From 06405d26a1c0391670e12765b08c42028795167d Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Mon, 27 May 2024 16:24:14 +0200 Subject: [PATCH 172/403] Add fitting and scaling step in the workflow. --- .../docs/examples/scaling_workflow.ipynb | 152 +++++++++++++++++- packages/essnmx/src/ess/nmx/scaling.py | 93 +++++++++++ 2 files changed, 238 insertions(+), 7 deletions(-) diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index fa50d242..f71da98b 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -14,6 +14,15 @@ "They are wrapping ``MTZ`` IO functions of ``gemmi``." ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline" + ] + }, { "cell_type": "code", "execution_count": null, @@ -54,6 +63,7 @@ "source": [ "import sciline as sl\n", "import scipp as sc\n", + "import numpy as np\n", "\n", "from ess.nmx.mtz_io import mtz_io_providers, mtz_io_params\n", "from ess.nmx.mtz_io import MTZFileIndex, SpaceGroupDesc\n", @@ -64,6 +74,7 @@ " ReferenceWavelength,\n", " ScaledIntensityLeftTailThreshold,\n", " ScaledIntensityRightTailThreshold,\n", + " InitialGuess,\n", ")\n", "\n", "pl = sl.Pipeline(\n", @@ -80,6 +91,7 @@ " ScaledIntensityRightTailThreshold: sc.scalar(\n", " 4.0, # Decrease it to remove more outliers\n", " ),\n", + " InitialGuess: np.ones(7),\n", " **mtz_io_params,\n", " **scaling_params,\n", " },\n", @@ -91,7 +103,8 @@ " # columns={\n", " # MTZFilePath: [pathlib.Path(f\"../developer/sample_{i}.mtz\") for i in range(1, 6)]\n", " # },\n", - " row_dim=MTZFileIndex, columns={MTZFilePath: get_small_random_mtz_samples()}\n", + " row_dim=MTZFileIndex,\n", + " columns={MTZFilePath: get_small_random_mtz_samples()},\n", ")\n", "\n", "pl.set_param_table(file_path_table)\n", @@ -111,7 +124,9 @@ "metadata": {}, "outputs": [], "source": [ - "scaling_nmx_workflow = pl.get(FilteredEstimatedScaledIntensities)\n", + "from ess.nmx.scaling import WavelengthScaleFactors\n", + "\n", + "scaling_nmx_workflow = pl.get(WavelengthScaleFactors)\n", "scaling_nmx_workflow.visualize(graph_attr={\"rankdir\": \"LR\"})" ] }, @@ -128,7 +143,33 @@ "metadata": {}, "outputs": [], "source": [ - "scaling_nmx_workflow.compute(FilteredEstimatedScaledIntensities)" + "from ess.nmx.scaling import (\n", + " SelectedReferenceWavelength,\n", + " FittingResult,\n", + " WavelengthScaleFactors,\n", + ")\n", + "\n", + "results = scaling_nmx_workflow.compute(\n", + " (\n", + " FilteredEstimatedScaledIntensities,\n", + " SelectedReferenceWavelength,\n", + " FittingResult,\n", + " WavelengthScaleFactors,\n", + " )\n", + ")\n", + "\n", + "results[FilteredEstimatedScaledIntensities]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plots\n", + "\n", + "Here are plotting examples of the fitting/estimation results.\n", + "\n", + "### Estimated Scaled Intensities." ] }, { @@ -137,13 +178,108 @@ "metadata": {}, "outputs": [], "source": [ - "scaled = scaling_nmx_workflow.compute(FilteredEstimatedScaledIntensities)\n", + "import scipy.stats as stats\n", + "import matplotlib.pyplot as plt\n", "\n", - "sc.values(scaled.data).hist(intensities=30).plot(\n", - " grid=True, linewidth=3, title=\"Density Plot of Estimated Scaled Intensities\"\n", + "fig, (density_ax, prob_ax) = plt.subplots(1, 2, figsize=(10, 5))\n", + "\n", + "densities = sc.values(results[FilteredEstimatedScaledIntensities].data).values\n", + "sc.values(results[FilteredEstimatedScaledIntensities].data).hist(intensity=50).plot(\n", + " title=\"Filtered Estimated Scaled Intensities\\nDensity Plot\",\n", + " grid=True,\n", + " linewidth=3,\n", + " ax=density_ax,\n", + ")\n", + "stats.probplot(densities, dist=\"norm\", plot=prob_ax)\n", + "prob_ax.set_title(\"Filtered Estimated Scaled Intensities\\nProbability Plot\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Curve Fitting" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import plopp as pp\n", + "from ess.nmx.scaling import FittingResult\n", + "\n", + "chebyshev_func = np.polynomial.chebyshev.Chebyshev(np.array([1, -1, 1]))\n", + "scale_function = np.vectorize(\n", + " chebyshev_func / chebyshev_func(results[SelectedReferenceWavelength].value)\n", + ")\n", + "chevyshev_normed = sc.DataArray(\n", + " data=sc.array(\n", + " dims=[\"wavelength\"],\n", + " values=scale_function(\n", + " results[FittingResult].fit_output.coords[\"wavelength\"].values\n", + " ),\n", + " ),\n", + " coords={\n", + " \"wavelength\": results[FittingResult].fit_output.coords[\"wavelength\"],\n", + " },\n", + ")\n", + "pp.plot(\n", + " {\n", + " \"Original Data\": results[FilteredEstimatedScaledIntensities],\n", + " \"Fit Result\": results[FittingResult].fit_output,\n", + " \"Chebyshev\": chevyshev_normed,\n", + " },\n", + " grid=True,\n", + " title=\"Fit Result [Intensities vs Wavelength]\",\n", + " marker={\"Chebyshev\": None, \"Fit Result\": None},\n", + " linestyle={\"Chebyshev\": \"solid\", \"Fit Result\": \"solid\"},\n", ")" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "reference_wavelength = sc.DataArray(\n", + " data=sc.concat(\n", + " [\n", + " results[WavelengthScaleFactors].data.min(),\n", + " results[WavelengthScaleFactors].data.max(),\n", + " ],\n", + " \"wavelength\",\n", + " ),\n", + " coords={\n", + " \"wavelength\": sc.broadcast(\n", + " results[SelectedReferenceWavelength], dims=[\"wavelength\"], shape=[2]\n", + " )\n", + " },\n", + ")\n", + "wavelength_scale_factor_plot = pp.plot(\n", + " {\n", + " \"scale_factor\": results[WavelengthScaleFactors],\n", + " \"reference_wavelength\": reference_wavelength,\n", + " },\n", + " title=\"Wavelength Scale Factors\",\n", + " grid=True,\n", + " marker={\"reference_wavelength\": None},\n", + " linestyle={\"reference_wavelength\": \"solid\"},\n", + ")\n", + "wavelength_scale_factor_plot.ax.set_xlim(2.8, 3.2)\n", + "reference_wavelength = results[SelectedReferenceWavelength].value\n", + "wavelength_scale_factor_plot.ax.text(\n", + " 3.0,\n", + " 0.25,\n", + " f\"{reference_wavelength=:} [{results[SelectedReferenceWavelength].unit}]\",\n", + " fontsize=8,\n", + " color=\"black\",\n", + ")\n", + "wavelength_scale_factor_plot" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -151,7 +287,9 @@ "## Change Provider\n", "Here is an example of how to insert different filter function.\n", "\n", - "In this example, we will swap a provider that filters ``EstimatedScaledIntensities`` and provide ``FilteredEstimatedScaledIntensities``." + "In this example, we will swap a provider that filters ``EstimatedScaledIntensities`` and provide ``FilteredEstimatedScaledIntensities``.\n", + "\n", + "After updating the providers, you can go back to [Compute Desired Type](#Compute-Desired-Type) and start over." ] }, { diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index d2c70c81..74ded47a 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -1,7 +1,10 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass from typing import NewType, Optional, TypeVar +import numpy as np import scipp as sc from .mtz_io import DEFAULT_WAVELENGTH_COORD_NAME, NMXMtzDataArray @@ -358,6 +361,93 @@ def cut_tails( ) +FittingInputX = NewType("FittingInputX", sc.Variable) +FittingOutputY = NewType("FittingOutputY", Sequence) +FittingFunc = Callable[..., FittingOutputY] + + +def chepyshev_polynomial( + wavelength: FittingInputX, + param1, + param2, + param3, + param4, + param5, + param6, + param7, +) -> FittingOutputY: + return np.polynomial.chebyshev.Chebyshev( + (param1, param2, param3, param4, param5, param6, param7) + )(wavelength) + + +FittingParams = NewType("FittingParams", Mapping) +"""Parameters of the fitting function.""" +FittingParamsCovariances = NewType("FittingParamsCovariances", Mapping) +"""Covariance of the :attr:`~FittingParams`.""" +InitialGuess = NewType("InitialGuess", np.ndarray) +"""Initial guess of the parameters for :attr:`~FittingFunc`.""" + + +@dataclass +class FittingResult: + """Result of the fitting process.""" + + fitting_func: FittingFunc + params: FittingParams + covariance: FittingParamsCovariances + fit_output: sc.DataArray + + +def fit_scale_factor_curve( + estimated_intensities: FilteredEstimatedScaledIntensities, + *, + initial_guess: InitialGuess, + fitting_func: FittingFunc = chepyshev_polynomial, +) -> FittingResult: + from scipy.optimize import curve_fit + + params, cov = curve_fit( + f=fitting_func, + xdata=estimated_intensities.coords["wavelength"].values, + ydata=estimated_intensities.data.values, + p0=initial_guess, + ) + return FittingResult( + fitting_func=fitting_func, + params=FittingParams(params), + covariance=FittingParamsCovariances(cov), + fit_output=sc.DataArray( + data=sc.array( + dims=["wavelength"], + values=fitting_func( + estimated_intensities.coords["wavelength"].values, *params + ), + ), + coords={"wavelength": estimated_intensities.coords["wavelength"]}, + ), + ) + + +WavelengthScaleFactors = NewType("WavelengthScaleFactors", sc.DataArray) +"""The scale factors of :attr:`~DEFAULT_WAVELENGTH_COORD_NAME`.""" + + +def calculate_wavelength_scale_factor( + fitting_result: FittingResult, + reference_wavelength: SelectedReferenceWavelength, +) -> WavelengthScaleFactors: + """Calculate the scale factors of the :attr:`~DEFAULT_WAVELENGTH_COORD_NAME`.""" + + scaled_reference = sc.scalar( + fitting_result.fitting_func(reference_wavelength.value, *fitting_result.params), + unit=reference_wavelength.unit, + dtype=reference_wavelength.dtype, + ) + scale_factor = fitting_result.fit_output / scaled_reference + return WavelengthScaleFactors(scale_factor) + + # Providers and default parameters scaling_providers = ( cut_tails, @@ -366,6 +456,8 @@ def cut_tails( get_reference_intensities, estimate_scale_factor_per_hkl_asu_from_reference, average_roughly_scaled_intensities, + fit_scale_factor_curve, + calculate_wavelength_scale_factor, ) """Providers for scaling data.""" @@ -374,5 +466,6 @@ def cut_tails( MaxWavelengthBinEdge: DEFAULT_MAX_WAVELENGTH_BIN_EDGE, ScaledIntensityLeftTailThreshold: DEFAULT_LEFT_TAIL_THRESHOLD, ScaledIntensityRightTailThreshold: DEFAULT_RIGHT_TAIL_THRESHOLD, + FittingFunc: chepyshev_polynomial, } """Default parameters for scaling data.""" From 8a054a42e0f1bc4354b31a39c60adf8d7584fde7 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 28 May 2024 12:49:47 +0200 Subject: [PATCH 173/403] Fix typo and simplify fitting function. --- .../essnmx/docs/examples/scaling_workflow.ipynb | 4 ++-- packages/essnmx/src/ess/nmx/scaling.py | 15 ++------------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index f71da98b..e84fd31e 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -214,7 +214,7 @@ "scale_function = np.vectorize(\n", " chebyshev_func / chebyshev_func(results[SelectedReferenceWavelength].value)\n", ")\n", - "chevyshev_normed = sc.DataArray(\n", + "chebyshev_normed = sc.DataArray(\n", " data=sc.array(\n", " dims=[\"wavelength\"],\n", " values=scale_function(\n", @@ -229,7 +229,7 @@ " {\n", " \"Original Data\": results[FilteredEstimatedScaledIntensities],\n", " \"Fit Result\": results[FittingResult].fit_output,\n", - " \"Chebyshev\": chevyshev_normed,\n", + " \"Chebyshev\": chebyshev_normed,\n", " },\n", " grid=True,\n", " title=\"Fit Result [Intensities vs Wavelength]\",\n", diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 74ded47a..72130b93 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -366,19 +366,8 @@ def cut_tails( FittingFunc = Callable[..., FittingOutputY] -def chepyshev_polynomial( - wavelength: FittingInputX, - param1, - param2, - param3, - param4, - param5, - param6, - param7, -) -> FittingOutputY: - return np.polynomial.chebyshev.Chebyshev( - (param1, param2, param3, param4, param5, param6, param7) - )(wavelength) +def chepyshev_polynomial(wavelength: FittingInputX, *params) -> FittingOutputY: + return np.polynomial.chebyshev.Chebyshev(params)(wavelength) FittingParams = NewType("FittingParams", Mapping) From 1f0e10e54797b40efee9dc98dee53d83d118bcc1 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 28 May 2024 15:16:40 +0200 Subject: [PATCH 174/403] Use scipp curve_fit. --- .../docs/examples/scaling_workflow.ipynb | 9 +-- packages/essnmx/src/ess/nmx/scaling.py | 58 +++++++++++-------- 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index e84fd31e..2eb0e80e 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -63,7 +63,6 @@ "source": [ "import sciline as sl\n", "import scipp as sc\n", - "import numpy as np\n", "\n", "from ess.nmx.mtz_io import mtz_io_providers, mtz_io_params\n", "from ess.nmx.mtz_io import MTZFileIndex, SpaceGroupDesc\n", @@ -91,7 +90,7 @@ " ScaledIntensityRightTailThreshold: sc.scalar(\n", " 4.0, # Decrease it to remove more outliers\n", " ),\n", - " InitialGuess: np.ones(7),\n", + " InitialGuess: {f\"param{i+1}\": 1 / (sc.scalar(1, unit=\"angstrom\") ** i) for i in range(7)},\n", " **mtz_io_params,\n", " **scaling_params,\n", " },\n", @@ -99,10 +98,6 @@ "\n", "\n", "file_path_table = sl.ParamTable(\n", - " # row_dim=MTZFileIndex,\n", - " # columns={\n", - " # MTZFilePath: [pathlib.Path(f\"../developer/sample_{i}.mtz\") for i in range(1, 6)]\n", - " # },\n", " row_dim=MTZFileIndex,\n", " columns={MTZFilePath: get_small_random_mtz_samples()},\n", ")\n", @@ -158,7 +153,7 @@ " )\n", ")\n", "\n", - "results[FilteredEstimatedScaledIntensities]" + "results[WavelengthScaleFactors]" ] }, { diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 72130b93..8056f538 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -1,10 +1,9 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Mapping from dataclasses import dataclass from typing import NewType, Optional, TypeVar -import numpy as np import scipp as sc from .mtz_io import DEFAULT_WAVELENGTH_COORD_NAME, NMXMtzDataArray @@ -362,19 +361,36 @@ def cut_tails( FittingInputX = NewType("FittingInputX", sc.Variable) -FittingOutputY = NewType("FittingOutputY", Sequence) +FittingOutputY = NewType("FittingOutputY", sc.Variable) FittingFunc = Callable[..., FittingOutputY] -def chepyshev_polynomial(wavelength: FittingInputX, *params) -> FittingOutputY: - return np.polynomial.chebyshev.Chebyshev(params)(wavelength) +def polynomial_7( + wavelength: FittingInputX, + param1: sc.Variable, + param2: sc.Variable, + param3: sc.Variable, + param4: sc.Variable, + param5: sc.Variable, + param6: sc.Variable, + param7: sc.Variable, +) -> FittingOutputY: + return FittingOutputY( + (wavelength**6) * sc.values(param7) + + (wavelength**5) * sc.values(param6) + + (wavelength**4) * sc.values(param5) + + (wavelength**3) * sc.values(param4) + + (wavelength**2) * sc.values(param3) + + (wavelength**1) * sc.values(param2) + + sc.values(param1) + ) FittingParams = NewType("FittingParams", Mapping) """Parameters of the fitting function.""" FittingParamsCovariances = NewType("FittingParamsCovariances", Mapping) """Covariance of the :attr:`~FittingParams`.""" -InitialGuess = NewType("InitialGuess", np.ndarray) +InitialGuess = NewType("InitialGuess", dict) """Initial guess of the parameters for :attr:`~FittingFunc`.""" @@ -392,27 +408,21 @@ def fit_scale_factor_curve( estimated_intensities: FilteredEstimatedScaledIntensities, *, initial_guess: InitialGuess, - fitting_func: FittingFunc = chepyshev_polynomial, + fitting_func: FittingFunc = polynomial_7, ) -> FittingResult: - from scipy.optimize import curve_fit - - params, cov = curve_fit( + p_result, cov_result = sc.curve_fit( + coords=["wavelength"], f=fitting_func, - xdata=estimated_intensities.coords["wavelength"].values, - ydata=estimated_intensities.data.values, + da=estimated_intensities, p0=initial_guess, ) + data = fitting_func(estimated_intensities.coords["wavelength"], **p_result) return FittingResult( fitting_func=fitting_func, - params=FittingParams(params), - covariance=FittingParamsCovariances(cov), + params=FittingParams(p_result), + covariance=FittingParamsCovariances(cov_result), fit_output=sc.DataArray( - data=sc.array( - dims=["wavelength"], - values=fitting_func( - estimated_intensities.coords["wavelength"].values, *params - ), - ), + data=data.data if isinstance(data, sc.DataArray) else data, coords={"wavelength": estimated_intensities.coords["wavelength"]}, ), ) @@ -428,10 +438,8 @@ def calculate_wavelength_scale_factor( ) -> WavelengthScaleFactors: """Calculate the scale factors of the :attr:`~DEFAULT_WAVELENGTH_COORD_NAME`.""" - scaled_reference = sc.scalar( - fitting_result.fitting_func(reference_wavelength.value, *fitting_result.params), - unit=reference_wavelength.unit, - dtype=reference_wavelength.dtype, + scaled_reference = fitting_result.fitting_func( + reference_wavelength, **fitting_result.params ) scale_factor = fitting_result.fit_output / scaled_reference return WavelengthScaleFactors(scale_factor) @@ -455,6 +463,6 @@ def calculate_wavelength_scale_factor( MaxWavelengthBinEdge: DEFAULT_MAX_WAVELENGTH_BIN_EDGE, ScaledIntensityLeftTailThreshold: DEFAULT_LEFT_TAIL_THRESHOLD, ScaledIntensityRightTailThreshold: DEFAULT_RIGHT_TAIL_THRESHOLD, - FittingFunc: chepyshev_polynomial, + FittingFunc: polynomial_7, } """Default parameters for scaling data.""" From 755e16a2e8d1ac9f7dcf29353cd1a195dd2b471f Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 28 May 2024 15:21:05 +0200 Subject: [PATCH 175/403] Fix docs. --- packages/essnmx/docs/examples/scaling_workflow.ipynb | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index 2eb0e80e..fa329fcd 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -203,6 +203,7 @@ "outputs": [], "source": [ "import plopp as pp\n", + "import numpy as np\n", "from ess.nmx.scaling import FittingResult\n", "\n", "chebyshev_func = np.polynomial.chebyshev.Chebyshev(np.array([1, -1, 1]))\n", From a33e85c099c0b794783d560f58fa9b0706709848 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Mon, 10 Jun 2024 06:32:27 +0200 Subject: [PATCH 176/403] Update scaling helper. --- .../docs/examples/scaling_workflow.ipynb | 12 -- packages/essnmx/src/ess/nmx/scaling.py | 126 +++++++++++------- 2 files changed, 79 insertions(+), 59 deletions(-) diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index fa329fcd..149dfe09 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -210,22 +210,10 @@ "scale_function = np.vectorize(\n", " chebyshev_func / chebyshev_func(results[SelectedReferenceWavelength].value)\n", ")\n", - "chebyshev_normed = sc.DataArray(\n", - " data=sc.array(\n", - " dims=[\"wavelength\"],\n", - " values=scale_function(\n", - " results[FittingResult].fit_output.coords[\"wavelength\"].values\n", - " ),\n", - " ),\n", - " coords={\n", - " \"wavelength\": results[FittingResult].fit_output.coords[\"wavelength\"],\n", - " },\n", - ")\n", "pp.plot(\n", " {\n", " \"Original Data\": results[FilteredEstimatedScaledIntensities],\n", " \"Fit Result\": results[FittingResult].fit_output,\n", - " \"Chebyshev\": chebyshev_normed,\n", " },\n", " grid=True,\n", " title=\"Fit Result [Intensities vs Wavelength]\",\n", diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 8056f538..d7d88310 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -360,69 +360,101 @@ def cut_tails( ) -FittingInputX = NewType("FittingInputX", sc.Variable) -FittingOutputY = NewType("FittingOutputY", sc.Variable) -FittingFunc = Callable[..., FittingOutputY] - - -def polynomial_7( - wavelength: FittingInputX, - param1: sc.Variable, - param2: sc.Variable, - param3: sc.Variable, - param4: sc.Variable, - param5: sc.Variable, - param6: sc.Variable, - param7: sc.Variable, -) -> FittingOutputY: - return FittingOutputY( - (wavelength**6) * sc.values(param7) - + (wavelength**5) * sc.values(param6) - + (wavelength**4) * sc.values(param5) - + (wavelength**3) * sc.values(param4) - + (wavelength**2) * sc.values(param3) - + (wavelength**1) * sc.values(param2) - + sc.values(param1) - ) - - -FittingParams = NewType("FittingParams", Mapping) -"""Parameters of the fitting function.""" -FittingParamsCovariances = NewType("FittingParamsCovariances", Mapping) -"""Covariance of the :attr:`~FittingParams`.""" -InitialGuess = NewType("InitialGuess", dict) -"""Initial guess of the parameters for :attr:`~FittingFunc`.""" - - @dataclass class FittingResult: """Result of the fitting process.""" - fitting_func: FittingFunc - params: FittingParams - covariance: FittingParamsCovariances + fitting_func: Callable[..., sc.DataArray] + """The fitting function to be used for fitting.""" + params: Mapping + """Parameters of the fitting function.""" + covariance: Mapping + """Covariance of the :attr:`~FittingParams`.""" fit_output: sc.DataArray + """The final output of the fitting function.""" + + +def polyval_wavelength( + wavelength: sc.Variable, *, out_unit: str, **kwargs +) -> sc.DataArray: + """Polynomial helper for fitting. + + The coefficients are adjusted to make the fitting result + have ``out_unit`` as unit. + + Parameters + ---------- + wavelength: + The wavelength coordinate. + out_unit: + The unit of the output. + **kwargs: + The polynomial coefficients. + + Returns + ------- + : + The polynomial calculated at the wavelength. + + + """ + out = sc.zeros_like(wavelength) + out.unit = out_unit + xk = sc.ones_like(wavelength) + for _, arg_value in enumerate(kwargs.values()): + out += sc.values(arg_value) * xk * sc.scalar(1.0, unit=out.unit / xk.unit) + xk *= wavelength + return out + + +WavelengthFittingPolynomialDegree = NewType("WavelengthFittingPolynomialDegree", int) +DEFAULT_WAVELENGTH_FITTING_POLYNOMIAL_DEGREE = WavelengthFittingPolynomialDegree(7) -def fit_scale_factor_curve( +def fit_wavelength_scale_factor_polynomial( estimated_intensities: FilteredEstimatedScaledIntensities, *, - initial_guess: InitialGuess, - fitting_func: FittingFunc = polynomial_7, + n_degree: WavelengthFittingPolynomialDegree, ) -> FittingResult: + """Fit the wavelength scale factor polynomial. + + It uses :func:`polyval_wavelength` as the fitting function + and :func:`scipp.optimize.curve_fit` for the fitting process. + The initial guess for the polynomial coefficients is set to 1 + for all degrees. + The unit of the coefficients is adjusted to make the fitting result + dimensionless. + + Parameters + ---------- + estimated_intensities: + The estimated scaled intensities to be fitted. + n_degree: + The degree of the polynomial to be fitted. + + Returns + ------- + : + The fitting result. + + """ + + from functools import partial + + fitting_func = partial(polyval_wavelength, out_unit="dimensionless") p_result, cov_result = sc.curve_fit( coords=["wavelength"], f=fitting_func, da=estimated_intensities, - p0=initial_guess, + p0={f"arg{i}": sc.scalar(1) for i in range(n_degree)}, ) data = fitting_func(estimated_intensities.coords["wavelength"], **p_result) return FittingResult( fitting_func=fitting_func, - params=FittingParams(p_result), - covariance=FittingParamsCovariances(cov_result), + params=p_result, + covariance=cov_result, fit_output=sc.DataArray( - data=data.data if isinstance(data, sc.DataArray) else data, + data=data.data, coords={"wavelength": estimated_intensities.coords["wavelength"]}, ), ) @@ -436,7 +468,7 @@ def calculate_wavelength_scale_factor( fitting_result: FittingResult, reference_wavelength: SelectedReferenceWavelength, ) -> WavelengthScaleFactors: - """Calculate the scale factors of the :attr:`~DEFAULT_WAVELENGTH_COORD_NAME`.""" + """Calculate the scale factors along the :attr:`~DEFAULT_WAVELENGTH_COORD_NAME`.""" scaled_reference = fitting_result.fitting_func( reference_wavelength, **fitting_result.params @@ -453,7 +485,7 @@ def calculate_wavelength_scale_factor( get_reference_intensities, estimate_scale_factor_per_hkl_asu_from_reference, average_roughly_scaled_intensities, - fit_scale_factor_curve, + fit_wavelength_scale_factor_polynomial, calculate_wavelength_scale_factor, ) """Providers for scaling data.""" @@ -463,6 +495,6 @@ def calculate_wavelength_scale_factor( MaxWavelengthBinEdge: DEFAULT_MAX_WAVELENGTH_BIN_EDGE, ScaledIntensityLeftTailThreshold: DEFAULT_LEFT_TAIL_THRESHOLD, ScaledIntensityRightTailThreshold: DEFAULT_RIGHT_TAIL_THRESHOLD, - FittingFunc: polynomial_7, + WavelengthFittingPolynomialDegree: WavelengthFittingPolynomialDegree(7), } """Default parameters for scaling data.""" From d39708abd7118a2082a82da07800635d2809e8dd Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Mon, 10 Jun 2024 06:40:31 +0200 Subject: [PATCH 177/403] Update workflow example. --- packages/essnmx/docs/examples/scaling_workflow.ipynb | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index 149dfe09..71d98944 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -73,7 +73,6 @@ " ReferenceWavelength,\n", " ScaledIntensityLeftTailThreshold,\n", " ScaledIntensityRightTailThreshold,\n", - " InitialGuess,\n", ")\n", "\n", "pl = sl.Pipeline(\n", @@ -90,7 +89,6 @@ " ScaledIntensityRightTailThreshold: sc.scalar(\n", " 4.0, # Decrease it to remove more outliers\n", " ),\n", - " InitialGuess: {f\"param{i+1}\": 1 / (sc.scalar(1, unit=\"angstrom\") ** i) for i in range(7)},\n", " **mtz_io_params,\n", " **scaling_params,\n", " },\n", From 4b6f5fd527229956ecf49f28ab4ba3eb333c30ce Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Mon, 3 Jun 2024 14:34:20 +0200 Subject: [PATCH 178/403] refactor: hardcode dim names --- packages/essnmx/src/ess/nmx/const.py | 5 ----- packages/essnmx/src/ess/nmx/mcstas_loader.py | 7 ++----- packages/essnmx/src/ess/nmx/mcstas_xml.py | 15 +++++++-------- packages/essnmx/src/ess/nmx/reduction.py | 7 +++---- packages/essnmx/tests/loader_test.py | 9 +++------ packages/essnmx/tests/workflow_test.py | 5 ++--- 6 files changed, 17 insertions(+), 31 deletions(-) delete mode 100644 packages/essnmx/src/ess/nmx/const.py diff --git a/packages/essnmx/src/ess/nmx/const.py b/packages/essnmx/src/ess/nmx/const.py deleted file mode 100644 index 12bb34ef..00000000 --- a/packages/essnmx/src/ess/nmx/const.py +++ /dev/null @@ -1,5 +0,0 @@ -DETECTOR_DIM = 'panel' -DETECTOR_SHAPE = (1280, 1280) -PIXEL_DIM = 'id' -TOF_DIM = 't' -DETECTOR_DIM = 'panel' diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 73e916ac..087fdeb2 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -6,7 +6,6 @@ import scipp as sc import scippnexus as snx -from .const import PIXEL_DIM, TOF_DIM from .mcstas_xml import McStasInstrument, read_mcstas_geometry_xml from .reduction import NMXData from .types import ( @@ -50,15 +49,13 @@ def raw_event_data( data = root[bank_name]["events"][()].rename_dims({'dim_0': 'event'}) return sc.DataArray( coords={ - PIXEL_DIM: sc.array( + 'id': sc.array( dims=['event'], values=data['dim_1', 4].values, dtype='int64', unit=None, ), - TOF_DIM: sc.array( - dims=['event'], values=data['dim_1', 5].values, unit='s' - ), + 't': sc.array(dims=['event'], values=data['dim_1', 5].values, unit='s'), }, data=sc.array( dims=['event'], values=data['dim_1', 0].values, unit='counts' diff --git a/packages/essnmx/src/ess/nmx/mcstas_xml.py b/packages/essnmx/src/ess/nmx/mcstas_xml.py index b42b1952..8dfb2291 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas_xml.py @@ -7,7 +7,6 @@ import scipp as sc -from .const import DETECTOR_DIM, PIXEL_DIM from .types import FilePath T = TypeVar('T') @@ -314,8 +313,8 @@ def _construct_pixel_ids(detector_descs: Tuple[DetectorDesc, ...]) -> sc.Variabl intervals = [ (desc.id_start, desc.id_start + desc.total_pixels) for desc in detector_descs ] - ids = [sc.arange(PIXEL_DIM, start, stop, unit=None) for start, stop in intervals] - return sc.concat(ids, PIXEL_DIM) + ids = [sc.arange('id', start, stop, unit=None) for start, stop in intervals] + return sc.concat(ids, 'id') def _pixel_positions( @@ -325,7 +324,7 @@ def _pixel_positions( Position of each pixel is relative to the position_offset. """ - pixel_idx = sc.arange(PIXEL_DIM, detector.total_pixels) + pixel_idx = sc.arange('id', detector.total_pixels) n_col = sc.scalar(detector.num_fast_pixels_per_row) pixel_n_slow = pixel_idx // n_col @@ -349,7 +348,7 @@ def _detector_pixel_positions( _pixel_positions(detector, sample.position_from_sample(detector.position)) for detector in detector_descs ] - return sc.concat(positions, DETECTOR_DIM) + return sc.concat(positions, 'panel') @dataclass @@ -391,9 +390,9 @@ def to_coords(self, *det_names: str) -> dict[str, sc.Variable]: return { 'pixel_id': _construct_pixel_ids(detectors), - 'fast_axis': sc.concat(fast_axes, DETECTOR_DIM), - 'slow_axis': sc.concat(slow_axes, DETECTOR_DIM), - 'origin_position': sc.concat(origins, DETECTOR_DIM), + 'fast_axis': sc.concat(fast_axes, 'panel'), + 'slow_axis': sc.concat(slow_axes, 'panel'), + 'origin_position': sc.concat(origins, 'panel'), 'sample_position': self.sample.position_from_sample(self.sample.position), 'source_position': self.sample.position_from_sample(self.source.position), 'sample_name': sc.scalar(self.sample.name), diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index d0d92a88..ccd31f75 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -8,7 +8,6 @@ import sciline import scipp as sc -from .const import DETECTOR_DIM, PIXEL_DIM, TOF_DIM from .mcstas_xml import McStasInstrument from .types import DetectorIndex, DetectorName, TimeBinSteps @@ -155,7 +154,7 @@ def _create_instrument_group(self, nx_entry: h5py.Group) -> h5py.Group: # Time of arrival bin edges self._create_dataset_from_var( root_entry=nx_detector_1, - var=self.counts.coords[TOF_DIM], + var=self.counts.coords['t'], name="t_bin", long_name="t_bin TOF (ms)", ) @@ -163,7 +162,7 @@ def _create_instrument_group(self, nx_entry: h5py.Group) -> h5py.Group: self._create_compressed_dataset( root_entry=nx_detector_1, name="pixel_id", - var=self.counts.coords[PIXEL_DIM], + var=self.counts.coords['id'], long_name="pixel ID", ) return nx_instrument @@ -234,7 +233,7 @@ def bin_time_of_arrival( """Bin time of arrival data into ``time_bin_step`` bins.""" nmx_data = list(nmx_data.values()) - nmx_data = sc.concat(nmx_data, DETECTOR_DIM) + nmx_data = sc.concat(nmx_data, 'panel') counts = nmx_data.pop('weights').hist(t=time_bin_step) new_coords = instrument.to_coords(*detector_name.values()) new_coords.pop('pixel_id') diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 527943bf..ccbc43ea 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -11,7 +11,6 @@ from scipp.testing import assert_allclose, assert_identical from ess.nmx import default_parameters -from ess.nmx.const import DETECTOR_DIM, DETECTOR_SHAPE from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample from ess.nmx.mcstas_loader import bank_names_to_detector_names from ess.nmx.mcstas_loader import providers as loader_providers @@ -51,7 +50,7 @@ def check_scalar_properties_mcstas_2(dg: NMXData): def check_nmxdata_properties(dg: NMXData, fast_axis, slow_axis) -> None: assert isinstance(dg, sc.DataGroup) - assert dg.shape == (DETECTOR_SHAPE[0] * DETECTOR_SHAPE[1], 1) + assert dg.shape == ((1280, 1280)[0] * (1280, 1280)[1], 1) # Check maximum value of weights. assert_allclose( dg.weights.max().data, @@ -59,10 +58,8 @@ def check_nmxdata_properties(dg: NMXData, fast_axis, slow_axis) -> None: atol=sc.scalar(1e-10, unit='counts'), rtol=sc.scalar(1e-8), ) - assert_allclose( - sc.squeeze(dg.fast_axis, DETECTOR_DIM), fast_axis, atol=sc.scalar(0.005) - ) - assert_identical(sc.squeeze(dg.slow_axis, DETECTOR_DIM), slow_axis) + assert_allclose(sc.squeeze(dg.fast_axis, 'panel'), fast_axis, atol=sc.scalar(0.005)) + assert_identical(sc.squeeze(dg.slow_axis, 'panel'), slow_axis) @pytest.mark.parametrize( diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index d29f29e5..df72ace4 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -5,7 +5,6 @@ import scipp as sc from ess.nmx import default_parameters -from ess.nmx.const import DETECTOR_SHAPE, PIXEL_DIM, TOF_DIM from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample from ess.nmx.mcstas_loader import providers as load_providers from ess.nmx.reduction import bin_time_of_arrival @@ -48,7 +47,7 @@ def test_pipeline_mcstas_loader(mcstas_workflow: sl.Pipeline) -> None: mcstas_workflow[DetectorIndex] = 0 nmx_data = mcstas_workflow.compute(NMXData) assert isinstance(nmx_data, sc.DataGroup) - assert nmx_data.sizes[PIXEL_DIM] == DETECTOR_SHAPE[0] * DETECTOR_SHAPE[1] + assert nmx_data.sizes['id'] == (1280, 1280)[0] * (1280, 1280)[1] def test_pipeline_mcstas_reduction(mcstas_workflow: sl.Pipeline) -> None: @@ -57,4 +56,4 @@ def test_pipeline_mcstas_reduction(mcstas_workflow: sl.Pipeline) -> None: nmx_reduced_data = mcstas_workflow.compute(NMXReducedData) assert isinstance(nmx_reduced_data, sc.DataGroup) - assert nmx_reduced_data.sizes[TOF_DIM] == 50 + assert nmx_reduced_data.sizes['t'] == 50 From cd89df218e858f90ba06440bbb87d06689f4b459 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Mon, 3 Jun 2024 14:39:40 +0200 Subject: [PATCH 179/403] refactor: use standard names for providers and default parameters --- packages/essnmx/docs/examples/scaling_workflow.ipynb | 2 +- packages/essnmx/src/ess/nmx/mtz_io.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index 71d98944..6158519a 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -64,7 +64,7 @@ "import sciline as sl\n", "import scipp as sc\n", "\n", - "from ess.nmx.mtz_io import mtz_io_providers, mtz_io_params\n", + "from ess.nmx.mtz_io import providers as mtz_io_providers, default_parameters as mtz_io_params\n", "from ess.nmx.mtz_io import MTZFileIndex, SpaceGroupDesc\n", "from ess.nmx.scaling import scaling_providers, scaling_params\n", "from ess.nmx.scaling import (\n", diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index 7e41d458..302cc6a7 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -322,7 +322,7 @@ def nmx_mtz_dataframe_to_scipp_dataarray( return NMXMtzDataArray(nmx_mtz_da[nmx_mtz_da.data > 0]) -mtz_io_providers = ( +providers = ( read_mtz_file, process_single_mtz_to_dataframe, get_space_group, @@ -333,7 +333,7 @@ def nmx_mtz_dataframe_to_scipp_dataarray( ) """The providers related to the MTZ IO.""" -mtz_io_params = { +default_parameters = { WavelengthColumnName: DEFAULT_WAVELENGTH_COLUMN_NAME, IntensityColumnName: DEFAULT_INTENSITY_COLUMN_NAME, StdDevColumnName: DEFAULT_STD_DEV_COLUMN_NAME, From fb010fb588c365fe102e704e1d8c00fb9ebaf053 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Mon, 3 Jun 2024 14:49:02 +0200 Subject: [PATCH 180/403] refactor: rename 'read' to 'load' --- packages/essnmx/src/ess/nmx/mcstas_loader.py | 26 +++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas_loader.py index 087fdeb2..5ee0e4ad 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas_loader.py @@ -25,16 +25,18 @@ def detector_name_from_index(index: DetectorIndex) -> DetectorName: return f'nD_Mantid_{index}' -def event_data_bank_name( +def load_event_data_bank_name( detector_name: DetectorName, file_path: FilePath ) -> DetectorBankPrefix: '''Finds the filename associated with a detector''' - for bank_name, det_names in read_bank_names_to_detector_names(file_path).items(): - if detector_name in det_names: - return bank_name.partition('.')[0] + with snx.File(file_path) as file: + description = file['entry1/instrument/description'][()] + for bank_name, det_names in bank_names_to_detector_names(description).items(): + if detector_name in det_names: + return bank_name.partition('.')[0] -def raw_event_data( +def load_raw_event_data( file_path: FilePath, bank_prefix: DetectorBankPrefix, detector_name: DetectorName, @@ -63,7 +65,7 @@ def raw_event_data( ).group(coords.pop('pixel_id')) -def crystal_rotation( +def load_crystal_rotation( file_path: FilePath, instrument: McStasInstrument ) -> CrystalRotation: """Retrieve crystal rotation from the file.""" @@ -115,12 +117,6 @@ def proton_charge_from_event_data(da: EventData) -> ProtonCharge: return ProtonCharge(sc.scalar(1 / 10_000, unit=None) * da.bins.size().sum().data) -def read_bank_names_to_detector_names(file_path: str) -> Dict[str, List[str]]: - with snx.File(file_path) as file: - description = file['entry1/instrument/description'][()] - return bank_names_to_detector_names(description) - - def bank_names_to_detector_names(description: str) -> Dict[str, List[str]]: """Associates event data names with the names of the detectors where the events were detected""" @@ -168,10 +164,10 @@ def load_mcstas( providers = ( read_mcstas_geometry_xml, detector_name_from_index, - event_data_bank_name, - raw_event_data, + load_event_data_bank_name, + load_raw_event_data, event_weights_from_probability, proton_charge_from_event_data, - crystal_rotation, + load_crystal_rotation, load_mcstas, ) From ccaa51de4d855561ecfd9303ee1656fd25ef3c21 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 11 Jun 2024 09:32:59 +0200 Subject: [PATCH 181/403] refactor: hardcode wavelength coord name in scaling module --- packages/essnmx/src/ess/nmx/mtz_io.py | 5 ++--- packages/essnmx/src/ess/nmx/scaling.py | 20 +++++++++----------- packages/essnmx/tests/scaling_test.py | 11 ++++------- 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index 302cc6a7..29944dea 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -25,7 +25,6 @@ WavelengthColumnName = NewType("WavelengthColumnName", str) """The name of the wavelength column in the mtz file.""" DEFAULT_WAVELENGTH_COLUMN_NAME = WavelengthColumnName("LAMBDA") -DEFAULT_WAVELENGTH_COORD_NAME = "wavelength" IntensityColumnName = NewType("IntensityColumnName", str) """The name of the intensity column in the mtz file.""" @@ -146,7 +145,7 @@ def _calculate_d(row: pd.Series) -> float: mtz_df["d"] = mtz_df.apply(_calculate_d, axis=1) mtz_df["resolution"] = (1 / mtz_df["d"]) ** 2 / 4 - mtz_df[DEFAULT_WAVELENGTH_COORD_NAME] = orig_df[wavelength_column_name] + mtz_df["wavelength"] = orig_df[wavelength_column_name] mtz_df[DEFAULT_INTENSITY_COLUMN_NAME] = orig_df[intensity_column_name] mtz_df[DEFAULT_STD_DEV_COLUMN_NAME] = orig_df[intensity_sig_col_name] # Keep other columns @@ -310,7 +309,7 @@ def nmx_mtz_dataframe_to_scipp_dataarray( # The result of `astype` will have `PyObject` as a dtype. ) # Add units - nmx_mtz_ds.coords[DEFAULT_WAVELENGTH_COORD_NAME].unit = sc.units.angstrom + nmx_mtz_ds.coords["wavelength"].unit = sc.units.angstrom for key in nmx_mtz_ds.keys(): nmx_mtz_ds[key].unit = sc.units.dimensionless diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index d7d88310..d8a93eb9 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -6,7 +6,7 @@ import scipp as sc -from .mtz_io import DEFAULT_WAVELENGTH_COORD_NAME, NMXMtzDataArray +from .mtz_io import NMXMtzDataArray # User defined or configurable types WavelengthBinSize = NewType("WavelengthBinSize", int) @@ -70,7 +70,7 @@ def get_wavelength_binned( Wavelength(LAMBDA) binning should always be done on the merged dataset. """ - wavelength_coord = mtz_da.coords[DEFAULT_WAVELENGTH_COORD_NAME] + wavelength_coord = mtz_da.coords["wavelength"] start = ( min_wavelength_bin_edge if min_wavelength_bin_edge is not None @@ -82,14 +82,14 @@ def get_wavelength_binned( else wavelength_coord.max() ) binning_var = sc.linspace( - dim=DEFAULT_WAVELENGTH_COORD_NAME, + dim="wavelength", start=start, stop=stop, num=wavelength_bin_size, unit=wavelength_coord.unit, ) - return WavelengthBinned(mtz_da.bin({DEFAULT_WAVELENGTH_COORD_NAME: binning_var})) + return WavelengthBinned(mtz_da.bin({"wavelength": binning_var})) def _is_bin_empty(binned: sc.DataArray, idx: int) -> bool: @@ -134,9 +134,7 @@ def get_reference_wavelength( """ if reference_wavelength is None: ref_idx = _get_middle_bin_idx(binned) - return SelectedReferenceWavelength( - binned.coords[DEFAULT_WAVELENGTH_COORD_NAME][ref_idx] - ) + return SelectedReferenceWavelength(binned.coords["wavelength"][ref_idx]) else: return SelectedReferenceWavelength(reference_wavelength) @@ -310,8 +308,8 @@ def average_roughly_scaled_intensities( # to represent the average wavelength of the bin # It is because the bin-edges are dropped while flattening the data # and the data is expected to be filtered after this step. - intensities.coords[DEFAULT_WAVELENGTH_COORD_NAME] = sc.midpoints( - intensities.coords[DEFAULT_WAVELENGTH_COORD_NAME], + intensities.coords["wavelength"] = sc.midpoints( + intensities.coords["wavelength"], ) return EstimatedScaledIntensities(intensities) @@ -461,14 +459,14 @@ def fit_wavelength_scale_factor_polynomial( WavelengthScaleFactors = NewType("WavelengthScaleFactors", sc.DataArray) -"""The scale factors of :attr:`~DEFAULT_WAVELENGTH_COORD_NAME`.""" +"""The scale factors of `"wavelength"`.""" def calculate_wavelength_scale_factor( fitting_result: FittingResult, reference_wavelength: SelectedReferenceWavelength, ) -> WavelengthScaleFactors: - """Calculate the scale factors along the :attr:`~DEFAULT_WAVELENGTH_COORD_NAME`.""" + """Calculate the scale factors along the `"wavelength"`.""" scaled_reference = fitting_result.fitting_func( reference_wavelength, **fitting_result.params diff --git a/packages/essnmx/tests/scaling_test.py b/packages/essnmx/tests/scaling_test.py index c8f1e130..cfcab70f 100644 --- a/packages/essnmx/tests/scaling_test.py +++ b/packages/essnmx/tests/scaling_test.py @@ -3,7 +3,6 @@ import pytest import scipp as sc -from ess.nmx.mtz_io import DEFAULT_WAVELENGTH_COORD_NAME from ess.nmx.scaling import ( ReferenceIntensities, estimate_scale_factor_per_hkl_asu_from_reference, @@ -17,9 +16,7 @@ def nmx_data_array() -> sc.DataArray: da = sc.DataArray( data=sc.array(dims=["row"], values=[1, 2, 3, 4, 5, 3.1, 3.2]), coords={ - DEFAULT_WAVELENGTH_COORD_NAME: sc.Variable( - dims=["row"], values=[1, 2, 3, 4, 5, 3, 3] - ), + "wavelength": sc.Variable(dims=["row"], values=[1, 2, 3, 4, 5, 3, 3]), "hkl_asu": sc.array( dims=["row"], values=[ @@ -43,11 +40,11 @@ def nmx_data_array() -> sc.DataArray: def test_get_reference_bin_middle(nmx_data_array: sc.DataArray) -> None: """Test the middle bin.""" - binned = nmx_data_array.bin({DEFAULT_WAVELENGTH_COORD_NAME: 6}) + binned = nmx_data_array.bin({"wavelength": 6}) reference_wavelength = get_reference_wavelength(binned) ref_bin = get_reference_intensities( - nmx_data_array.bin({DEFAULT_WAVELENGTH_COORD_NAME: 6}), + nmx_data_array.bin({"wavelength": 6}), reference_wavelength, ) selected_idx = (2, 5, 6) @@ -58,7 +55,7 @@ def test_get_reference_bin_middle(nmx_data_array: sc.DataArray) -> None: @pytest.fixture def reference_bin(nmx_data_array: sc.DataArray) -> ReferenceIntensities: - binned = nmx_data_array.bin({DEFAULT_WAVELENGTH_COORD_NAME: 6}) + binned = nmx_data_array.bin({"wavelength": 6}) reference_wavelength = get_reference_wavelength(binned) return get_reference_intensities( From b8019cc54551473e78f45dde4908afd359cbc591 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 11 Jun 2024 11:18:36 +0200 Subject: [PATCH 182/403] Use WavelengthBins instead of separate parameters for min, max and number of bins --- .../docs/examples/scaling_workflow.ipynb | 2 - packages/essnmx/src/ess/nmx/scaling.py | 52 +++---------------- 2 files changed, 7 insertions(+), 47 deletions(-) diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index 6158519a..f4d18107 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -68,7 +68,6 @@ "from ess.nmx.mtz_io import MTZFileIndex, SpaceGroupDesc\n", "from ess.nmx.scaling import scaling_providers, scaling_params\n", "from ess.nmx.scaling import (\n", - " WavelengthBinSize,\n", " FilteredEstimatedScaledIntensities,\n", " ReferenceWavelength,\n", " ScaledIntensityLeftTailThreshold,\n", @@ -79,7 +78,6 @@ " providers=mtz_io_providers + scaling_providers,\n", " params={\n", " SpaceGroupDesc: \"C 1 2 1\",\n", - " WavelengthBinSize: 250,\n", " ReferenceWavelength: sc.scalar(\n", " 3, unit=sc.units.angstrom\n", " ), # Remove it if you want to use the middle of the bin\n", diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index d8a93eb9..64c2085f 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -9,16 +9,8 @@ from .mtz_io import NMXMtzDataArray # User defined or configurable types -WavelengthBinSize = NewType("WavelengthBinSize", int) -"""The size of the wavelength(LAMBDA) bins.""" -MinWavelengthBinEdge = NewType("MinWavelengthBinEdge", sc.Variable) -"""The minimum edge of the wavelength(LAMBDA) bins.""" -DEFAULT_MIN_WAVELENGTH_BIN_EDGE = MinWavelengthBinEdge(sc.scalar(2.6, unit="angstrom")) -"""Default minimum edge of the wavelength(LAMBDA) bins.""" -MaxWavelengthBinEdge = NewType("MaxWavelengthBinEdge", sc.Variable) -"""The maximum edge of the wavelength(LAMBDA) bins.""" -DEFAULT_MAX_WAVELENGTH_BIN_EDGE = MaxWavelengthBinEdge(sc.scalar(3.6, unit="angstrom")) -"""Default maximum edge of the wavelength(LAMBDA) bins.""" +WavelengthBins = NewType("WavelengthBins", sc.Variable) +"""User configurable wavelength binning""" ReferenceWavelength = NewType("ReferenceWavelength", sc.Variable) """The wavelength to select reference intensities.""" @@ -43,9 +35,7 @@ def get_wavelength_binned( mtz_da: NMXMtzDataArray, - wavelength_bin_size: WavelengthBinSize, - min_wavelength_bin_edge: Optional[MinWavelengthBinEdge] = None, - max_wavelength_bin_edge: Optional[MaxWavelengthBinEdge] = None, + wavelength_bins: WavelengthBins, ) -> WavelengthBinned: """Bin the whole dataset by wavelength(LAMBDA). @@ -54,42 +44,15 @@ def get_wavelength_binned( mtz_da: The merged dataset. - wavelength_bin_size: - The size of the wavelength(LAMBDA) bins. - - min_wavelength_bin_edge: - The minimum edge of the wavelength(LAMBDA) bins. - Minimum value of the wavelength(LAMBDA) coordinate will be used if ``None``. - - max_wavelength_bin_edge: - The maximum edge of the wavelength(LAMBDA) bins. - Maximum value of the wavelength(LAMBDA) coordinate will be used if ``None``. + wavelength_bins: + The wavelength(LAMBDA) bins. Notes ----- Wavelength(LAMBDA) binning should always be done on the merged dataset. """ - wavelength_coord = mtz_da.coords["wavelength"] - start = ( - min_wavelength_bin_edge - if min_wavelength_bin_edge is not None - else wavelength_coord.min() - ) - stop = ( - max_wavelength_bin_edge - if max_wavelength_bin_edge is not None - else wavelength_coord.max() - ) - binning_var = sc.linspace( - dim="wavelength", - start=start, - stop=stop, - num=wavelength_bin_size, - unit=wavelength_coord.unit, - ) - - return WavelengthBinned(mtz_da.bin({"wavelength": binning_var})) + return WavelengthBinned(mtz_da.bin({"wavelength": wavelength_bins})) def _is_bin_empty(binned: sc.DataArray, idx: int) -> bool: @@ -489,8 +452,7 @@ def calculate_wavelength_scale_factor( """Providers for scaling data.""" scaling_params = { - MinWavelengthBinEdge: DEFAULT_MIN_WAVELENGTH_BIN_EDGE, - MaxWavelengthBinEdge: DEFAULT_MAX_WAVELENGTH_BIN_EDGE, + WavelengthBins: sc.linspace("wavelength", 2.6, 3.6, 250, unit="angstrom"), ScaledIntensityLeftTailThreshold: DEFAULT_LEFT_TAIL_THRESHOLD, ScaledIntensityRightTailThreshold: DEFAULT_RIGHT_TAIL_THRESHOLD, WavelengthFittingPolynomialDegree: WavelengthFittingPolynomialDegree(7), From 7df792857d47e161abfbe7b901edbfd73b2f214c Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 11 Jun 2024 14:51:08 +0200 Subject: [PATCH 183/403] refactor: add mcstas workflow, remove custom datastructures --- packages/essnmx/docs/examples/workflow.ipynb | 30 +-- .../essnmx/src/ess/nmx/mcstas/__init__.py | 25 ++ .../nmx/{mcstas_loader.py => mcstas/load.py} | 18 +- .../ess/nmx/{mcstas_xml.py => mcstas/xml.py} | 10 +- packages/essnmx/src/ess/nmx/nexus.py | 139 +++++++++++ packages/essnmx/src/ess/nmx/reduction.py | 228 +----------------- packages/essnmx/tests/exporter_test.py | 88 +++---- packages/essnmx/tests/loader_test.py | 34 +-- packages/essnmx/tests/workflow_test.py | 8 +- 9 files changed, 269 insertions(+), 311 deletions(-) create mode 100644 packages/essnmx/src/ess/nmx/mcstas/__init__.py rename packages/essnmx/src/ess/nmx/{mcstas_loader.py => mcstas/load.py} (94%) rename packages/essnmx/src/ess/nmx/{mcstas_xml.py => mcstas/xml.py} (98%) create mode 100644 packages/essnmx/src/ess/nmx/nexus.py diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index 32eee68c..fb744734 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -21,27 +21,28 @@ "source": [ "import sciline as sl\n", "\n", - "from ess.nmx.mcstas_loader import providers as loader_providers\n", - "from ess.nmx.mcstas_xml import read_mcstas_geometry_xml\n", + "from ess.nmx.mcstas import McStasWorkflow\n", "from ess.nmx.data import small_mcstas_3_sample, small_mcstas_2_sample\n", + "\n", "from ess.nmx.types import *\n", - "from ess.nmx.reduction import NMXData, NMXReducedData, bin_time_of_arrival\n", + "from ess.nmx.reduction import NMXData, NMXReducedData\n", + "from ess.nmx.nexus import export_as_nexus\n", "\n", - "pl = sl.Pipeline((*loader_providers, read_mcstas_geometry_xml, bin_time_of_arrival))\n", + "wf = McStasWorkflow()\n", "# Replace with the path to your own file\n", - "pl[FilePath] = small_mcstas_3_sample()\n", - "pl[MaximumProbability] = 10000\n", - "pl[TimeBinSteps] = 50\n", + "wf[FilePath] = small_mcstas_3_sample()\n", + "wf[MaximumProbability] = 10000\n", + "wf[TimeBinSteps] = 50\n", "# DetectorIndex selects what detector panels to include in the run\n", "# in this case we select all three panels.\n", - "pl.set_param_series(DetectorIndex, range(3))" + "wf.set_param_series(DetectorIndex, range(3))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "To see what the pipeline can produce, display it:" + "To see what the workflow can produce, display it:" ] }, { @@ -50,7 +51,7 @@ "metadata": {}, "outputs": [], "source": [ - "pl" + "wf" ] }, { @@ -66,8 +67,7 @@ "metadata": {}, "outputs": [], "source": [ - "nmx_workflow = pl.get(NMXReducedData)\n", - "nmx_workflow.visualize()" + "wf.visualize(NMXReducedData)" ] }, { @@ -84,7 +84,7 @@ "outputs": [], "source": [ "# Event data grouped by pixel id for each of the selected detectors\n", - "dg = nmx_workflow.compute(sl.Series[DetectorIndex, NMXData])\n", + "dg = wf.compute(sl.Series[DetectorIndex, NMXData])\n", "dg" ] }, @@ -95,7 +95,7 @@ "outputs": [], "source": [ "# Data from all selected detectors binned by panel, pixel and timeslice\n", - "binned_dg = nmx_workflow.compute(NMXReducedData)\n", + "binned_dg = wf.compute(NMXReducedData)\n", "binned_dg" ] }, @@ -116,7 +116,7 @@ "metadata": {}, "outputs": [], "source": [ - "#binned_dg.export_as_nexus('test.nxs')" + "export_as_nexus(binned_dg, 'test.nxs')" ] }, { diff --git a/packages/essnmx/src/ess/nmx/mcstas/__init__.py b/packages/essnmx/src/ess/nmx/mcstas/__init__.py new file mode 100644 index 00000000..5082ff9e --- /dev/null +++ b/packages/essnmx/src/ess/nmx/mcstas/__init__.py @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) + +# flake8: noqa +import importlib.metadata + +try: + __version__ = importlib.metadata.version(__package__ or __name__) +except importlib.metadata.PackageNotFoundError: + __version__ = "0.0.0" + +del importlib + + +def McStasWorkflow(): + import sciline as sl + + from ess.nmx.reduction import bin_time_of_arrival + + from .load import providers as loader_providers + from .xml import read_mcstas_geometry_xml + + return sl.Pipeline( + (*loader_providers, read_mcstas_geometry_xml, bin_time_of_arrival) + ) diff --git a/packages/essnmx/src/ess/nmx/mcstas_loader.py b/packages/essnmx/src/ess/nmx/mcstas/load.py similarity index 94% rename from packages/essnmx/src/ess/nmx/mcstas_loader.py rename to packages/essnmx/src/ess/nmx/mcstas/load.py index 5ee0e4ad..9df10385 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_loader.py +++ b/packages/essnmx/src/ess/nmx/mcstas/load.py @@ -6,9 +6,8 @@ import scipp as sc import scippnexus as snx -from .mcstas_xml import McStasInstrument, read_mcstas_geometry_xml -from .reduction import NMXData -from .types import ( +from ..reduction import NMXData +from ..types import ( CrystalRotation, DetectorBankPrefix, DetectorIndex, @@ -19,6 +18,7 @@ ProtonCharge, RawEventData, ) +from .xml import McStasInstrument, read_mcstas_geometry_xml def detector_name_from_index(index: DetectorIndex) -> DetectorName: @@ -154,10 +154,14 @@ def load_mcstas( coords = instrument.to_coords(detector_name) coords.pop('pixel_id') return NMXData( - weights=da, - proton_charge=proton_charge, - crystal_rotation=crystal_rotation, - **coords, + sc.DataGroup( + dict( + weights=da, + proton_charge=proton_charge, + crystal_rotation=crystal_rotation, + **coords, + ) + ) ) diff --git a/packages/essnmx/src/ess/nmx/mcstas_xml.py b/packages/essnmx/src/ess/nmx/mcstas/xml.py similarity index 98% rename from packages/essnmx/src/ess/nmx/mcstas_xml.py rename to packages/essnmx/src/ess/nmx/mcstas/xml.py index 8dfb2291..3aa43aae 100644 --- a/packages/essnmx/src/ess/nmx/mcstas_xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas/xml.py @@ -5,9 +5,12 @@ from types import MappingProxyType from typing import Iterable, Optional, Protocol, Tuple, TypeVar +import h5py import scipp as sc +from defusedxml.ElementTree import fromstring -from .types import FilePath +from ..rotation import axis_angle_to_quaternion, quaternion_to_matrix +from ..types import FilePath T = TypeVar('T') @@ -120,7 +123,6 @@ def _rotation_matrix_from_location( location: _XML, angle_unit: str = 'degree' ) -> sc.Variable: """Retrieve rotation matrix from location.""" - from .rotation import axis_angle_to_quaternion, quaternion_to_matrix attribs = find_attributes(location, 'axis-x', 'axis-y', 'axis-z', 'rot') x, y, z, w = axis_angle_to_quaternion( @@ -387,7 +389,6 @@ def to_coords(self, *det_names: str) -> dict[str, sc.Variable]: slow_axes = [det.slow_axis for det in detectors] fast_axes = [det.fast_axis for det in detectors] origins = [self.sample.position_from_sample(det.position) for det in detectors] - return { 'pixel_id': _construct_pixel_ids(detectors), 'fast_axis': sc.concat(fast_axes, 'panel'), @@ -402,9 +403,6 @@ def to_coords(self, *det_names: str) -> dict[str, sc.Variable]: def read_mcstas_geometry_xml(file_path: FilePath) -> McStasInstrument: """Retrieve geometry parameters from mcstas file""" - import h5py - from defusedxml.ElementTree import fromstring - instrument_xml_path = 'entry1/instrument/instrument_xml/data' with h5py.File(file_path) as file: tree = fromstring(file[instrument_xml_path][...][0]) diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/nexus.py new file mode 100644 index 00000000..1944800c --- /dev/null +++ b/packages/essnmx/src/ess/nmx/nexus.py @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +import io +import pathlib +from functools import partial +from typing import Optional, Union + +import h5py +import scipp as sc + + +def _create_dataset_from_var( + *, + root_entry: h5py.Group, + var: sc.Variable, + name: str, + long_name: Optional[str] = None, + compression: Optional[str] = None, + compression_opts: Optional[int] = None, +) -> h5py.Dataset: + compression_options = dict() + if compression is not None: + compression_options["compression"] = compression + if compression_opts is not None: + compression_options["compression_opts"] = compression_opts + + dataset = root_entry.create_dataset( + name, + data=var.values, + **compression_options, + ) + dataset.attrs["units"] = str(var.unit) + if long_name is not None: + dataset.attrs["long_name"] = long_name + return dataset + + +_create_compressed_dataset = partial( + _create_dataset_from_var, + compression="gzip", + compression_opts=4, +) + + +def _create_root_data_entry(file_obj: h5py.File) -> h5py.Group: + nx_entry = file_obj.create_group("NMX_data") + nx_entry.attrs["NX_class"] = "NXentry" + nx_entry.attrs["default"] = "data" + nx_entry.attrs["name"] = "NMX" + nx_entry["name"] = "NMX" + nx_entry["definition"] = "TOFRAW" + return nx_entry + + +def _create_sample_group(data: sc.DataGroup, nx_entry: h5py.Group) -> h5py.Group: + nx_sample = nx_entry.create_group("NXsample") + nx_sample["name"] = data['sample_name'].value + _create_dataset_from_var( + root_entry=nx_sample, + var=data['crystal_rotation'], + name='crystal_rotation', + long_name='crystal rotation in Phi (XYZ)', + ) + return nx_sample + + +def _create_instrument_group(data: sc.DataGroup, nx_entry: h5py.Group) -> h5py.Group: + nx_instrument = nx_entry.create_group("NXinstrument") + nx_instrument.create_dataset("proton_charge", data=data['proton_charge'].values) + + nx_detector_1 = nx_instrument.create_group("detector_1") + # Detector counts + _create_compressed_dataset( + root_entry=nx_detector_1, + name="counts", + var=data['counts'], + ) + # Time of arrival bin edges + _create_dataset_from_var( + root_entry=nx_detector_1, + var=data['counts'].coords['t'], + name="t_bin", + long_name="t_bin TOF (ms)", + ) + # Pixel IDs + _create_compressed_dataset( + root_entry=nx_detector_1, + name="pixel_id", + var=data['counts'].coords['id'], + long_name="pixel ID", + ) + return nx_instrument + + +def _create_detector_group(data: sc.DataGroup, nx_entry: h5py.Group) -> h5py.Group: + nx_detector = nx_entry.create_group("NXdetector") + # Position of the first pixel (lowest ID) in the detector + _create_compressed_dataset( + root_entry=nx_detector, + name="origin", + var=data['origin_position'], + ) + # Fast axis, along where the pixel ID increases by 1 + _create_dataset_from_var( + root_entry=nx_detector, var=data['fast_axis'], name="fast_axis" + ) + # Slow axis, along where the pixel ID increases + # by the number of pixels in the fast axis + _create_dataset_from_var( + root_entry=nx_detector, var=data['slow_axis'], name="slow_axis" + ) + return nx_detector + + +def _create_source_group(data: sc.DataGroup, nx_entry: h5py.Group) -> h5py.Group: + nx_source = nx_entry.create_group("NXsource") + nx_source["name"] = "European Spallation Source" + nx_source["short_name"] = "ESS" + nx_source["type"] = "Spallation Neutron Source" + nx_source["distance"] = sc.norm(data['source_position']).value + nx_source["probe"] = "neutron" + nx_source["target_material"] = "W" + return nx_source + + +def export_as_nexus( + data: sc.DataGroup, output_file: Union[str, pathlib.Path, io.BytesIO] +) -> None: + """Export the reduced data to a NeXus file. + + Currently exporting step is not expected to be part of sciline pipelines. + """ + with h5py.File(output_file, "w") as f: + f.attrs["default"] = "NMX_data" + nx_entry = _create_root_data_entry(f) + _create_sample_group(data, nx_entry) + _create_instrument_group(data, nx_entry) + _create_detector_group(data, nx_entry) + _create_source_group(data, nx_entry) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index ccd31f75..8b1f4f9b 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -1,227 +1,15 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) -import io -import pathlib -from typing import Optional, Union +from typing import NewType -import h5py import sciline import scipp as sc -from .mcstas_xml import McStasInstrument +from .mcstas.xml import McStasInstrument from .types import DetectorIndex, DetectorName, TimeBinSteps - -class _SharedFields(sc.DataGroup): - """All shared fields between NMXData and NMXReducedData. - - ``weights`` is only present in NMXData due to potential memory issues. - """ - - @property - def origin_position(self) -> sc.Variable: - """Position of the first pixel (lowest ID) in the detector. - - Relative position from the sample.""" - return self['origin_position'] - - @property - def crystal_rotation(self) -> sc.Variable: - """Rotation of the crystal.""" - - return self['crystal_rotation'] - - @property - def sample_name(self) -> sc.Variable: - return self['sample_name'] - - @property - def fast_axis(self) -> sc.Variable: - """Fast axis, along where the pixel ID increases by 1.""" - return self['fast_axis'] - - @property - def slow_axis(self) -> sc.Variable: - """Slow axis, along where the pixel ID increases by > 1. - - The pixel ID increases by the number of pixels in the fast axis.""" - return self['slow_axis'] - - @property - def proton_charge(self) -> sc.Variable: - """Accumulated number of protons during the measurement.""" - return self['proton_charge'] - - @property - def source_position(self) -> sc.Variable: - """Relative position of the source from the sample.""" - return self['source_position'] - - @property - def sample_position(self) -> sc.Variable: - """Relative position of the sample from the sample. (0, 0, 0)""" - return self['sample_position'] - - -class NMXData(_SharedFields, sc.DataGroup): - @property - def weights(self) -> sc.DataArray: - """Event data grouped by pixel id.""" - return self['weights'] - - -class NMXReducedData(_SharedFields, sc.DataGroup): - @property - def counts(self) -> sc.DataArray: - """Binned time of arrival data from flattened event data.""" - return self['counts'] - - def _create_dataset_from_var( - self, - *, - root_entry: h5py.Group, - var: sc.Variable, - name: str, - long_name: Optional[str] = None, - compression: Optional[str] = None, - compression_opts: Optional[int] = None, - ) -> h5py.Dataset: - compression_options = dict() - if compression is not None: - compression_options["compression"] = compression - if compression_opts is not None: - compression_options["compression_opts"] = compression_opts - - dataset = root_entry.create_dataset( - name, - data=var.values, - **compression_options, - ) - dataset.attrs["units"] = str(var.unit) - if long_name is not None: - dataset.attrs["long_name"] = long_name - return dataset - - def _create_compressed_dataset( - self, - *, - root_entry: h5py.Group, - name: str, - var: sc.Variable, - long_name: Optional[str] = None, - ) -> h5py.Dataset: - return self._create_dataset_from_var( - root_entry=root_entry, - var=var, - name=name, - long_name=long_name, - compression="gzip", - compression_opts=4, - ) - - def _create_root_data_entry(self, file_obj: h5py.File) -> h5py.Group: - nx_entry = file_obj.create_group("NMX_data") - nx_entry.attrs["NX_class"] = "NXentry" - nx_entry.attrs["default"] = "data" - nx_entry.attrs["name"] = "NMX" - nx_entry["name"] = "NMX" - nx_entry["definition"] = "TOFRAW" - return nx_entry - - def _create_sample_group(self, nx_entry: h5py.Group) -> h5py.Group: - nx_sample = nx_entry.create_group("NXsample") - nx_sample["name"] = self.sample_name.value - # Crystal rotation - self._create_dataset_from_var( - root_entry=nx_sample, - var=self.crystal_rotation, - name='crystal_rotation', - long_name='crystal rotation in Phi (XYZ)', - ) - return nx_sample - - def _create_instrument_group(self, nx_entry: h5py.Group) -> h5py.Group: - nx_instrument = nx_entry.create_group("NXinstrument") - nx_instrument.create_dataset("proton_charge", data=self.proton_charge.values) - - nx_detector_1 = nx_instrument.create_group("detector_1") - # Detector counts - self._create_compressed_dataset( - root_entry=nx_detector_1, - name="counts", - var=self.counts, - ) - # Time of arrival bin edges - self._create_dataset_from_var( - root_entry=nx_detector_1, - var=self.counts.coords['t'], - name="t_bin", - long_name="t_bin TOF (ms)", - ) - # Pixel IDs - self._create_compressed_dataset( - root_entry=nx_detector_1, - name="pixel_id", - var=self.counts.coords['id'], - long_name="pixel ID", - ) - return nx_instrument - - def _create_detector_group(self, nx_entry: h5py.Group) -> h5py.Group: - nx_detector = nx_entry.create_group("NXdetector") - # Position of the first pixel (lowest ID) in the detector - self._create_compressed_dataset( - root_entry=nx_detector, - name="origin", - var=self.origin_position, - ) - # Fast axis, along where the pixel ID increases by 1 - self._create_dataset_from_var( - root_entry=nx_detector, var=self.fast_axis, name="fast_axis" - ) - # Slow axis, along where the pixel ID increases - # by the number of pixels in the fast axis - self._create_dataset_from_var( - root_entry=nx_detector, var=self.slow_axis, name="slow_axis" - ) - return nx_detector - - def _create_source_group(self, nx_entry: h5py.Group) -> h5py.Group: - nx_source = nx_entry.create_group("NXsource") - nx_source["name"] = "European Spallation Source" - nx_source["short_name"] = "ESS" - nx_source["type"] = "Spallation Neutron Source" - nx_source["distance"] = sc.norm(self.source_position).value - nx_source["probe"] = "neutron" - nx_source["target_material"] = "W" - return nx_source - - def export_as_nexus( - self, output_file_base: Union[str, pathlib.Path, io.BytesIO] - ) -> None: - """Export the reduced data to a NeXus file. - - Currently exporting step is not expected to be part of sciline pipelines. - """ - if isinstance(output_file_base, (str, pathlib.Path)): - file_base = pathlib.Path(output_file_base) - if file_base.suffix not in (".h5", ".nxs"): - raise ValueError("Output file name must end with .h5 or .nxs") - else: - file_base = output_file_base - - with h5py.File(file_base, "w") as out_file: - out_file.attrs["default"] = "NMX_data" - # Root Data Entry - nx_entry = self._create_root_data_entry(out_file) - # Sample - self._create_sample_group(nx_entry) - # Instrument - self._create_instrument_group(nx_entry) - # Detector - self._create_detector_group(nx_entry) - # Source - self._create_source_group(nx_entry) +NMXData = NewType("NMXData", sc.DataGroup) +NMXReducedData = NewType("NMXData", sc.DataGroup) def bin_time_of_arrival( @@ -239,6 +27,10 @@ def bin_time_of_arrival( new_coords.pop('pixel_id') return NMXReducedData( - counts=counts, - **{**nmx_data, **new_coords}, + sc.DataGroup( + dict( + counts=counts, + **{**nmx_data, **new_coords}, + ) + ) ) diff --git a/packages/essnmx/tests/exporter_test.py b/packages/essnmx/tests/exporter_test.py index 927a8dad..4e772f98 100644 --- a/packages/essnmx/tests/exporter_test.py +++ b/packages/essnmx/tests/exporter_test.py @@ -2,16 +2,16 @@ # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) import io +import numpy as np import pytest +import scipp as sc +from ess.nmx.nexus import export_as_nexus from ess.nmx.reduction import NMXReducedData @pytest.fixture def reduced_data() -> NMXReducedData: - import numpy as np - import scipp as sc - rng = np.random.default_rng(42) id_list = sc.array(dims=['event'], values=rng.integers(0, 12, size=100)) t_list = sc.array(dims=['event'], values=rng.random(size=100, dtype=float)) @@ -25,28 +25,32 @@ def reduced_data() -> NMXReducedData: ) return NMXReducedData( - counts=counts, - proton_charge=sc.scalar(1.0, unit='counts'), - crystal_rotation=sc.vector(value=[0.0, 20.0, 0.0], unit='deg'), - fast_axis=sc.vectors( - dims=['panel'], - values=[[1.0, 0.0, 0.0], [1.0, 0.0, 0.0], [1.0, 0.0, 0.0]], - unit='m', - ), - slow_axis=sc.vectors( - dims=['panel'], - values=[[0.0, 1.0, 0.0], [0.0, 1.0, 0.0], [0.0, 1.0, 0.0]], - unit='m', - ), - origin_position=sc.vectors( - dims=['panel'], - values=[[-0.2, 0.0, 0.0], [0.0, 0.0, 0.0], [0.2, 0.0, 0.0]], - unit='m', - ), - sample_position=sc.vector(value=[0.0, 0.0, 0.0], unit='m'), - source_position=sc.vector(value=[-3, 0.0, -4], unit='m'), - sample_name=sc.scalar('Unit Test Sample'), - position=sc.zeros(dims=['panel', 'id'], shape=[3, 4], unit='m'), + sc.DataGroup( + dict( + counts=counts, + proton_charge=sc.scalar(1.0, unit='counts'), + crystal_rotation=sc.vector(value=[0.0, 20.0, 0.0], unit='deg'), + fast_axis=sc.vectors( + dims=['panel'], + values=[[1.0, 0.0, 0.0], [1.0, 0.0, 0.0], [1.0, 0.0, 0.0]], + unit='m', + ), + slow_axis=sc.vectors( + dims=['panel'], + values=[[0.0, 1.0, 0.0], [0.0, 1.0, 0.0], [0.0, 1.0, 0.0]], + unit='m', + ), + origin_position=sc.vectors( + dims=['panel'], + values=[[-0.2, 0.0, 0.0], [0.0, 0.0, 0.0], [0.2, 0.0, 0.0]], + unit='m', + ), + sample_position=sc.vector(value=[0.0, 0.0, 0.0], unit='m'), + source_position=sc.vector(value=[-3, 0.0, -4], unit='m'), + sample_name=sc.scalar('Unit Test Sample'), + position=sc.zeros(dims=['panel', 'id'], shape=[3, 4], unit='m'), + ) + ) ) @@ -66,7 +70,7 @@ def test_mcstas_reduction_export_to_bytestream(reduced_data: NMXReducedData) -> ] with io.BytesIO() as bio: - reduced_data.export_as_nexus(bio) + export_as_nexus(reduced_data, bio) with h5py.File(bio, 'r') as f: assert 'NMX_data' in f nmx_data: h5py.Group = f.require_group('NMX_data') @@ -74,39 +78,37 @@ def test_mcstas_reduction_export_to_bytestream(reduced_data: NMXReducedData) -> assert field in nmx_data nx_detector = nmx_data.require_group('NXdetector') - assert np.all(nx_detector['fast_axis'][()] == reduced_data.fast_axis.values) - assert np.all(nx_detector['slow_axis'][()] == reduced_data.slow_axis.values) assert np.all( - nx_detector['origin'][()] == reduced_data.origin_position.values + nx_detector['fast_axis'][()] == reduced_data['fast_axis'].values + ) + assert np.all( + nx_detector['slow_axis'][()] == reduced_data['slow_axis'].values + ) + assert np.all( + nx_detector['origin'][()] == reduced_data['origin_position'].values ) instrument_data = nmx_data.require_group('NXinstrument') assert ( - instrument_data['proton_charge'][()] == reduced_data.proton_charge.value + instrument_data['proton_charge'][()] + == reduced_data['proton_charge'].value ) det1_data = instrument_data.require_group('detector_1') - assert np.all(det1_data['counts'][()] == reduced_data.counts.values) + assert np.all(det1_data['counts'][()] == reduced_data['counts'].values) assert np.all( - det1_data['pixel_id'][()] == reduced_data.counts.coords['id'].values + det1_data['pixel_id'][()] == reduced_data['counts'].coords['id'].values ) assert np.all( - det1_data['t_bin'][()] == reduced_data.counts.coords['t'].values + det1_data['t_bin'][()] == reduced_data['counts'].coords['t'].values ) nx_sample = nmx_data.require_group('NXsample') sample_name: bytes = nx_sample['name'][()] - assert sample_name.decode() == reduced_data.sample_name.value + assert sample_name.decode() == reduced_data['sample_name'].value nx_source = nmx_data.require_group('NXsource') assert ( - nx_source['distance'][()] == sc.norm(reduced_data.source_position).value + nx_source['distance'][()] + == sc.norm(reduced_data['source_position']).value ) - - -def test_mcstas_reduction_export_wrong_extension_raises( - reduced_data: NMXReducedData, -) -> None: - """Test export method.""" - with pytest.raises(ValueError, match="Output file name must end with .h5 or .nxs"): - reduced_data.export_as_nexus('wrong_name.txt') diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index ccbc43ea..55b06d06 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -12,8 +12,8 @@ from ess.nmx import default_parameters from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample -from ess.nmx.mcstas_loader import bank_names_to_detector_names -from ess.nmx.mcstas_loader import providers as loader_providers +from ess.nmx.mcstas.load import bank_names_to_detector_names +from ess.nmx.mcstas.load import providers as loader_providers from ess.nmx.reduction import NMXData from ess.nmx.types import ( DetectorBankPrefix, @@ -37,15 +37,15 @@ def check_scalar_properties_mcstas_2(dg: NMXData): Expected numbers are hard-coded based on the sample file. """ assert_identical( - dg.proton_charge, + dg['proton_charge'], sc.scalar(1e-4 * dg['weights'].bins.size().sum().data.values, unit=None), ) - assert_identical(dg.crystal_rotation, sc.vector([20, 0, 90], unit='deg')) - assert_identical(dg.sample_position, sc.vector(value=[0, 0, 0], unit='m')) + assert_identical(dg['crystal_rotation'], sc.vector([20, 0, 90], unit='deg')) + assert_identical(dg['sample_position'], sc.vector(value=[0, 0, 0], unit='m')) assert_identical( - dg.source_position, sc.vector(value=[-0.53123, 0.0, -157.405], unit='m') + dg['source_position'], sc.vector(value=[-0.53123, 0.0, -157.405], unit='m') ) - assert dg.sample_name == sc.scalar("sampleMantid") + assert dg['sample_name'] == sc.scalar("sampleMantid") def check_nmxdata_properties(dg: NMXData, fast_axis, slow_axis) -> None: @@ -53,13 +53,15 @@ def check_nmxdata_properties(dg: NMXData, fast_axis, slow_axis) -> None: assert dg.shape == ((1280, 1280)[0] * (1280, 1280)[1], 1) # Check maximum value of weights. assert_allclose( - dg.weights.max().data, + dg['weights'].max().data, sc.scalar(default_parameters[MaximumProbability], unit='counts', dtype=float), atol=sc.scalar(1e-10, unit='counts'), rtol=sc.scalar(1e-8), ) - assert_allclose(sc.squeeze(dg.fast_axis, 'panel'), fast_axis, atol=sc.scalar(0.005)) - assert_identical(sc.squeeze(dg.slow_axis, 'panel'), slow_axis) + assert_allclose( + sc.squeeze(dg['fast_axis'], 'panel'), fast_axis, atol=sc.scalar(0.005) + ) + assert_identical(sc.squeeze(dg['slow_axis'], 'panel'), slow_axis) @pytest.mark.parametrize( @@ -101,15 +103,15 @@ def check_scalar_properties_mcstas_3(dg: NMXData): Expected numbers are hard-coded based on the sample file. """ assert_identical( - dg.proton_charge, + dg['proton_charge'], sc.scalar(1e-4 * dg['weights'].bins.size().sum().data.values, unit=None), ) - assert_identical(dg.crystal_rotation, sc.vector([0, 0, 0], unit='deg')) - assert_identical(dg.sample_position, sc.vector(value=[0, 0, 0], unit='m')) + assert_identical(dg['crystal_rotation'], sc.vector([0, 0, 0], unit='deg')) + assert_identical(dg['sample_position'], sc.vector(value=[0, 0, 0], unit='m')) assert_identical( - dg.source_position, sc.vector(value=[-0.53123, 0.0, -157.405], unit='m') + dg['source_position'], sc.vector(value=[-0.53123, 0.0, -157.405], unit='m') ) - assert dg.sample_name == sc.scalar("sampleMantid") + assert dg['sample_name'] == sc.scalar("sampleMantid") @pytest.mark.parametrize( @@ -141,7 +143,7 @@ def test_file_reader_mcstas3(detector_index, fast_axis, slow_axis) -> None: data_length = raw_data.sizes['dim_0'] check_scalar_properties_mcstas_3(dg) - assert dg.weights.bins.size().sum().value == data_length + assert dg['weights'].bins.size().sum().value == data_length check_nmxdata_properties(dg, sc.vector(fast_axis), sc.vector(slow_axis)) diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index df72ace4..70dfaef1 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -6,8 +6,8 @@ from ess.nmx import default_parameters from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample -from ess.nmx.mcstas_loader import providers as load_providers -from ess.nmx.reduction import bin_time_of_arrival +from ess.nmx.mcstas.load import providers as load_providers +from ess.nmx.reduction import NMXData, NMXReducedData, bin_time_of_arrival from ess.nmx.types import DetectorIndex, FilePath, TimeBinSteps @@ -42,8 +42,6 @@ def test_pipeline_builder(mcstas_workflow: sl.Pipeline, mcstas_file_path: str) - def test_pipeline_mcstas_loader(mcstas_workflow: sl.Pipeline) -> None: """Test if the loader graph is complete.""" - from ess.nmx.mcstas_loader import NMXData - mcstas_workflow[DetectorIndex] = 0 nmx_data = mcstas_workflow.compute(NMXData) assert isinstance(nmx_data, sc.DataGroup) @@ -52,8 +50,6 @@ def test_pipeline_mcstas_loader(mcstas_workflow: sl.Pipeline) -> None: def test_pipeline_mcstas_reduction(mcstas_workflow: sl.Pipeline) -> None: """Test if the loader graph is complete.""" - from ess.nmx.reduction import NMXReducedData - nmx_reduced_data = mcstas_workflow.compute(NMXReducedData) assert isinstance(nmx_reduced_data, sc.DataGroup) assert nmx_reduced_data.sizes['t'] == 50 From 6ab5ccd96f0690559b4ceae05c43bfdb3578816e Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 11 Jun 2024 14:54:02 +0200 Subject: [PATCH 184/403] fixup --- packages/essnmx/src/ess/nmx/mcstas/__init__.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas/__init__.py b/packages/essnmx/src/ess/nmx/mcstas/__init__.py index 5082ff9e..24397a5c 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/__init__.py +++ b/packages/essnmx/src/ess/nmx/mcstas/__init__.py @@ -1,16 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) -# flake8: noqa -import importlib.metadata - -try: - __version__ = importlib.metadata.version(__package__ or __name__) -except importlib.metadata.PackageNotFoundError: - __version__ = "0.0.0" - -del importlib - def McStasWorkflow(): import sciline as sl From da17ae2dba02c2707671f5b21cce3dac77030a78 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 12 Jun 2024 06:02:37 +0200 Subject: [PATCH 185/403] Begin refactor by removing references to sciline.Series --- packages/essnmx/src/ess/nmx/mtz_io.py | 15 +++---- packages/essnmx/src/ess/nmx/reduction.py | 13 +++--- packages/essnmx/tests/mtz_io_test.py | 57 +++++++++--------------- 3 files changed, 31 insertions(+), 54 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index 29944dea..c8f9eddb 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -6,7 +6,6 @@ import gemmi import numpy as np import pandas as pd -import sciline as sl import scipp as sc # Index types for param table. @@ -156,7 +155,7 @@ def _calculate_d(row: pd.Series) -> float: def get_space_group( - mtzs: sl.Series[MTZFileIndex, RawMtz], + mtzs: list[RawMtz], spacegroup_desc: Optional[SpaceGroupDesc] = None, ) -> SpaceGroup: """Retrieves spacegroup from file or uses parameter. @@ -168,7 +167,7 @@ def get_space_group( Parameters ---------- mtzs: - A series of raw mtz datasets. + A list of raw mtz datasets. spacegroup_desc: The space group description to use if not found in the mtz files. @@ -187,9 +186,7 @@ def get_space_group( """ space_groups = { - sgrp.short_name(): sgrp - for mtz in mtzs.values() - if (sgrp := mtz.spacegroup) is not None + sgrp.short_name(): sgrp for mtz in mtzs if (sgrp := mtz.spacegroup) is not None } if spacegroup_desc is not None: # Use the provided space group description return SpaceGroup(gemmi.SpaceGroup(spacegroup_desc)) @@ -209,12 +206,10 @@ def get_reciprocal_asu(spacegroup: SpaceGroup) -> ReciprocalAsymmetricUnit: return ReciprocalAsymmetricUnit(gemmi.ReciprocalAsu(spacegroup)) -def merge_mtz_dataframes( - mtz_dfs: sl.Series[MTZFileIndex, RawMtzDataFrame], -) -> MergedMtzDataFrame: +def merge_mtz_dataframes(*mtz_dfs: RawMtzDataFrame) -> MergedMtzDataFrame: """Merge multiple mtz dataframes into one.""" - return MergedMtzDataFrame(pd.concat(mtz_dfs.values(), ignore_index=True)) + return MergedMtzDataFrame(pd.concat(mtz_dfs, ignore_index=True)) def process_merged_mtz_dataframe( diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 8b1f4f9b..c4eaf360 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -2,28 +2,25 @@ # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) from typing import NewType -import sciline import scipp as sc from .mcstas.xml import McStasInstrument -from .types import DetectorIndex, DetectorName, TimeBinSteps +from .types import DetectorName, TimeBinSteps NMXData = NewType("NMXData", sc.DataGroup) -NMXReducedData = NewType("NMXData", sc.DataGroup) +NMXReducedData = NewType("NMXReducedData", sc.DataGroup) def bin_time_of_arrival( - nmx_data: sciline.Series[DetectorIndex, NMXData], - detector_name: sciline.Series[DetectorIndex, DetectorName], + nmx_data: NMXData, + detector_name: DetectorName, instrument: McStasInstrument, time_bin_step: TimeBinSteps, ) -> NMXReducedData: """Bin time of arrival data into ``time_bin_step`` bins.""" - nmx_data = list(nmx_data.values()) - nmx_data = sc.concat(nmx_data, 'panel') counts = nmx_data.pop('weights').hist(t=time_bin_step) - new_coords = instrument.to_coords(*detector_name.values()) + new_coords = instrument.to_coords(detector_name) new_coords.pop('pixel_id') return NMXReducedData( diff --git a/packages/essnmx/tests/mtz_io_test.py b/packages/essnmx/tests/mtz_io_test.py index 7c0487d3..afd0499f 100644 --- a/packages/essnmx/tests/mtz_io_test.py +++ b/packages/essnmx/tests/mtz_io_test.py @@ -4,7 +4,6 @@ import gemmi import pytest -import sciline as sl import scipp as sc from ess.nmx.data import get_small_mtz_samples @@ -69,46 +68,40 @@ def test_mtz_to_process_pandas_dataframe(gemmi_mtz_object: gemmi.Mtz) -> None: @pytest.fixture -def mtz_series() -> sl.Series[MTZFileIndex, RawMtz]: - return sl.Series( - row_dim=MTZFileIndex, - items={ - MTZFileIndex(i_file): read_mtz_file(MTZFilePath(file_path)) - for i_file, file_path in enumerate(get_small_mtz_samples()) - }, - ) +def mtz_list() -> list[RawMtz]: + return [ + read_mtz_file(MTZFilePath(file_path)) for file_path in get_small_mtz_samples() + ] -def test_get_space_group(mtz_series: sl.Series[MTZFileIndex, RawMtz]) -> None: +def test_get_space_group(mtz_list: list[RawMtz]) -> None: assert ( - get_space_group(mtz_series).short_name() == "C2" + get_space_group(mtz_list).short_name() == "C2" ) # Expected value in test files def test_get_space_group_with_spacegroup_desc( - mtz_series: sl.Series[MTZFileIndex, RawMtz], + mtz_list: list[RawMtz], ) -> None: - assert ( - get_space_group(mtz_series, DEFAULT_SPACE_GROUP_DESC).short_name() == "P212121" - ) + assert get_space_group(mtz_list, DEFAULT_SPACE_GROUP_DESC).short_name() == "P212121" @pytest.fixture def conflicting_mtz_series( - mtz_series: sl.Series[MTZFileIndex, RawMtz], -) -> sl.Series[MTZFileIndex, RawMtz]: - mtz_series[MTZFileIndex(0)].spacegroup = gemmi.SpaceGroup(DEFAULT_SPACE_GROUP_DESC) + mtz_list: list[RawMtz], +) -> list[RawMtz]: + mtz_list[MTZFileIndex(0)].spacegroup = gemmi.SpaceGroup(DEFAULT_SPACE_GROUP_DESC) # Make sure the space groups are different assert ( - mtz_series[MTZFileIndex(0)].spacegroup.short_name() - != mtz_series[MTZFileIndex(1)].spacegroup.short_name() + mtz_list[MTZFileIndex(0)].spacegroup.short_name() + != mtz_list[MTZFileIndex(1)].spacegroup.short_name() ) - return mtz_series + return mtz_list def test_get_space_group_conflict_raises( - conflicting_mtz_series: sl.Series[MTZFileIndex, RawMtz], + conflicting_mtz_series: list[RawMtz], ) -> None: reg = r"Multiple space groups found:.+P 21 21 21.+C 1 2 1" with pytest.raises(ValueError, match=reg): @@ -116,7 +109,7 @@ def test_get_space_group_conflict_raises( def test_get_space_conflict_but_desc_provided( - conflicting_mtz_series: sl.Series[MTZFileIndex, RawMtz], + conflicting_mtz_series: list[RawMtz], ) -> None: assert ( get_space_group(conflicting_mtz_series, DEFAULT_SPACE_GROUP_DESC).short_name() @@ -125,26 +118,18 @@ def test_get_space_conflict_but_desc_provided( @pytest.fixture -def merged_mtz_dataframe( - mtz_series: sl.Series[MTZFileIndex, RawMtz], -) -> MergedMtzDataFrame: +def merged_mtz_dataframe(mtz_list: list[RawMtz]) -> MergedMtzDataFrame: """Tests if the merged data frame has the expected columns.""" - reduced_mtz_series = sl.Series( - row_dim=MTZFileIndex, - items={ - i_file: process_single_mtz_to_dataframe(mtz) - for i_file, mtz in mtz_series.items() - }, - ) - return merge_mtz_dataframes(reduced_mtz_series) + reduced_mtz = [process_single_mtz_to_dataframe(mtz) for mtz in mtz_list] + return merge_mtz_dataframes(*reduced_mtz) @pytest.fixture def nmx_data_frame( - mtz_series: sl.Series[MTZFileIndex, RawMtz], + mtz_list: list[RawMtz], merged_mtz_dataframe: MergedMtzDataFrame, ) -> NMXMtzDataFrame: - space_gr = get_space_group(mtz_series) + space_gr = get_space_group(mtz_list) reciprocal_asu = get_reciprocal_asu(space_gr) return process_merged_mtz_dataframe( From 7eaef8d848a56bb00e246d6b5db5b4c76169b373 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 12 Jun 2024 06:20:29 +0200 Subject: [PATCH 186/403] Panel merge --- packages/essnmx/src/ess/nmx/mcstas/load.py | 10 ++++------ packages/essnmx/src/ess/nmx/reduction.py | 13 +++++-------- packages/essnmx/tests/workflow_test.py | 22 ++++++++++++++++------ 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas/load.py b/packages/essnmx/src/ess/nmx/mcstas/load.py index 9df10385..50b25efe 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/load.py +++ b/packages/essnmx/src/ess/nmx/mcstas/load.py @@ -155,12 +155,10 @@ def load_mcstas( coords.pop('pixel_id') return NMXData( sc.DataGroup( - dict( - weights=da, - proton_charge=proton_charge, - crystal_rotation=crystal_rotation, - **coords, - ) + weights=da, + proton_charge=proton_charge, + crystal_rotation=crystal_rotation, + **coords, ) ) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index c4eaf360..f5443980 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -23,11 +23,8 @@ def bin_time_of_arrival( new_coords = instrument.to_coords(detector_name) new_coords.pop('pixel_id') - return NMXReducedData( - sc.DataGroup( - dict( - counts=counts, - **{**nmx_data, **new_coords}, - ) - ) - ) + return NMXReducedData(sc.DataGroup(counts=counts, **{**nmx_data, **new_coords})) + + +def merge_panels(*panel: NMXReducedData) -> NMXReducedData: + return NMXReducedData(sc.concat(panel, 'panel')) diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index 70dfaef1..cf8e14d0 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +import pandas as pd import pytest import sciline as sl import scipp as sc @@ -7,7 +8,7 @@ from ess.nmx import default_parameters from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample from ess.nmx.mcstas.load import providers as load_providers -from ess.nmx.reduction import NMXData, NMXReducedData, bin_time_of_arrival +from ess.nmx.reduction import NMXData, NMXReducedData, bin_time_of_arrival, merge_panels from ess.nmx.types import DetectorIndex, FilePath, TimeBinSteps @@ -24,7 +25,7 @@ def mcstas_file_path( @pytest.fixture def mcstas_workflow(mcstas_file_path: str) -> sl.Pipeline: - pl = sl.Pipeline( + return sl.Pipeline( [*load_providers, bin_time_of_arrival], params={ FilePath: mcstas_file_path, @@ -32,7 +33,16 @@ def mcstas_workflow(mcstas_file_path: str) -> sl.Pipeline: **default_parameters, }, ) - pl.set_param_series(DetectorIndex, range(3)) + + +@pytest.fixture +def multi_bank_mcstas_workflow(mcstas_workflow: sl.Pipeline) -> sl.Pipeline: + pl = mcstas_workflow.copy() + pl[NMXReducedData] = ( + pl[NMXReducedData] + .map(pd.DataFrame({DetectorIndex: range(3)}).rename_axis('panel')) + .reduce(index='panel', func=merge_panels) + ) return pl @@ -45,11 +55,11 @@ def test_pipeline_mcstas_loader(mcstas_workflow: sl.Pipeline) -> None: mcstas_workflow[DetectorIndex] = 0 nmx_data = mcstas_workflow.compute(NMXData) assert isinstance(nmx_data, sc.DataGroup) - assert nmx_data.sizes['id'] == (1280, 1280)[0] * (1280, 1280)[1] + assert nmx_data.sizes['id'] == 1280 * 1280 -def test_pipeline_mcstas_reduction(mcstas_workflow: sl.Pipeline) -> None: +def test_pipeline_mcstas_reduction(multi_bank_mcstas_workflow: sl.Pipeline) -> None: """Test if the loader graph is complete.""" - nmx_reduced_data = mcstas_workflow.compute(NMXReducedData) + nmx_reduced_data = multi_bank_mcstas_workflow.compute(NMXReducedData) assert isinstance(nmx_reduced_data, sc.DataGroup) assert nmx_reduced_data.sizes['t'] == 50 From 579d8dc0165e5edc21073536714637f61ba4d179 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 12 Jun 2024 07:04:03 +0200 Subject: [PATCH 187/403] Update docs scaling workflow --- .../docs/examples/scaling_workflow.ipynb | 28 ++-- packages/essnmx/src/ess/nmx/mtz_io.py | 123 ++++++++++-------- packages/essnmx/src/ess/nmx/scaling.py | 6 +- packages/essnmx/tests/mtz_io_test.py | 6 +- 4 files changed, 91 insertions(+), 72 deletions(-) diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index f4d18107..6b6a691c 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -61,11 +61,12 @@ "metadata": {}, "outputs": [], "source": [ + "import pandas as pd\n", "import sciline as sl\n", "import scipp as sc\n", "\n", "from ess.nmx.mtz_io import providers as mtz_io_providers, default_parameters as mtz_io_params\n", - "from ess.nmx.mtz_io import MTZFileIndex, SpaceGroupDesc\n", + "from ess.nmx.mtz_io import SpaceGroupDesc, SpaceGroup, get_unique_space_group, merge_mtz_dataframes, MtzDataFrame\n", "from ess.nmx.scaling import scaling_providers, scaling_params\n", "from ess.nmx.scaling import (\n", " FilteredEstimatedScaledIntensities,\n", @@ -91,15 +92,20 @@ " **scaling_params,\n", " },\n", ")\n", - "\n", - "\n", - "file_path_table = sl.ParamTable(\n", - " row_dim=MTZFileIndex,\n", - " columns={MTZFilePath: get_small_random_mtz_samples()},\n", - ")\n", - "\n", - "pl.set_param_table(file_path_table)\n", - "pl" + "pl\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "file_paths = pd.DataFrame({MTZFilePath: get_small_random_mtz_samples()}).rename_axis(\"mtzfile\")\n", + "mapped = pl.map(file_paths)\n", + "pl[SpaceGroup] = mapped[SpaceGroup|None].reduce(index='mtzfile', func=get_unique_space_group)\n", + "pl[MtzDataFrame] = mapped[MtzDataFrame].reduce(index='mtzfile', func=merge_mtz_dataframes)" ] }, { @@ -374,7 +380,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index c8f9eddb..81bb10dd 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) import pathlib -from typing import NewType, Optional +from typing import NewType import gemmi import numpy as np @@ -17,6 +17,7 @@ """Path to the mtz file""" SpaceGroupDesc = NewType("SpaceGroupDesc", str) """The space group description. e.g. 'P 21 21 21'""" +# TODO unused, set as workflow default? DEFAULT_SPACE_GROUP_DESC = SpaceGroupDesc("P 21 21 21") """The default space group description to use if not found in the mtz files.""" @@ -36,14 +37,12 @@ # Computed types RawMtz = NewType("RawMtz", gemmi.Mtz) """The mtz file as a gemmi object""" -RawMtzDataFrame = NewType("RawMtzDataFrame", pd.DataFrame) +MtzDataFrame = NewType("MtzDataFrame", pd.DataFrame) """The raw mtz dataframe.""" SpaceGroup = NewType("SpaceGroup", gemmi.SpaceGroup) """The space group.""" ReciprocalAsymmetricUnit = NewType("ReciprocalAsymmetricUnit", gemmi.ReciprocalAsu) """The reciprocal asymmetric unit.""" -MergedMtzDataFrame = NewType("MergedMtzDataFrame", pd.DataFrame) -"""The merged mtz dataframe with derived columns.""" NMXMtzDataFrame = NewType("NMXMtzDataFrame", pd.DataFrame) """The processed mtz dataframe with derived columns.""" NMXMtzDataArray = NewType("NMXMtzDataArray", sc.DataArray) @@ -73,10 +72,8 @@ def mtz_to_pandas(mtz: gemmi.Mtz) -> pd.DataFrame: """ - return RawMtzDataFrame( - pd.DataFrame( # Recommended in the gemmi documentation. - data=np.array(mtz, copy=False), columns=mtz.column_labels() - ) + return pd.DataFrame( # Recommended in the gemmi documentation. + data=np.array(mtz, copy=False), columns=mtz.column_labels() ) @@ -85,7 +82,7 @@ def process_single_mtz_to_dataframe( wavelength_column_name: WavelengthColumnName = DEFAULT_WAVELENGTH_COLUMN_NAME, intensity_column_name: IntensityColumnName = DEFAULT_INTENSITY_COLUMN_NAME, intensity_sig_col_name: StdDevColumnName = DEFAULT_STD_DEV_COLUMN_NAME, -) -> RawMtzDataFrame: +) -> MtzDataFrame: """Select and derive columns from the original ``MtzDataFrame``. Parameters @@ -151,53 +148,69 @@ def _calculate_d(row: pd.Series) -> float: for column in [col for col in orig_df.columns if col not in mtz_df]: mtz_df[column] = orig_df[column] - return RawMtzDataFrame(mtz_df) + return MtzDataFrame(mtz_df) + + +def get_space_group_from_description(desc: SpaceGroupDesc) -> SpaceGroup: + """Retrieves spacegroup from parameter. + + Parameters + ---------- + desc: + The space group description to use if not found in the mtz files. + + Returns + ------- + : + The space group. + """ + return SpaceGroup(gemmi.SpaceGroup(desc)) -def get_space_group( - mtzs: list[RawMtz], - spacegroup_desc: Optional[SpaceGroupDesc] = None, -) -> SpaceGroup: - """Retrieves spacegroup from file or uses parameter. +def get_space_group_from_mtz(mtz: RawMtz) -> SpaceGroup | None: + """Retrieves spacegroup from file. - Manually provided space group description is prioritized over - space group descriptions found in the mtz files. Spacegroup is always expected in any MTZ files, but it may be missing. Parameters ---------- - mtzs: - A list of raw mtz datasets. + mtz: + Raw mtz dataset. - spacegroup_desc: - The space group description to use if not found in the mtz files. - If None, :attr:`~DEFAULT_SPACE_GROUP_DESC` is used. + Returns + ------- + : + The space group, or None if not found. + """ + if (sgrp := mtz.spacegroup) is not None: + return SpaceGroup(sgrp) + + +def get_unique_space_group(*spacegroups: SpaceGroup | None) -> SpaceGroup: + """Retrieves the unique space group from multiple space groups. + + Parameters + ---------- + spacegroups: + The space groups to check. Returns ------- - SpaceGroup - The space group. + : + The unique space group. Raises ------ - ValueError - If multiple or no space groups are found - but space group description is not provided. - + ValueError: + If there are multiple space groups. """ - space_groups = { - sgrp.short_name(): sgrp for mtz in mtzs if (sgrp := mtz.spacegroup) is not None - } - if spacegroup_desc is not None: # Use the provided space group description - return SpaceGroup(gemmi.SpaceGroup(spacegroup_desc)) - elif len(space_groups) > 1: - raise ValueError(f"Multiple space groups found: {space_groups}") - elif len(space_groups) == 1: - return SpaceGroup(space_groups.popitem()[1]) - else: - raise ValueError( - "No space group found and no space group description provided." - ) + spacegroups = [sgrp for sgrp in spacegroups if sgrp is not None] + if len(spacegroups) == 0: + raise ValueError("No space group found.") + first = spacegroups[0] + if all(sgrp == first for sgrp in spacegroups): + return first + raise ValueError(f"Multiple space groups found: {spacegroups}") def get_reciprocal_asu(spacegroup: SpaceGroup) -> ReciprocalAsymmetricUnit: @@ -206,36 +219,36 @@ def get_reciprocal_asu(spacegroup: SpaceGroup) -> ReciprocalAsymmetricUnit: return ReciprocalAsymmetricUnit(gemmi.ReciprocalAsu(spacegroup)) -def merge_mtz_dataframes(*mtz_dfs: RawMtzDataFrame) -> MergedMtzDataFrame: +def merge_mtz_dataframes(*mtz_dfs: MtzDataFrame) -> MtzDataFrame: """Merge multiple mtz dataframes into one.""" - return MergedMtzDataFrame(pd.concat(mtz_dfs, ignore_index=True)) + return MtzDataFrame(pd.concat(mtz_dfs, ignore_index=True)) -def process_merged_mtz_dataframe( +def process_mtz_dataframe( *, - merged_mtz_df: MergedMtzDataFrame, + mtz_df: MtzDataFrame, reciprocal_asu: ReciprocalAsymmetricUnit, sg: SpaceGroup, ) -> NMXMtzDataFrame: - """Modify/Add columns of the shallow copy of a merged mtz dataframes. + """Modify/Add columns of the shallow copy of a mtz dataframe. - This method must be called after merging multiple mtz dataframes. + This method must be called after merging multiple mtz dataframe. """ - merged_df = merged_mtz_df.copy(deep=False) + df = mtz_df.copy(deep=False) def _reciprocal_asu(row: pd.Series) -> list[int]: """Converts miller indices(HKL) to ASU indices.""" return reciprocal_asu.to_asu(row["hkl"], sg.operations())[0] - merged_df["hkl_asu"] = merged_df.apply(_reciprocal_asu, axis=1) + df["hkl_asu"] = df.apply(_reciprocal_asu, axis=1) # Unpack the indices for later. - merged_df[["H_ASU", "K_ASU", "L_ASU"]] = pd.DataFrame( - merged_df["hkl_asu"].to_list(), index=merged_df.index + df[["H_ASU", "K_ASU", "L_ASU"]] = pd.DataFrame( + df["hkl_asu"].to_list(), index=df.index ) - return NMXMtzDataFrame(merged_df) + return NMXMtzDataFrame(df) def nmx_mtz_dataframe_to_scipp_dataarray( @@ -319,10 +332,10 @@ def nmx_mtz_dataframe_to_scipp_dataarray( providers = ( read_mtz_file, process_single_mtz_to_dataframe, - get_space_group, + # get_space_group_from_description, + get_space_group_from_mtz, get_reciprocal_asu, - merge_mtz_dataframes, - process_merged_mtz_dataframe, + process_mtz_dataframe, nmx_mtz_dataframe_to_scipp_dataarray, ) """The providers related to the MTZ IO.""" diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 64c2085f..ec74682d 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -2,7 +2,7 @@ # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) from collections.abc import Callable, Mapping from dataclasses import dataclass -from typing import NewType, Optional, TypeVar +from typing import NewType, TypeVar import scipp as sc @@ -11,7 +11,7 @@ # User defined or configurable types WavelengthBins = NewType("WavelengthBins", sc.Variable) """User configurable wavelength binning""" -ReferenceWavelength = NewType("ReferenceWavelength", sc.Variable) +ReferenceWavelength = NewType("ReferenceWavelength", sc.Variable | None) """The wavelength to select reference intensities.""" # Computed types @@ -80,7 +80,7 @@ def _get_middle_bin_idx(binned: sc.DataArray) -> int: def get_reference_wavelength( binned: WavelengthBinned, - reference_wavelength: Optional[ReferenceWavelength] = None, + reference_wavelength: ReferenceWavelength, ) -> SelectedReferenceWavelength: """Select the reference wavelength. diff --git a/packages/essnmx/tests/mtz_io_test.py b/packages/essnmx/tests/mtz_io_test.py index afd0499f..cb97946a 100644 --- a/packages/essnmx/tests/mtz_io_test.py +++ b/packages/essnmx/tests/mtz_io_test.py @@ -20,7 +20,7 @@ merge_mtz_dataframes, mtz_to_pandas, nmx_mtz_dataframe_to_scipp_dataarray, - process_merged_mtz_dataframe, + process_mtz_dataframe, process_single_mtz_to_dataframe, read_mtz_file, ) @@ -132,8 +132,8 @@ def nmx_data_frame( space_gr = get_space_group(mtz_list) reciprocal_asu = get_reciprocal_asu(space_gr) - return process_merged_mtz_dataframe( - merged_mtz_df=merged_mtz_dataframe, + return process_mtz_dataframe( + mtz_df=merged_mtz_dataframe, reciprocal_asu=reciprocal_asu, sg=space_gr, ) From e7acd67d8c66e1856e63b39d10e70742c6b588f9 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 12 Jun 2024 07:16:39 +0200 Subject: [PATCH 188/403] Fix tests --- packages/essnmx/src/ess/nmx/scaling.py | 2 +- packages/essnmx/tests/mtz_io_test.py | 45 ++++++++++---------------- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index ec74682d..99a9ff97 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -80,7 +80,7 @@ def _get_middle_bin_idx(binned: sc.DataArray) -> int: def get_reference_wavelength( binned: WavelengthBinned, - reference_wavelength: ReferenceWavelength, + reference_wavelength: ReferenceWavelength = None, ) -> SelectedReferenceWavelength: """Select the reference wavelength. diff --git a/packages/essnmx/tests/mtz_io_test.py b/packages/essnmx/tests/mtz_io_test.py index cb97946a..b2736cf0 100644 --- a/packages/essnmx/tests/mtz_io_test.py +++ b/packages/essnmx/tests/mtz_io_test.py @@ -6,18 +6,17 @@ import pytest import scipp as sc +from ess.nmx import mtz_io from ess.nmx.data import get_small_mtz_samples from ess.nmx.mtz_io import DEFAULT_SPACE_GROUP_DESC # P 21 21 21 from ess.nmx.mtz_io import ( - MergedMtzDataFrame, + MtzDataFrame, MTZFileIndex, MTZFilePath, NMXMtzDataArray, NMXMtzDataFrame, RawMtz, get_reciprocal_asu, - get_space_group, - merge_mtz_dataframes, mtz_to_pandas, nmx_mtz_dataframe_to_scipp_dataarray, process_mtz_dataframe, @@ -74,16 +73,11 @@ def mtz_list() -> list[RawMtz]: ] -def test_get_space_group(mtz_list: list[RawMtz]) -> None: +def test_get_space_group_with_spacegroup_desc() -> None: assert ( - get_space_group(mtz_list).short_name() == "C2" - ) # Expected value in test files - - -def test_get_space_group_with_spacegroup_desc( - mtz_list: list[RawMtz], -) -> None: - assert get_space_group(mtz_list, DEFAULT_SPACE_GROUP_DESC).short_name() == "P212121" + mtz_io.get_space_group_from_description(DEFAULT_SPACE_GROUP_DESC).short_name() + == "P212121" + ) @pytest.fixture @@ -100,36 +94,31 @@ def conflicting_mtz_series( return mtz_list -def test_get_space_group_conflict_raises( +def test_get_unique_space_group_raises_on_conflict( conflicting_mtz_series: list[RawMtz], ) -> None: reg = r"Multiple space groups found:.+P 21 21 21.+C 1 2 1" + space_groups = [ + mtz_io.get_space_group_from_mtz(mtz) for mtz in conflicting_mtz_series + ] with pytest.raises(ValueError, match=reg): - get_space_group(conflicting_mtz_series) - - -def test_get_space_conflict_but_desc_provided( - conflicting_mtz_series: list[RawMtz], -) -> None: - assert ( - get_space_group(conflicting_mtz_series, DEFAULT_SPACE_GROUP_DESC).short_name() - == "P212121" - ) + mtz_io.get_unique_space_group(*space_groups) @pytest.fixture -def merged_mtz_dataframe(mtz_list: list[RawMtz]) -> MergedMtzDataFrame: +def merged_mtz_dataframe(mtz_list: list[RawMtz]) -> MtzDataFrame: """Tests if the merged data frame has the expected columns.""" reduced_mtz = [process_single_mtz_to_dataframe(mtz) for mtz in mtz_list] - return merge_mtz_dataframes(*reduced_mtz) + return mtz_io.merge_mtz_dataframes(*reduced_mtz) @pytest.fixture def nmx_data_frame( mtz_list: list[RawMtz], - merged_mtz_dataframe: MergedMtzDataFrame, + merged_mtz_dataframe: MtzDataFrame, ) -> NMXMtzDataFrame: - space_gr = get_space_group(mtz_list) + space_grs = [mtz_io.get_space_group_from_mtz(mtz) for mtz in mtz_list] + space_gr = mtz_io.get_unique_space_group(*space_grs) reciprocal_asu = get_reciprocal_asu(space_gr) return process_mtz_dataframe( @@ -140,7 +129,7 @@ def nmx_data_frame( def test_process_merged_mtz_dataframe( - merged_mtz_dataframe: MergedMtzDataFrame, + merged_mtz_dataframe: MtzDataFrame, nmx_data_frame: NMXMtzDataFrame, ) -> None: assert "hkl_asu" not in merged_mtz_dataframe.columns From 097505cfc500e1525e06a989c31a995a6aa3d517 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 12 Jun 2024 07:21:38 +0200 Subject: [PATCH 189/403] Default seems to break Sciline? --- packages/essnmx/src/ess/nmx/scaling.py | 2 +- packages/essnmx/tests/scaling_test.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 99a9ff97..ec74682d 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -80,7 +80,7 @@ def _get_middle_bin_idx(binned: sc.DataArray) -> int: def get_reference_wavelength( binned: WavelengthBinned, - reference_wavelength: ReferenceWavelength = None, + reference_wavelength: ReferenceWavelength, ) -> SelectedReferenceWavelength: """Select the reference wavelength. diff --git a/packages/essnmx/tests/scaling_test.py b/packages/essnmx/tests/scaling_test.py index cfcab70f..219603bc 100644 --- a/packages/essnmx/tests/scaling_test.py +++ b/packages/essnmx/tests/scaling_test.py @@ -41,7 +41,7 @@ def test_get_reference_bin_middle(nmx_data_array: sc.DataArray) -> None: """Test the middle bin.""" binned = nmx_data_array.bin({"wavelength": 6}) - reference_wavelength = get_reference_wavelength(binned) + reference_wavelength = get_reference_wavelength(binned, reference_wavelength=None) ref_bin = get_reference_intensities( nmx_data_array.bin({"wavelength": 6}), @@ -56,7 +56,7 @@ def test_get_reference_bin_middle(nmx_data_array: sc.DataArray) -> None: @pytest.fixture def reference_bin(nmx_data_array: sc.DataArray) -> ReferenceIntensities: binned = nmx_data_array.bin({"wavelength": 6}) - reference_wavelength = get_reference_wavelength(binned) + reference_wavelength = get_reference_wavelength(binned, reference_wavelength=None) return get_reference_intensities( binned, From 2f60fe164ec19466b358fe8ae7ea3f1c01f36826 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 12 Jun 2024 07:28:08 +0200 Subject: [PATCH 190/403] More fixes, not complete --- packages/essnmx/docs/examples/workflow.ipynb | 15 +++++++++++---- packages/essnmx/src/ess/nmx/reduction.py | 1 + 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index fb744734..d2f48ba8 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -19,13 +19,14 @@ "metadata": {}, "outputs": [], "source": [ + "import pandas as pd\n", "import sciline as sl\n", "\n", "from ess.nmx.mcstas import McStasWorkflow\n", "from ess.nmx.data import small_mcstas_3_sample, small_mcstas_2_sample\n", "\n", "from ess.nmx.types import *\n", - "from ess.nmx.reduction import NMXData, NMXReducedData\n", + "from ess.nmx.reduction import NMXData, NMXReducedData, merge_panels\n", "from ess.nmx.nexus import export_as_nexus\n", "\n", "wf = McStasWorkflow()\n", @@ -35,7 +36,11 @@ "wf[TimeBinSteps] = 50\n", "# DetectorIndex selects what detector panels to include in the run\n", "# in this case we select all three panels.\n", - "wf.set_param_series(DetectorIndex, range(3))" + "wf[NMXReducedData] = (\n", + " wf[NMXReducedData]\n", + " .map(pd.DataFrame({DetectorIndex: range(3)}).rename_axis('panel'))\n", + " .reduce(index='panel', func=merge_panels)\n", + ")\n" ] }, { @@ -83,8 +88,10 @@ "metadata": {}, "outputs": [], "source": [ + "from cyclebane.graph import NodeName, IndexValues\n", "# Event data grouped by pixel id for each of the selected detectors\n", - "dg = wf.compute(sl.Series[DetectorIndex, NMXData])\n", + "targets = [NodeName(NMXData, IndexValues(('panel',),(i,))) for i in range(3)]\n", + "dg = wf.compute(targets)\n", "dg" ] }, @@ -167,7 +174,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index f5443980..6a275a77 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -27,4 +27,5 @@ def bin_time_of_arrival( def merge_panels(*panel: NMXReducedData) -> NMXReducedData: + # TODO Is this the correct kind of reduce? return NMXReducedData(sc.concat(panel, 'panel')) From 7e7214feef799bb83ddf2a70f7189ed57dc2e3ee Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 12 Jun 2024 08:16:29 +0200 Subject: [PATCH 191/403] Fix merge --- packages/essnmx/src/ess/nmx/reduction.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 6a275a77..ecc29633 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -26,6 +26,20 @@ def bin_time_of_arrival( return NMXReducedData(sc.DataGroup(counts=counts, **{**nmx_data, **new_coords})) +def _concat_or_same( + obj: list[sc.Variable | sc.DataArray], dim: str +) -> sc.Variable | sc.DataArray: + first = obj[0] + if all(sc.identical(first, o) for o in obj): + return first + return sc.concat(obj, dim) + + def merge_panels(*panel: NMXReducedData) -> NMXReducedData: # TODO Is this the correct kind of reduce? - return NMXReducedData(sc.concat(panel, 'panel')) + keys = panel[0].keys() + if not all(p.keys() == keys for p in panel): + raise ValueError("All panels must have the same keys.") + return NMXReducedData( + {key: _concat_or_same([p[key] for p in panel], 'panel') for key in keys} + ) From 3f58d11f1fa65031053f33beb6c42cd8a34ea773 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 12 Jun 2024 14:01:48 +0200 Subject: [PATCH 192/403] Fix notebook --- packages/essnmx/docs/examples/workflow.ipynb | 50 +++++++++++++------- packages/essnmx/src/ess/nmx/reduction.py | 7 +-- 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index d2f48ba8..0c9cebc1 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -20,10 +20,9 @@ "outputs": [], "source": [ "import pandas as pd\n", - "import sciline as sl\n", "\n", "from ess.nmx.mcstas import McStasWorkflow\n", - "from ess.nmx.data import small_mcstas_3_sample, small_mcstas_2_sample\n", + "from ess.nmx.data import small_mcstas_3_sample\n", "\n", "from ess.nmx.types import *\n", "from ess.nmx.reduction import NMXData, NMXReducedData, merge_panels\n", @@ -33,14 +32,7 @@ "# Replace with the path to your own file\n", "wf[FilePath] = small_mcstas_3_sample()\n", "wf[MaximumProbability] = 10000\n", - "wf[TimeBinSteps] = 50\n", - "# DetectorIndex selects what detector panels to include in the run\n", - "# in this case we select all three panels.\n", - "wf[NMXReducedData] = (\n", - " wf[NMXReducedData]\n", - " .map(pd.DataFrame({DetectorIndex: range(3)}).rename_axis('panel'))\n", - " .reduce(index='panel', func=merge_panels)\n", - ")\n" + "wf[TimeBinSteps] = 50" ] }, { @@ -59,6 +51,28 @@ "wf" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We want to reduce all three panels, so we map the relevant part of the workflow over a list of the three panels:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# DetectorIndex selects what detector panels to include in the run\n", + "# in this case we select all three panels.\n", + "wf[NMXReducedData] = (\n", + " wf[NMXReducedData]\n", + " .map(pd.DataFrame({DetectorIndex: range(3)}).rename_axis(\"panel\"))\n", + " .reduce(index=\"panel\", func=merge_panels)\n", + ")" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -72,7 +86,7 @@ "metadata": {}, "outputs": [], "source": [ - "wf.visualize(NMXReducedData)" + "wf.visualize(NMXReducedData, graph_attr={\"rankdir\": \"TD\"}, compact=True)" ] }, { @@ -89,9 +103,10 @@ "outputs": [], "source": [ "from cyclebane.graph import NodeName, IndexValues\n", + "\n", "# Event data grouped by pixel id for each of the selected detectors\n", - "targets = [NodeName(NMXData, IndexValues(('panel',),(i,))) for i in range(3)]\n", - "dg = wf.compute(targets)\n", + "targets = [NodeName(NMXData, IndexValues((\"panel\",), (i,))) for i in range(3)]\n", + "dg = merge_panels(*wf.compute(targets).values())\n", "dg" ] }, @@ -123,7 +138,7 @@ "metadata": {}, "outputs": [], "source": [ - "export_as_nexus(binned_dg, 'test.nxs')" + "export_as_nexus(binned_dg, \"test.nxs\")" ] }, { @@ -149,11 +164,10 @@ "source": [ "import scippneutron as scn\n", "\n", - "da = dg[0]['weights']\n", - "da.coords['position'] = dg[0]['position']['panel', 0]\n", + "da = dg[\"weights\"]\n", + "da.coords[\"position\"] = dg[\"position\"]\n", "# Plot one out of 100 pixels to reduce size of docs output\n", - "view = scn.instrument_view(da['id', ::100].hist(), pixel_size=0.0075)\n", - "view.children[0].toolbar.cameraz()\n", + "view = scn.instrument_view(da[\"id\", ::100].hist(), pixel_size=0.0075)\n", "view" ] } diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index ecc29633..1770b88d 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -30,16 +30,17 @@ def _concat_or_same( obj: list[sc.Variable | sc.DataArray], dim: str ) -> sc.Variable | sc.DataArray: first = obj[0] - if all(sc.identical(first, o) for o in obj): + if all(dim not in o.dims and sc.identical(first, o) for o in obj): return first return sc.concat(obj, dim) def merge_panels(*panel: NMXReducedData) -> NMXReducedData: - # TODO Is this the correct kind of reduce? keys = panel[0].keys() if not all(p.keys() == keys for p in panel): raise ValueError("All panels must have the same keys.") return NMXReducedData( - {key: _concat_or_same([p[key] for p in panel], 'panel') for key in keys} + sc.DataGroup( + {key: _concat_or_same([p[key] for p in panel], 'panel') for key in keys} + ) ) From aad6e35d9252a8c19b317ed886fca42f898b34b6 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 12 Jun 2024 15:30:03 +0200 Subject: [PATCH 193/403] Remove superfluous aliases --- packages/essnmx/src/ess/nmx/mtz_io.py | 33 +++++++++++---------------- packages/essnmx/tests/mtz_io_test.py | 15 ++++++------ 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index 81bb10dd..d9e1ec66 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -35,23 +35,17 @@ DEFAULT_STD_DEV_COLUMN_NAME = StdDevColumnName("SIGI") # Computed types -RawMtz = NewType("RawMtz", gemmi.Mtz) -"""The mtz file as a gemmi object""" MtzDataFrame = NewType("MtzDataFrame", pd.DataFrame) """The raw mtz dataframe.""" -SpaceGroup = NewType("SpaceGroup", gemmi.SpaceGroup) -"""The space group.""" -ReciprocalAsymmetricUnit = NewType("ReciprocalAsymmetricUnit", gemmi.ReciprocalAsu) -"""The reciprocal asymmetric unit.""" NMXMtzDataFrame = NewType("NMXMtzDataFrame", pd.DataFrame) """The processed mtz dataframe with derived columns.""" NMXMtzDataArray = NewType("NMXMtzDataArray", sc.DataArray) -def read_mtz_file(file_path: MTZFilePath) -> RawMtz: +def read_mtz_file(file_path: MTZFilePath) -> gemmi.Mtz: """read mtz file""" - return RawMtz(gemmi.read_mtz_file(file_path.as_posix())) + return gemmi.read_mtz_file(file_path.as_posix()) def mtz_to_pandas(mtz: gemmi.Mtz) -> pd.DataFrame: @@ -78,7 +72,7 @@ def mtz_to_pandas(mtz: gemmi.Mtz) -> pd.DataFrame: def process_single_mtz_to_dataframe( - mtz: RawMtz, + mtz: gemmi.Mtz, wavelength_column_name: WavelengthColumnName = DEFAULT_WAVELENGTH_COLUMN_NAME, intensity_column_name: IntensityColumnName = DEFAULT_INTENSITY_COLUMN_NAME, intensity_sig_col_name: StdDevColumnName = DEFAULT_STD_DEV_COLUMN_NAME, @@ -151,7 +145,7 @@ def _calculate_d(row: pd.Series) -> float: return MtzDataFrame(mtz_df) -def get_space_group_from_description(desc: SpaceGroupDesc) -> SpaceGroup: +def get_space_group_from_description(desc: SpaceGroupDesc) -> gemmi.SpaceGroup: """Retrieves spacegroup from parameter. Parameters @@ -164,10 +158,10 @@ def get_space_group_from_description(desc: SpaceGroupDesc) -> SpaceGroup: : The space group. """ - return SpaceGroup(gemmi.SpaceGroup(desc)) + return gemmi.SpaceGroup(desc) -def get_space_group_from_mtz(mtz: RawMtz) -> SpaceGroup | None: +def get_space_group_from_mtz(mtz: gemmi.Mtz) -> gemmi.SpaceGroup | None: """Retrieves spacegroup from file. Spacegroup is always expected in any MTZ files, but it may be missing. @@ -182,11 +176,10 @@ def get_space_group_from_mtz(mtz: RawMtz) -> SpaceGroup | None: : The space group, or None if not found. """ - if (sgrp := mtz.spacegroup) is not None: - return SpaceGroup(sgrp) + return mtz.spacegroup -def get_unique_space_group(*spacegroups: SpaceGroup | None) -> SpaceGroup: +def get_unique_space_group(*spacegroups: gemmi.SpaceGroup | None) -> gemmi.SpaceGroup: """Retrieves the unique space group from multiple space groups. Parameters @@ -213,10 +206,10 @@ def get_unique_space_group(*spacegroups: SpaceGroup | None) -> SpaceGroup: raise ValueError(f"Multiple space groups found: {spacegroups}") -def get_reciprocal_asu(spacegroup: SpaceGroup) -> ReciprocalAsymmetricUnit: +def get_reciprocal_asu(spacegroup: gemmi.SpaceGroup) -> gemmi.ReciprocalAsu: """Returns the reciprocal asymmetric unit from the space group.""" - return ReciprocalAsymmetricUnit(gemmi.ReciprocalAsu(spacegroup)) + return gemmi.ReciprocalAsu(spacegroup) def merge_mtz_dataframes(*mtz_dfs: MtzDataFrame) -> MtzDataFrame: @@ -228,8 +221,8 @@ def merge_mtz_dataframes(*mtz_dfs: MtzDataFrame) -> MtzDataFrame: def process_mtz_dataframe( *, mtz_df: MtzDataFrame, - reciprocal_asu: ReciprocalAsymmetricUnit, - sg: SpaceGroup, + reciprocal_asu: gemmi.ReciprocalAsu, + sg: gemmi.SpaceGroup, ) -> NMXMtzDataFrame: """Modify/Add columns of the shallow copy of a mtz dataframe. @@ -312,7 +305,7 @@ def nmx_mtz_dataframe_to_scipp_dataarray( for indices_name in ("hkl", "hkl_asu"): nmx_mtz_ds.coords[indices_name] = sc.array( dims=nmx_mtz_ds.coords[indices_name].dims, - values=nmx_mtz_df[indices_name].astype(str).tolist() + values=nmx_mtz_df[indices_name].astype(str).tolist(), # `astype`` is not enough to convert the dtype to string. # The result of `astype` will have `PyObject` as a dtype. ) diff --git a/packages/essnmx/tests/mtz_io_test.py b/packages/essnmx/tests/mtz_io_test.py index b2736cf0..b882ce69 100644 --- a/packages/essnmx/tests/mtz_io_test.py +++ b/packages/essnmx/tests/mtz_io_test.py @@ -15,7 +15,6 @@ MTZFilePath, NMXMtzDataArray, NMXMtzDataFrame, - RawMtz, get_reciprocal_asu, mtz_to_pandas, nmx_mtz_dataframe_to_scipp_dataarray, @@ -55,7 +54,7 @@ def test_mtz_to_pandas_dataframe(gemmi_mtz_object: gemmi.Mtz) -> None: def test_mtz_to_process_pandas_dataframe(gemmi_mtz_object: gemmi.Mtz) -> None: - df = process_single_mtz_to_dataframe(RawMtz(gemmi_mtz_object)) + df = process_single_mtz_to_dataframe(gemmi_mtz_object) for expected_colum in ["hkl", "d", "resolution", *"HKL", "wavelength", "I", "SIGI"]: assert expected_colum in df.columns @@ -67,7 +66,7 @@ def test_mtz_to_process_pandas_dataframe(gemmi_mtz_object: gemmi.Mtz) -> None: @pytest.fixture -def mtz_list() -> list[RawMtz]: +def mtz_list() -> list[gemmi.Mtz]: return [ read_mtz_file(MTZFilePath(file_path)) for file_path in get_small_mtz_samples() ] @@ -82,8 +81,8 @@ def test_get_space_group_with_spacegroup_desc() -> None: @pytest.fixture def conflicting_mtz_series( - mtz_list: list[RawMtz], -) -> list[RawMtz]: + mtz_list: list[gemmi.Mtz], +) -> list[gemmi.Mtz]: mtz_list[MTZFileIndex(0)].spacegroup = gemmi.SpaceGroup(DEFAULT_SPACE_GROUP_DESC) # Make sure the space groups are different assert ( @@ -95,7 +94,7 @@ def conflicting_mtz_series( def test_get_unique_space_group_raises_on_conflict( - conflicting_mtz_series: list[RawMtz], + conflicting_mtz_series: list[gemmi.Mtz], ) -> None: reg = r"Multiple space groups found:.+P 21 21 21.+C 1 2 1" space_groups = [ @@ -106,7 +105,7 @@ def test_get_unique_space_group_raises_on_conflict( @pytest.fixture -def merged_mtz_dataframe(mtz_list: list[RawMtz]) -> MtzDataFrame: +def merged_mtz_dataframe(mtz_list: list[gemmi.Mtz]) -> MtzDataFrame: """Tests if the merged data frame has the expected columns.""" reduced_mtz = [process_single_mtz_to_dataframe(mtz) for mtz in mtz_list] return mtz_io.merge_mtz_dataframes(*reduced_mtz) @@ -114,7 +113,7 @@ def merged_mtz_dataframe(mtz_list: list[RawMtz]) -> MtzDataFrame: @pytest.fixture def nmx_data_frame( - mtz_list: list[RawMtz], + mtz_list: list[gemmi.Mtz], merged_mtz_dataframe: MtzDataFrame, ) -> NMXMtzDataFrame: space_grs = [mtz_io.get_space_group_from_mtz(mtz) for mtz in mtz_list] From 0d4db4f00545ff7f7a3a29080dbc2e5b6da29805 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 12 Jun 2024 15:36:05 +0200 Subject: [PATCH 194/403] Remove todo --- packages/essnmx/src/ess/nmx/mtz_io.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index d9e1ec66..fbfb50e8 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -17,7 +17,6 @@ """Path to the mtz file""" SpaceGroupDesc = NewType("SpaceGroupDesc", str) """The space group description. e.g. 'P 21 21 21'""" -# TODO unused, set as workflow default? DEFAULT_SPACE_GROUP_DESC = SpaceGroupDesc("P 21 21 21") """The default space group description to use if not found in the mtz files.""" From 044b0533c7f3e1994dcd2cc18fc19be206f47268 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 12 Jun 2024 15:40:19 +0200 Subject: [PATCH 195/403] Add note --- packages/essnmx/src/ess/nmx/reduction.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 1770b88d..fc2fb7dd 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -30,6 +30,10 @@ def _concat_or_same( obj: list[sc.Variable | sc.DataArray], dim: str ) -> sc.Variable | sc.DataArray: first = obj[0] + # instrument.to_coords in bin_time_of_arrival adds a panel coord to some fields, + # even if it has only length 1. If this is the case we concat, even if identical. + # Maybe McStasInstrument.to_coords should be changed to only handle a single + # panel, and not perform concatenation? if all(dim not in o.dims and sc.identical(first, o) for o in obj): return first return sc.concat(obj, dim) From 8a0baf6483fe7a0d090773de80f69817d70fda2e Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 12 Jun 2024 15:55:41 +0200 Subject: [PATCH 196/403] Bump Sciline requirement (temporary, move to release once available) --- packages/essnmx/pyproject.toml | 2 +- packages/essnmx/requirements/base.in | 2 +- packages/essnmx/requirements/base.txt | 50 ++++++++------- packages/essnmx/requirements/basetest.txt | 8 +-- packages/essnmx/requirements/ci.txt | 20 +++--- packages/essnmx/requirements/dev.txt | 52 ++++++++-------- packages/essnmx/requirements/docs.txt | 74 ++++++++++++----------- packages/essnmx/requirements/mypy.txt | 4 +- packages/essnmx/requirements/nightly.txt | 50 +++++++-------- packages/essnmx/requirements/static.txt | 15 ++--- packages/essnmx/requirements/wheels.txt | 14 ++--- packages/essnmx/src/ess/nmx/reduction.py | 1 + 12 files changed, 144 insertions(+), 148 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 1227ab17..948ffffd 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "dask", "graphviz", "plopp", - "sciline>=23.9.1", + "sciline @ git+https://github.com/scipp/sciline@main", "scipp>=23.8.0", "scippnexus>=23.9.0", "pooch", diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index ceeae5e3..fb91c693 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -5,7 +5,7 @@ dask graphviz plopp -sciline>=23.9.1 +sciline @ git+https://github.com/scipp/sciline@main scipp>=23.8.0 scippnexus>=23.9.0 pooch diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index f60ca9d4..e3fa4391 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,11 +1,11 @@ -# SHA1:755d32c56a7dbb9e3068b6bc07621e630837e6dc +# SHA1:82f72890967e2d05d03ca9e647a3287a16459fde # # This file is autogenerated by pip-compile-multi # To update, run: # # pip-compile-multi # -certifi==2024.2.2 +certifi==2024.6.2 # via requests charset-normalizer==3.3.2 # via requests @@ -13,34 +13,38 @@ click==8.1.7 # via dask cloudpickle==3.0.0 # via dask -contourpy==1.2.0 +contourpy==1.2.1 # via matplotlib +cyclebane==24.6.0 + # via sciline cycler==0.12.1 # via matplotlib -dask==2024.2.1 +dask==2024.5.2 # via -r base.in defusedxml==0.7.1 # via -r base.in -fonttools==4.49.0 +fonttools==4.53.0 # via matplotlib -fsspec==2024.2.0 +fsspec==2024.6.0 # via dask -gemmi==0.6.5 +gemmi==0.6.6 # via -r base.in -graphviz==0.20.1 +graphviz==0.20.3 # via -r base.in -h5py==3.10.0 +h5py==3.11.0 # via scippnexus -idna==3.6 +idna==3.7 # via requests -importlib-metadata==7.0.2 +importlib-metadata==7.1.0 # via dask kiwisolver==1.4.5 # via matplotlib locket==1.0.0 # via partd -matplotlib==3.8.3 +matplotlib==3.9.0 # via plopp +networkx==3.3 + # via cyclebane numpy==1.26.4 # via # contourpy @@ -49,22 +53,22 @@ numpy==1.26.4 # pandas # scipp # scipy -packaging==23.2 +packaging==24.1 # via # dask # matplotlib # pooch -pandas==2.2.1 +pandas==2.2.2 # via -r base.in -partd==1.4.1 +partd==1.4.2 # via dask -pillow==10.2.0 +pillow==10.3.0 # via matplotlib -platformdirs==4.2.0 +platformdirs==4.2.2 # via pooch -plopp==24.2.0 +plopp==24.5.0 # via -r base.in -pooch==1.8.1 +pooch==1.8.2 # via -r base.in pyparsing==3.1.2 # via matplotlib @@ -77,9 +81,9 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via dask -requests==2.31.0 +requests==2.32.3 # via pooch -sciline==24.2.1 +sciline @ git+https://github.com/scipp/sciline@main # via -r base.in scipp==24.5.1 # via @@ -87,7 +91,7 @@ scipp==24.5.1 # scippnexus scippnexus==24.3.1 # via -r base.in -scipy==1.12.0 +scipy==1.13.1 # via scippnexus six==1.16.0 # via python-dateutil @@ -99,5 +103,5 @@ tzdata==2024.1 # via pandas urllib3==2.2.1 # via requests -zipp==3.17.0 +zipp==3.19.2 # via importlib-metadata diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt index a1fc27b6..4d95d3c9 100644 --- a/packages/essnmx/requirements/basetest.txt +++ b/packages/essnmx/requirements/basetest.txt @@ -5,15 +5,15 @@ # # pip-compile-multi # -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 # via pytest iniconfig==2.0.0 # via pytest -packaging==23.2 +packaging==24.1 # via pytest -pluggy==1.4.0 +pluggy==1.5.0 # via pytest -pytest==8.0.2 +pytest==8.2.2 # via -r basetest.in tomli==2.0.1 # via pytest diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index 6b0041cf..1469754f 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -7,7 +7,7 @@ # cachetools==5.3.3 # via tox -certifi==2024.2.2 +certifi==2024.6.2 # via requests chardet==5.2.0 # via tox @@ -17,30 +17,30 @@ colorama==0.4.6 # via tox distlib==0.3.8 # via virtualenv -filelock==3.13.1 +filelock==3.14.0 # via # tox # virtualenv gitdb==4.0.11 # via gitpython -gitpython==3.1.42 +gitpython==3.1.43 # via -r ci.in -idna==3.6 +idna==3.7 # via requests -packaging==23.2 +packaging==24.1 # via # -r ci.in # pyproject-api # tox -platformdirs==4.2.0 +platformdirs==4.2.2 # via # tox # virtualenv -pluggy==1.4.0 +pluggy==1.5.0 # via tox pyproject-api==1.6.1 # via tox -requests==2.31.0 +requests==2.32.3 # via -r ci.in smmap==5.0.1 # via gitdb @@ -48,9 +48,9 @@ tomli==2.0.1 # via # pyproject-api # tox -tox==4.14.1 +tox==4.15.1 # via -r ci.in urllib3==2.2.1 # via requests -virtualenv==20.25.1 +virtualenv==20.26.2 # via tox diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index 33262f44..633c53bd 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -12,9 +12,9 @@ -r static.txt -r test.txt -r wheels.txt -annotated-types==0.6.0 +annotated-types==0.7.0 # via pydantic -anyio==4.3.0 +anyio==4.4.0 # via # httpx # jupyter-server @@ -28,9 +28,9 @@ async-lru==2.0.4 # via jupyterlab cffi==1.16.0 # via argon2-cffi-bindings -copier==9.1.1 +copier==9.2.0 # via -r dev.in -dunamai==1.19.2 +dunamai==1.21.1 # via copier fqdn==1.5.1 # via jsonschema @@ -38,7 +38,7 @@ funcy==2.0 # via copier h11==0.14.0 # via httpcore -httpcore==1.0.4 +httpcore==1.0.5 # via httpx httpx==0.27.0 # via jupyterlab @@ -46,30 +46,30 @@ isoduration==20.11.0 # via jsonschema jinja2-ansible-filters==1.3.2 # via copier -json5==0.9.22 +json5==0.9.25 # via jupyterlab-server -jsonpointer==2.4 +jsonpointer==3.0.0 # via jsonschema -jsonschema[format-nongpl]==4.21.1 +jsonschema[format-nongpl]==4.22.0 # via # jupyter-events # jupyterlab-server # nbformat -jupyter-events==0.9.0 +jupyter-events==0.10.0 # via jupyter-server -jupyter-lsp==2.2.4 +jupyter-lsp==2.2.5 # via jupyterlab -jupyter-server==2.13.0 +jupyter-server==2.14.1 # via # jupyter-lsp # jupyterlab # jupyterlab-server # notebook-shim -jupyter-server-terminals==0.5.2 +jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.1.4 +jupyterlab==4.2.2 # via -r dev.in -jupyterlab-server==2.25.3 +jupyterlab-server==2.27.2 # via jupyterlab notebook-shim==0.2.4 # via jupyterlab @@ -77,24 +77,22 @@ overrides==7.7.0 # via jupyter-server pathspec==0.12.1 # via copier -pip-compile-multi==2.6.3 +pip-compile-multi==2.6.4 # via -r dev.in pip-tools==7.4.1 # via pip-compile-multi -plumbum==1.8.2 +plumbum==1.8.3 # via copier prometheus-client==0.20.0 # via jupyter-server -pycparser==2.21 +pycparser==2.22 # via cffi -pydantic==2.6.3 +pydantic==2.7.3 # via copier -pydantic-core==2.16.3 +pydantic-core==2.18.4 # via pydantic python-json-logger==2.0.7 # via jupyter-events -pyyaml-include==1.3.2 - # via copier questionary==1.10.0 # via copier rfc3339-validator==0.1.4 @@ -105,27 +103,27 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -send2trash==1.8.2 +send2trash==1.8.3 # via jupyter-server sniffio==1.3.1 # via # anyio # httpx -terminado==0.18.0 +terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals toposort==1.10 # via pip-compile-multi -types-python-dateutil==2.8.19.20240106 +types-python-dateutil==2.9.0.20240316 # via arrow uri-template==1.3.0 # via jsonschema -webcolors==1.13 +webcolors==24.6.0 # via jsonschema -websocket-client==1.7.0 +websocket-client==1.8.0 # via jupyter-server -wheel==0.42.0 +wheel==0.43.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 69128c7c..c37e695d 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -6,7 +6,7 @@ # pip-compile-multi # -r base.txt -accessible-pygments==0.0.4 +accessible-pygments==0.0.5 # via pydata-sphinx-theme alabaster==0.7.16 # via sphinx @@ -16,7 +16,7 @@ attrs==23.2.0 # via # jsonschema # referencing -babel==2.14.0 +babel==2.15.0 # via # pydata-sphinx-theme # sphinx @@ -26,7 +26,7 @@ beautifulsoup4==4.12.3 # pydata-sphinx-theme bleach==6.1.0 # via nbconvert -comm==0.2.1 +comm==0.2.2 # via # ipykernel # ipywidgets @@ -34,13 +34,13 @@ debugpy==1.8.1 # via ipykernel decorator==5.1.1 # via ipython -docutils==0.20.1 +docutils==0.21.2 # via # myst-parser # nbsphinx # pydata-sphinx-theme # sphinx -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 # via ipython executing==2.0.1 # via stack-data @@ -50,34 +50,34 @@ imagesize==1.4.1 # via sphinx ipydatawidgets==4.3.5 # via pythreejs -ipykernel==6.29.3 +ipykernel==6.29.4 # via -r docs.in -ipython==8.18.1 +ipython==8.25.0 # via # -r docs.in # ipykernel # ipywidgets -ipywidgets==8.1.2 +ipywidgets==8.1.3 # via # ipydatawidgets # pythreejs jedi==0.19.1 # via ipython -jinja2==3.1.3 +jinja2==3.1.4 # via # myst-parser # nbconvert # nbsphinx # sphinx -jsonschema==4.21.1 +jsonschema==4.22.0 # via nbformat jsonschema-specifications==2023.12.1 # via jsonschema -jupyter-client==8.6.0 +jupyter-client==8.6.2 # via # ipykernel # nbclient -jupyter-core==5.7.1 +jupyter-core==5.7.2 # via # ipykernel # jupyter-client @@ -86,7 +86,7 @@ jupyter-core==5.7.1 # nbformat jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-widgets==3.0.10 +jupyterlab-widgets==3.0.11 # via ipywidgets markdown-it-py==3.0.0 # via @@ -96,38 +96,38 @@ markupsafe==2.1.5 # via # jinja2 # nbconvert -matplotlib-inline==0.1.6 +matplotlib-inline==0.1.7 # via # ipykernel # ipython -mdit-py-plugins==0.4.0 +mdit-py-plugins==0.4.1 # via myst-parser mdurl==0.1.2 # via markdown-it-py mistune==3.0.2 # via nbconvert -myst-parser==2.0.0 +myst-parser==3.0.1 # via -r docs.in -nbclient==0.9.0 +nbclient==0.10.0 # via nbconvert -nbconvert==7.16.2 +nbconvert==7.16.4 # via nbsphinx -nbformat==5.9.2 +nbformat==5.10.4 # via # nbclient # nbconvert # nbsphinx -nbsphinx==0.9.3 +nbsphinx==0.9.4 # via -r docs.in nest-asyncio==1.6.0 # via ipykernel pandocfilters==1.5.1 # via nbconvert -parso==0.8.3 +parso==0.8.4 # via jedi pexpect==4.9.0 # via ipython -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.47 # via ipython psutil==5.9.8 # via ipykernel @@ -135,9 +135,9 @@ ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 # via stack-data -pydata-sphinx-theme==0.15.2 +pydata-sphinx-theme==0.15.3 # via -r docs.in -pygments==2.17.2 +pygments==2.18.0 # via # accessible-pygments # ipython @@ -146,25 +146,25 @@ pygments==2.17.2 # sphinx pythreejs==2.4.2 # via -r docs.in -pyzmq==25.1.2 +pyzmq==26.0.3 # via # ipykernel # jupyter-client -referencing==0.33.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications -rpds-py==0.18.0 +rpds-py==0.18.1 # via # jsonschema # referencing -scippneutron==24.1.0 +scippneutron==24.5.0 # via -r docs.in snowballstemmer==2.2.0 # via sphinx soupsieve==2.5 # via beautifulsoup4 -sphinx==7.2.6 +sphinx==7.3.7 # via # -r docs.in # myst-parser @@ -173,11 +173,11 @@ sphinx==7.2.6 # sphinx-autodoc-typehints # sphinx-copybutton # sphinx-design -sphinx-autodoc-typehints==2.0.0 +sphinx-autodoc-typehints==2.1.1 # via -r docs.in sphinx-copybutton==0.5.2 # via -r docs.in -sphinx-design==0.5.0 +sphinx-design==0.6.0 # via -r docs.in sphinxcontrib-applehelp==1.0.8 # via sphinx @@ -193,13 +193,15 @@ sphinxcontrib-serializinghtml==1.1.10 # via sphinx stack-data==0.6.3 # via ipython -tinycss2==1.2.1 +tinycss2==1.3.0 # via nbconvert -tornado==6.4 +tomli==2.0.1 + # via sphinx +tornado==6.4.1 # via # ipykernel # jupyter-client -traitlets==5.14.1 +traitlets==5.14.3 # via # comm # ipykernel @@ -216,7 +218,7 @@ traitlets==5.14.1 # traittypes traittypes==0.2.1 # via ipydatawidgets -typing-extensions==4.10.0 +typing-extensions==4.12.2 # via # ipython # pydata-sphinx-theme @@ -226,5 +228,5 @@ webencodings==0.5.1 # via # bleach # tinycss2 -widgetsnbextension==4.0.10 +widgetsnbextension==4.0.11 # via ipywidgets diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index 49722576..3644c507 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -6,9 +6,9 @@ # pip-compile-multi # -r test.txt -mypy==1.8.0 +mypy==1.10.0 # via -r mypy.in mypy-extensions==1.0.0 # via mypy -typing-extensions==4.10.0 +typing-extensions==4.12.2 # via mypy diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index f671ace6..35d17e33 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:5ab5fa78fc427b600d881ae674cc81dcf0cc3eeb +# SHA1:38452d7c1b37c7bcf4e0819d4f3cfcbe168c62ad # # This file is autogenerated by pip-compile-multi # To update, run: @@ -6,7 +6,7 @@ # pip-compile-multi # -r basetest.txt -certifi==2024.2.2 +certifi==2024.6.2 # via requests charset-normalizer==3.3.2 # via requests @@ -14,36 +14,38 @@ click==8.1.7 # via dask cloudpickle==3.0.0 # via dask -contourpy==1.2.0 +contourpy==1.2.1 # via matplotlib +cyclebane==24.6.0 + # via sciline cycler==0.12.1 # via matplotlib -dask==2024.2.1 +dask==2024.5.2 # via -r nightly.in defusedxml==0.7.1 # via -r nightly.in -fonttools==4.49.0 +fonttools==4.53.0 # via matplotlib -fsspec==2024.2.0 +fsspec==2024.6.0 # via dask -gemmi==0.6.5 +gemmi==0.6.6 # via -r nightly.in -graphviz==0.20.1 +graphviz==0.20.3 # via -r nightly.in -h5py==3.10.0 +h5py==3.11.0 # via scippnexus -idna==3.6 +idna==3.7 # via requests -importlib-metadata==7.0.2 +importlib-metadata==7.1.0 # via dask -importlib-resources==6.1.3 - # via matplotlib kiwisolver==1.4.5 # via matplotlib locket==1.0.0 # via partd -matplotlib==3.8.3 +matplotlib==3.9.0 # via plopp +networkx==3.3 + # via cyclebane numpy==1.26.4 # via # contourpy @@ -52,17 +54,17 @@ numpy==1.26.4 # pandas # scipp # scipy -pandas==2.2.1 +pandas==2.2.2 # via -r nightly.in -partd==1.4.1 +partd==1.4.2 # via dask -pillow==10.2.0 +pillow==10.3.0 # via matplotlib -platformdirs==4.2.0 +platformdirs==4.2.2 # via pooch plopp @ git+https://github.com/scipp/plopp@main # via -r nightly.in -pooch==1.8.1 +pooch==1.8.2 # via -r nightly.in pyparsing==3.1.2 # via matplotlib @@ -75,7 +77,7 @@ pytz==2024.1 # via pandas pyyaml==6.0.1 # via dask -requests==2.31.0 +requests==2.32.3 # via pooch sciline @ git+https://github.com/scipp/sciline@main # via -r nightly.in @@ -85,7 +87,7 @@ scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-c # scippnexus scippnexus @ git+https://github.com/scipp/scippnexus@main # via -r nightly.in -scipy==1.12.0 +scipy==1.13.1 # via scippnexus six==1.16.0 # via python-dateutil @@ -97,7 +99,5 @@ tzdata==2024.1 # via pandas urllib3==2.2.1 # via requests -zipp==3.17.0 - # via - # importlib-metadata - # importlib-resources +zipp==3.19.2 + # via importlib-metadata diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index 0e3af8ac..8fb5c318 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -9,20 +9,17 @@ cfgv==3.4.0 # via pre-commit distlib==0.3.8 # via virtualenv -filelock==3.13.1 +filelock==3.14.0 # via virtualenv -identify==2.5.35 +identify==2.5.36 # via pre-commit -nodeenv==1.8.0 +nodeenv==1.9.1 # via pre-commit -platformdirs==4.2.0 +platformdirs==4.2.2 # via virtualenv -pre-commit==3.6.2 +pre-commit==3.7.1 # via -r static.in pyyaml==6.0.1 # via pre-commit -virtualenv==20.25.1 +virtualenv==20.26.2 # via pre-commit - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/packages/essnmx/requirements/wheels.txt b/packages/essnmx/requirements/wheels.txt index a0234e2c..a1fa46e2 100644 --- a/packages/essnmx/requirements/wheels.txt +++ b/packages/essnmx/requirements/wheels.txt @@ -5,17 +5,11 @@ # # pip-compile-multi # -build==1.1.1 +build==1.2.1 # via -r wheels.in -importlib-metadata==7.0.2 +packaging==24.1 # via build -packaging==23.2 - # via build -pyproject-hooks==1.0.0 +pyproject-hooks==1.1.0 # via build tomli==2.0.1 - # via - # build - # pyproject-hooks -zipp==3.17.0 - # via importlib-metadata + # via build diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index fc2fb7dd..1bb3df4f 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -40,6 +40,7 @@ def _concat_or_same( def merge_panels(*panel: NMXReducedData) -> NMXReducedData: + """Merge a list of panels by concatenating along the 'panel' dimension.""" keys = panel[0].keys() if not all(p.keys() == keys for p in panel): raise ValueError("All panels must have the same keys.") From d25397d327f3993fca54fab1c7bb1d2317436af3 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Mon, 17 Jun 2024 07:53:20 +0200 Subject: [PATCH 197/403] Update to released Sciline --- packages/essnmx/pyproject.toml | 3 ++- packages/essnmx/requirements/base.in | 3 ++- packages/essnmx/requirements/base.txt | 7 ++++--- packages/essnmx/requirements/ci.txt | 2 +- packages/essnmx/requirements/dev.txt | 2 +- packages/essnmx/requirements/docs.txt | 2 +- packages/essnmx/requirements/nightly.in | 1 + packages/essnmx/requirements/nightly.txt | 5 +++-- packages/essnmx/requirements/static.txt | 2 +- 9 files changed, 16 insertions(+), 11 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 948ffffd..f70ae645 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -32,8 +32,9 @@ requires-python = ">=3.10" dependencies = [ "dask", "graphviz", + "numpy<2.0.0", "plopp", - "sciline @ git+https://github.com/scipp/sciline@main", + "sciline>=24.06.0", "scipp>=23.8.0", "scippnexus>=23.9.0", "pooch", diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index fb91c693..22a38bad 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -4,8 +4,9 @@ # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask graphviz +numpy<2.0.0 plopp -sciline @ git+https://github.com/scipp/sciline@main +sciline>=24.06.0 scipp>=23.8.0 scippnexus>=23.9.0 pooch diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index e3fa4391..39699eb8 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:82f72890967e2d05d03ca9e647a3287a16459fde +# SHA1:d990037ca721ab7b651343616ad51b7121679fd1 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -19,7 +19,7 @@ cyclebane==24.6.0 # via sciline cycler==0.12.1 # via matplotlib -dask==2024.5.2 +dask==2024.6.0 # via -r base.in defusedxml==0.7.1 # via -r base.in @@ -47,6 +47,7 @@ networkx==3.3 # via cyclebane numpy==1.26.4 # via + # -r base.in # contourpy # h5py # matplotlib @@ -83,7 +84,7 @@ pyyaml==6.0.1 # via dask requests==2.32.3 # via pooch -sciline @ git+https://github.com/scipp/sciline@main +sciline==24.6.0 # via -r base.in scipp==24.5.1 # via diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index 1469754f..9c6e429f 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -17,7 +17,7 @@ colorama==0.4.6 # via tox distlib==0.3.8 # via virtualenv -filelock==3.14.0 +filelock==3.15.1 # via # tox # virtualenv diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index 633c53bd..077dbfc5 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -87,7 +87,7 @@ prometheus-client==0.20.0 # via jupyter-server pycparser==2.22 # via cffi -pydantic==2.7.3 +pydantic==2.7.4 # via copier pydantic-core==2.18.4 # via pydantic diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index c37e695d..30696ff7 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -44,7 +44,7 @@ exceptiongroup==1.2.1 # via ipython executing==2.0.1 # via stack-data -fastjsonschema==2.19.1 +fastjsonschema==2.20.0 # via nbformat imagesize==1.4.1 # via sphinx diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index 245b5831..e5eb360c 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -3,6 +3,7 @@ # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask graphviz +numpy<2.0.0 pooch pandas gemmi diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 35d17e33..563f4aca 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:38452d7c1b37c7bcf4e0819d4f3cfcbe168c62ad +# SHA1:cf810ac50e2bcd3c53242a4261e4ec4706454425 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -20,7 +20,7 @@ cyclebane==24.6.0 # via sciline cycler==0.12.1 # via matplotlib -dask==2024.5.2 +dask==2024.6.0 # via -r nightly.in defusedxml==0.7.1 # via -r nightly.in @@ -48,6 +48,7 @@ networkx==3.3 # via cyclebane numpy==1.26.4 # via + # -r nightly.in # contourpy # h5py # matplotlib diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index 8fb5c318..cb03fb85 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -9,7 +9,7 @@ cfgv==3.4.0 # via pre-commit distlib==0.3.8 # via virtualenv -filelock==3.14.0 +filelock==3.15.1 # via virtualenv identify==2.5.36 # via pre-commit From 266e22360c50d71449de2f45c23216859a0ce399 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Tue, 18 Jun 2024 09:11:28 +0200 Subject: [PATCH 198/403] Fix docs --- .../docs/examples/scaling_workflow.ipynb | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index 6b6a691c..cbf3c150 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -29,7 +29,15 @@ "metadata": {}, "outputs": [], "source": [ - "from ess.nmx.mtz_io import read_mtz_file, mtz_to_pandas, MTZFilePath\n", + "import gemmi\n", + "from ess.nmx.mtz_io import (\n", + " read_mtz_file,\n", + " mtz_to_pandas,\n", + " MTZFilePath,\n", + " get_unique_space_group,\n", + " MtzDataFrame,\n", + " merge_mtz_dataframes,\n", + ")\n", "from ess.nmx.data import get_small_random_mtz_samples\n", "\n", "\n", @@ -65,8 +73,11 @@ "import sciline as sl\n", "import scipp as sc\n", "\n", - "from ess.nmx.mtz_io import providers as mtz_io_providers, default_parameters as mtz_io_params\n", - "from ess.nmx.mtz_io import SpaceGroupDesc, SpaceGroup, get_unique_space_group, merge_mtz_dataframes, MtzDataFrame\n", + "from ess.nmx.mtz_io import (\n", + " providers as mtz_io_providers,\n", + " default_parameters as mtz_io_params,\n", + ")\n", + "from ess.nmx.mtz_io import SpaceGroupDesc\n", "from ess.nmx.scaling import scaling_providers, scaling_params\n", "from ess.nmx.scaling import (\n", " FilteredEstimatedScaledIntensities,\n", @@ -92,8 +103,7 @@ " **scaling_params,\n", " },\n", ")\n", - "pl\n", - "\n" + "pl" ] }, { @@ -102,10 +112,16 @@ "metadata": {}, "outputs": [], "source": [ - "file_paths = pd.DataFrame({MTZFilePath: get_small_random_mtz_samples()}).rename_axis(\"mtzfile\")\n", + "file_paths = pd.DataFrame({MTZFilePath: get_small_random_mtz_samples()}).rename_axis(\n", + " \"mtzfile\"\n", + ")\n", "mapped = pl.map(file_paths)\n", - "pl[SpaceGroup] = mapped[SpaceGroup|None].reduce(index='mtzfile', func=get_unique_space_group)\n", - "pl[MtzDataFrame] = mapped[MtzDataFrame].reduce(index='mtzfile', func=merge_mtz_dataframes)" + "pl[gemmi.SpaceGroup] = mapped[gemmi.SpaceGroup | None].reduce(\n", + " index='mtzfile', func=get_unique_space_group\n", + ")\n", + "pl[MtzDataFrame] = mapped[MtzDataFrame].reduce(\n", + " index='mtzfile', func=merge_mtz_dataframes\n", + ")" ] }, { From 12528d50c5bbfb4cdfdc9582566182fd922969ee Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Tue, 18 Jun 2024 11:34:53 +0200 Subject: [PATCH 199/403] Use Scipp instead of Pandas --- packages/essnmx/docs/examples/workflow.ipynb | 4 +--- packages/essnmx/src/ess/nmx/mcstas/load.py | 2 +- packages/essnmx/src/ess/nmx/types.py | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index 0c9cebc1..3edd44a4 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -19,8 +19,6 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", - "\n", "from ess.nmx.mcstas import McStasWorkflow\n", "from ess.nmx.data import small_mcstas_3_sample\n", "\n", @@ -68,7 +66,7 @@ "# in this case we select all three panels.\n", "wf[NMXReducedData] = (\n", " wf[NMXReducedData]\n", - " .map(pd.DataFrame({DetectorIndex: range(3)}).rename_axis(\"panel\"))\n", + " .map({DetectorIndex: sc.arange('panel', 3, unit=None)})\n", " .reduce(index=\"panel\", func=merge_panels)\n", ")" ] diff --git a/packages/essnmx/src/ess/nmx/mcstas/load.py b/packages/essnmx/src/ess/nmx/mcstas/load.py index 50b25efe..414a87f1 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/load.py +++ b/packages/essnmx/src/ess/nmx/mcstas/load.py @@ -22,7 +22,7 @@ def detector_name_from_index(index: DetectorIndex) -> DetectorName: - return f'nD_Mantid_{index}' + return f'nD_Mantid_{getattr(index, "value", index)}' def load_event_data_bank_name( diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index e9ed7077..c4906377 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -5,7 +5,7 @@ FilePath = NewType("FilePath", str) """File name of a file containing the results of a McStas run""" -DetectorIndex = NewType("DetectorIndex", int) +DetectorIndex = NewType("DetectorIndex", int | sc.Variable | sc.DataArray) """Index of the detector to load. Index ordered by the id:s of the pixels""" DetectorName = NewType("DetectorName", str) From f753d3e227e78dfa938ef53d0d2da5af21473c5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Jun 2024 17:31:08 +0000 Subject: [PATCH 200/403] Bump scipp from 24.5.1 to 24.6.0 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 24.5.1 to 24.6.0. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/24.05.1...24.06.0) --- updated-dependencies: - dependency-name: scipp dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 39699eb8..453b64e6 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -35,8 +35,6 @@ h5py==3.11.0 # via scippnexus idna==3.7 # via requests -importlib-metadata==7.1.0 - # via dask kiwisolver==1.4.5 # via matplotlib locket==1.0.0 @@ -86,7 +84,7 @@ requests==2.32.3 # via pooch sciline==24.6.0 # via -r base.in -scipp==24.5.1 +scipp==24.6.0 # via # -r base.in # scippnexus @@ -104,5 +102,3 @@ tzdata==2024.1 # via pandas urllib3==2.2.1 # via requests -zipp==3.19.2 - # via importlib-metadata From 48db8489931fa0b4a76c9dbb3165bfd963a415df Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 11 Jun 2024 11:18:36 +0200 Subject: [PATCH 201/403] Use WavelengthBins instead of separate parameters for min, max and number of bins --- packages/essnmx/docs/examples/scaling_workflow.ipynb | 9 ++++----- packages/essnmx/src/ess/nmx/scaling.py | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/examples/scaling_workflow.ipynb index cbf3c150..e5ecbb75 100644 --- a/packages/essnmx/docs/examples/scaling_workflow.ipynb +++ b/packages/essnmx/docs/examples/scaling_workflow.ipynb @@ -73,13 +73,11 @@ "import sciline as sl\n", "import scipp as sc\n", "\n", - "from ess.nmx.mtz_io import (\n", - " providers as mtz_io_providers,\n", - " default_parameters as mtz_io_params,\n", - ")\n", + "from ess.nmx.mtz_io import providers as mtz_io_providers, default_parameters as mtz_io_params\n", "from ess.nmx.mtz_io import SpaceGroupDesc\n", - "from ess.nmx.scaling import scaling_providers, scaling_params\n", + "from ess.nmx.scaling import providers as scaling_providers, default_parameters as scaling_params\n", "from ess.nmx.scaling import (\n", + " WavelengthBins,\n", " FilteredEstimatedScaledIntensities,\n", " ReferenceWavelength,\n", " ScaledIntensityLeftTailThreshold,\n", @@ -101,6 +99,7 @@ " ),\n", " **mtz_io_params,\n", " **scaling_params,\n", + " WavelengthBins: 250,\n", " },\n", ")\n", "pl" diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index ec74682d..cf0383ed 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -2,14 +2,14 @@ # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) from collections.abc import Callable, Mapping from dataclasses import dataclass -from typing import NewType, TypeVar +from typing import NewType, TypeVar, Union import scipp as sc from .mtz_io import NMXMtzDataArray # User defined or configurable types -WavelengthBins = NewType("WavelengthBins", sc.Variable) +WavelengthBins = NewType("WavelengthBins", Union[sc.Variable, int]) """User configurable wavelength binning""" ReferenceWavelength = NewType("ReferenceWavelength", sc.Variable | None) """The wavelength to select reference intensities.""" @@ -438,8 +438,7 @@ def calculate_wavelength_scale_factor( return WavelengthScaleFactors(scale_factor) -# Providers and default parameters -scaling_providers = ( +providers = ( cut_tails, get_wavelength_binned, get_reference_wavelength, @@ -451,7 +450,7 @@ def calculate_wavelength_scale_factor( ) """Providers for scaling data.""" -scaling_params = { +default_parameters = { WavelengthBins: sc.linspace("wavelength", 2.6, 3.6, 250, unit="angstrom"), ScaledIntensityLeftTailThreshold: DEFAULT_LEFT_TAIL_THRESHOLD, ScaledIntensityRightTailThreshold: DEFAULT_RIGHT_TAIL_THRESHOLD, From f2b4d8dbf943f4d8dcb4217073eeeef319afe329 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 18 Jun 2024 11:29:11 +0200 Subject: [PATCH 202/403] deps: new sciline + tox deps --- packages/essnmx/pyproject.toml | 3 +-- packages/essnmx/requirements/base.in | 3 +-- packages/essnmx/requirements/base.txt | 19 +++++++++++-------- packages/essnmx/requirements/ci.txt | 8 ++++---- packages/essnmx/requirements/docs.txt | 4 ++-- packages/essnmx/requirements/nightly.in | 1 - packages/essnmx/requirements/nightly.txt | 11 +++++------ packages/essnmx/requirements/static.txt | 4 ++-- 8 files changed, 26 insertions(+), 27 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index f70ae645..43c1efa9 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -32,11 +32,10 @@ requires-python = ">=3.10" dependencies = [ "dask", "graphviz", - "numpy<2.0.0", "plopp", "sciline>=24.06.0", "scipp>=23.8.0", - "scippnexus>=23.9.0", + "scippnexus>=23.12.0", "pooch", "pandas", "gemmi", diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index 22a38bad..f746e638 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -4,11 +4,10 @@ # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask graphviz -numpy<2.0.0 plopp sciline>=24.06.0 scipp>=23.8.0 -scippnexus>=23.9.0 +scippnexus>=23.12.0 pooch pandas gemmi diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 453b64e6..1895bdc3 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:d990037ca721ab7b651343616ad51b7121679fd1 +# SHA1:bea70253436877e109e43f0839bf995cf4bbc84f # # This file is autogenerated by pip-compile-multi # To update, run: @@ -19,7 +19,7 @@ cyclebane==24.6.0 # via sciline cycler==0.12.1 # via matplotlib -dask==2024.6.0 +dask==2024.6.2 # via -r base.in defusedxml==0.7.1 # via -r base.in @@ -35,6 +35,8 @@ h5py==3.11.0 # via scippnexus idna==3.7 # via requests +importlib-metadata==7.2.1 + # via dask kiwisolver==1.4.5 # via matplotlib locket==1.0.0 @@ -43,9 +45,8 @@ matplotlib==3.9.0 # via plopp networkx==3.3 # via cyclebane -numpy==1.26.4 +numpy==2.0.0 # via - # -r base.in # contourpy # h5py # matplotlib @@ -65,7 +66,7 @@ pillow==10.3.0 # via matplotlib platformdirs==4.2.2 # via pooch -plopp==24.5.0 +plopp==24.6.0 # via -r base.in pooch==1.8.2 # via -r base.in @@ -82,13 +83,13 @@ pyyaml==6.0.1 # via dask requests==2.32.3 # via pooch -sciline==24.6.0 +sciline==24.6.1 # via -r base.in scipp==24.6.0 # via # -r base.in # scippnexus -scippnexus==24.3.1 +scippnexus==24.6.0 # via -r base.in scipy==1.13.1 # via scippnexus @@ -100,5 +101,7 @@ toolz==0.12.1 # partd tzdata==2024.1 # via pandas -urllib3==2.2.1 +urllib3==2.2.2 # via requests +zipp==3.19.2 + # via importlib-metadata diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index 9c6e429f..0ef98e16 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -17,7 +17,7 @@ colorama==0.4.6 # via tox distlib==0.3.8 # via virtualenv -filelock==3.15.1 +filelock==3.15.4 # via # tox # virtualenv @@ -38,7 +38,7 @@ platformdirs==4.2.2 # virtualenv pluggy==1.5.0 # via tox -pyproject-api==1.6.1 +pyproject-api==1.7.1 # via tox requests==2.32.3 # via -r ci.in @@ -50,7 +50,7 @@ tomli==2.0.1 # tox tox==4.15.1 # via -r ci.in -urllib3==2.2.1 +urllib3==2.2.2 # via requests -virtualenv==20.26.2 +virtualenv==20.26.3 # via tox diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 30696ff7..437dd76d 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -129,7 +129,7 @@ pexpect==4.9.0 # via ipython prompt-toolkit==3.0.47 # via ipython -psutil==5.9.8 +psutil==6.0.0 # via ipykernel ptyprocess==0.7.0 # via pexpect @@ -173,7 +173,7 @@ sphinx==7.3.7 # sphinx-autodoc-typehints # sphinx-copybutton # sphinx-design -sphinx-autodoc-typehints==2.1.1 +sphinx-autodoc-typehints==2.2.2 # via -r docs.in sphinx-copybutton==0.5.2 # via -r docs.in diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index e5eb360c..245b5831 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -3,7 +3,6 @@ # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask graphviz -numpy<2.0.0 pooch pandas gemmi diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 563f4aca..2a8fc9da 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:cf810ac50e2bcd3c53242a4261e4ec4706454425 +# SHA1:38452d7c1b37c7bcf4e0819d4f3cfcbe168c62ad # # This file is autogenerated by pip-compile-multi # To update, run: @@ -20,7 +20,7 @@ cyclebane==24.6.0 # via sciline cycler==0.12.1 # via matplotlib -dask==2024.6.0 +dask==2024.6.2 # via -r nightly.in defusedxml==0.7.1 # via -r nightly.in @@ -36,7 +36,7 @@ h5py==3.11.0 # via scippnexus idna==3.7 # via requests -importlib-metadata==7.1.0 +importlib-metadata==7.2.1 # via dask kiwisolver==1.4.5 # via matplotlib @@ -46,9 +46,8 @@ matplotlib==3.9.0 # via plopp networkx==3.3 # via cyclebane -numpy==1.26.4 +numpy==2.0.0 # via - # -r nightly.in # contourpy # h5py # matplotlib @@ -98,7 +97,7 @@ toolz==0.12.1 # partd tzdata==2024.1 # via pandas -urllib3==2.2.1 +urllib3==2.2.2 # via requests zipp==3.19.2 # via importlib-metadata diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index cb03fb85..bdf1ccc0 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -9,7 +9,7 @@ cfgv==3.4.0 # via pre-commit distlib==0.3.8 # via virtualenv -filelock==3.15.1 +filelock==3.15.4 # via virtualenv identify==2.5.36 # via pre-commit @@ -21,5 +21,5 @@ pre-commit==3.7.1 # via -r static.in pyyaml==6.0.1 # via pre-commit -virtualenv==20.26.2 +virtualenv==20.26.3 # via pre-commit From b7e54ddcdd81b208ecf4e1e7d7c010556261eab4 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 4 Jul 2024 14:14:56 +0200 Subject: [PATCH 203/403] Update pre-commit hooks. --- packages/essnmx/.pre-commit-config.yaml | 29 +++++++------------------ 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/packages/essnmx/.pre-commit-config.yaml b/packages/essnmx/.pre-commit-config.yaml index 19830db7..4442b1b9 100644 --- a/packages/essnmx/.pre-commit-config.yaml +++ b/packages/essnmx/.pre-commit-config.yaml @@ -13,15 +13,6 @@ repos: - id: trailing-whitespace args: [ --markdown-linebreak-ext=md ] exclude: '\.svg' - - repo: https://github.com/pycqa/isort - rev: 5.12.0 - hooks: - - id: isort - name: isort (python) - - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.11.0 - hooks: - - id: black - repo: https://github.com/kynan/nbstripout rev: 0.6.0 hooks: @@ -29,18 +20,14 @@ repos: types: [ "jupyter" ] args: [ "--drop-empty-cells", "--extra-keys 'metadata.language_info.version cell.metadata.jp-MarkdownHeadingCollapsed cell.metadata.pycharm'" ] - - repo: https://github.com/pycqa/flake8 - rev: 6.1.0 - hooks: - - id: flake8 - types: ["python"] - additional_dependencies: ["flake8-bugbear==23.9.16"] - - repo: https://github.com/pycqa/bandit - rev: 1.7.5 - hooks: - - id: bandit - additional_dependencies: ["bandit[toml]"] - args: ["-c", "pyproject.toml"] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.3 + hooks: + - id: ruff + args: [ --fix ] + types_or: [ python, pyi, jupyter ] + - id: ruff-format + types_or: [ python, pyi ] - repo: https://github.com/codespell-project/codespell rev: v2.2.6 hooks: From fe5dfee0d774d1d7adf5ad711657801a6557fc40 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 4 Jul 2024 14:15:08 +0200 Subject: [PATCH 204/403] Update github actions. --- packages/essnmx/.github/workflows/ci.yml | 7 ++----- packages/essnmx/.github/workflows/docs.yml | 12 +++++------ .../.github/workflows/nightly_at_main.yml | 3 --- .../.github/workflows/nightly_at_release.yml | 3 --- packages/essnmx/.github/workflows/release.yml | 21 ++++++++----------- packages/essnmx/.github/workflows/test.yml | 9 +++----- .../essnmx/.github/workflows/unpinned.yml | 3 --- 7 files changed, 19 insertions(+), 39 deletions(-) diff --git a/packages/essnmx/.github/workflows/ci.yml b/packages/essnmx/.github/workflows/ci.yml index 497ab895..44266a23 100644 --- a/packages/essnmx/.github/workflows/ci.yml +++ b/packages/essnmx/.github/workflows/ci.yml @@ -1,6 +1,3 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) - name: CI on: @@ -24,13 +21,13 @@ jobs: run: | echo "min_python=$(cat .github/workflows/python-version-ci)" >> $GITHUB_OUTPUT echo "min_tox_env=py$(cat .github/workflows/python-version-ci | sed 's/\.//g')" >> $GITHUB_OUTPUT - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version-file: '.github/workflows/python-version-ci' - uses: pre-commit/action@v3.0.1 with: extra_args: --all-files - - uses: pre-commit-ci/lite-action@v1.0.1 + - uses: pre-commit-ci/lite-action@v1.0.2 if: always() with: msg: Apply automatic formatting diff --git a/packages/essnmx/.github/workflows/docs.yml b/packages/essnmx/.github/workflows/docs.yml index 5e189fb0..98aaf568 100644 --- a/packages/essnmx/.github/workflows/docs.yml +++ b/packages/essnmx/.github/workflows/docs.yml @@ -1,6 +1,3 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) - name: Docs on: @@ -47,11 +44,12 @@ jobs: runs-on: 'ubuntu-22.04' steps: - run: sudo apt install --yes graphviz pandoc - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ inputs.branch == '' && github.ref_name || inputs.branch }} + repository: ${{ github.event.pull_request.head.repo.full_name }} fetch-depth: 0 # history required so cmake can determine version - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version-file: '.github/workflows/python-version-ci' - run: python -m pip install --upgrade pip @@ -62,12 +60,12 @@ jobs: if: ${{ inputs.version == '' }} - run: tox -e linkcheck if: ${{ inputs.linkcheck }} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: docs_html path: html/ - - uses: JamesIves/github-pages-deploy-action@v4.4.3 + - uses: JamesIves/github-pages-deploy-action@v4.6.1 if: ${{ inputs.publish }} with: branch: gh-pages diff --git a/packages/essnmx/.github/workflows/nightly_at_main.yml b/packages/essnmx/.github/workflows/nightly_at_main.yml index 10730688..08fdddd2 100644 --- a/packages/essnmx/.github/workflows/nightly_at_main.yml +++ b/packages/essnmx/.github/workflows/nightly_at_main.yml @@ -1,6 +1,3 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) - name: Nightly test at main branch on: diff --git a/packages/essnmx/.github/workflows/nightly_at_release.yml b/packages/essnmx/.github/workflows/nightly_at_release.yml index 7f1653bb..373c4546 100644 --- a/packages/essnmx/.github/workflows/nightly_at_release.yml +++ b/packages/essnmx/.github/workflows/nightly_at_release.yml @@ -1,6 +1,3 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) - name: Nightly tests at latest release on: diff --git a/packages/essnmx/.github/workflows/release.yml b/packages/essnmx/.github/workflows/release.yml index e492978a..d1317a76 100644 --- a/packages/essnmx/.github/workflows/release.yml +++ b/packages/essnmx/.github/workflows/release.yml @@ -1,6 +1,3 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) - name: Release on: @@ -18,7 +15,7 @@ jobs: runs-on: 'ubuntu-22.04' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true fetch-depth: 0 # history required so setuptools_scm can determine version @@ -31,7 +28,7 @@ jobs: boa - run: conda mambabuild --channel conda-forge --channel scipp --no-anaconda-upload --override-channels --output-folder conda/package conda - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: conda-package-noarch path: conda/package/noarch/*.tar.bz2 @@ -41,11 +38,11 @@ jobs: runs-on: 'ubuntu-22.04' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 # history required so setuptools_scm can determine version - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version-file: '.github/workflows/python-version-ci' @@ -56,7 +53,7 @@ jobs: run: python -m build - name: Upload wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: dist path: dist @@ -70,8 +67,8 @@ jobs: id-token: write if: github.event_name == 'release' && github.event.action == 'published' steps: - - uses: actions/download-artifact@v3 - - uses: pypa/gh-action-pypi-publish@v1.8.10 + - uses: actions/download-artifact@v4 + - uses: pypa/gh-action-pypi-publish@v1.8.14 upload_conda: name: Deploy Conda @@ -80,7 +77,7 @@ jobs: if: github.event_name == 'release' && github.event.action == 'published' steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 - uses: mamba-org/setup-micromamba@v1 with: environment-name: upload-env @@ -104,7 +101,7 @@ jobs: permissions: contents: write # This is needed so that the action can upload the asset steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 - name: Zip documentation run: | mv docs_html documentation-${{ github.ref_name }} diff --git a/packages/essnmx/.github/workflows/test.yml b/packages/essnmx/.github/workflows/test.yml index 3cfb75de..5f56a069 100644 --- a/packages/essnmx/.github/workflows/test.yml +++ b/packages/essnmx/.github/workflows/test.yml @@ -1,6 +1,3 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) - name: Test on: @@ -48,16 +45,16 @@ jobs: runs-on: ${{ inputs.os-variant }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ inputs.checkout_ref }} - - uses: actions/setup-python@v3 + - uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version }} - run: python -m pip install --upgrade pip - run: python -m pip install -r ${{ inputs.pip-recipe }} - run: tox -e ${{ inputs.tox-env }} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: ${{ inputs.coverage-report }} with: name: CoverageReport diff --git a/packages/essnmx/.github/workflows/unpinned.yml b/packages/essnmx/.github/workflows/unpinned.yml index 853c1ec5..46a84c1c 100644 --- a/packages/essnmx/.github/workflows/unpinned.yml +++ b/packages/essnmx/.github/workflows/unpinned.yml @@ -1,6 +1,3 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) - name: Unpinned tests at latest release on: From 65e1c74ac9d7ee184f4004e03389beb22aa9d8a3 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 4 Jul 2024 14:16:06 +0200 Subject: [PATCH 205/403] Update docs materials. --- .../essnmx/docs/_templates/doc_version.html | 2 +- packages/essnmx/docs/conf.py | 109 ++++++++++-------- .../docs/developer/coding-conventions.md | 2 +- packages/essnmx/docs/developer/index.md | 2 +- 4 files changed, 64 insertions(+), 51 deletions(-) diff --git a/packages/essnmx/docs/_templates/doc_version.html b/packages/essnmx/docs/_templates/doc_version.html index a348e28c..64fad220 100644 --- a/packages/essnmx/docs/_templates/doc_version.html +++ b/packages/essnmx/docs/_templates/doc_version.html @@ -1,2 +1,2 @@ -Current {{ project }} version: {{ version }} (older versions). +Current ESSnmx version: {{ version }} (older versions). diff --git a/packages/essnmx/docs/conf.py b/packages/essnmx/docs/conf.py index bea8cd80..adc0f235 100644 --- a/packages/essnmx/docs/conf.py +++ b/packages/essnmx/docs/conf.py @@ -1,42 +1,48 @@ -# -*- coding: utf-8 -*- - import doctest import os import sys +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as get_version + +from sphinx.util import logging -from ess import nmx +sys.path.insert(0, os.path.abspath(".")) -sys.path.insert(0, os.path.abspath('.')) +logger = logging.getLogger(__name__) # General information about the project. -project = u'ESSnmx' -copyright = u'2024 Scipp contributors' -author = u'Scipp contributors' +project = "ESSnmx" +copyright = "2024 Scipp contributors" +author = "Scipp contributors" html_show_sourcelink = True extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.autosummary', - 'sphinx.ext.doctest', - 'sphinx.ext.githubpages', - 'sphinx.ext.intersphinx', - 'sphinx.ext.mathjax', - 'sphinx.ext.napoleon', - 'sphinx.ext.viewcode', - 'sphinx_autodoc_typehints', - 'sphinx_copybutton', - 'sphinx_design', - 'nbsphinx', - 'myst_parser', + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.doctest", + "sphinx.ext.githubpages", + "sphinx.ext.intersphinx", + "sphinx.ext.mathjax", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinx_autodoc_typehints", + "sphinx_copybutton", + "sphinx_design", + "nbsphinx", + "myst_parser", ] + try: import sciline.sphinxext.domain_types # noqa: F401 - extensions.append('sciline.sphinxext.domain_types') + extensions.append("sciline.sphinxext.domain_types") + # See https://github.com/tox-dev/sphinx-autodoc-typehints/issues/457 + suppress_warnings = ["config.cache"] except ModuleNotFoundError: pass + myst_enable_extensions = [ "amsmath", "colon_fence", @@ -55,13 +61,13 @@ myst_heading_anchors = 3 autodoc_type_aliases = { - 'array_like': 'array_like', + "array_like": "array_like", } intersphinx_mapping = { - 'python': ('https://docs.python.org/3', None), - 'numpy': ('https://numpy.org/doc/stable/', None), - 'scipp': ('https://scipp.github.io/', None), + "python": ("https://docs.python.org/3", None), + "numpy": ("https://numpy.org/doc/stable/", None), + "scipp": ("https://scipp.github.io/", None), } # autodocs includes everything, even irrelevant API internals. autosummary @@ -78,40 +84,47 @@ # objects without namespace: numpy "ndarray": "~numpy.ndarray", } -typehints_defaults = 'comma' +typehints_defaults = "comma" typehints_use_rtype = False -sciline_domain_types_prefix = 'ess.nmx' + +sciline_domain_types_prefix = "ess.nmx" sciline_domain_types_aliases = { - 'scipp._scipp.core.DataArray': 'scipp.DataArray', - 'scipp._scipp.core.Dataset': 'scipp.Dataset', - 'scipp._scipp.core.DType': 'scipp.DType', - 'scipp._scipp.core.Unit': 'scipp.Unit', - 'scipp._scipp.core.Variable': 'scipp.Variable', - 'scipp.core.data_group.DataGroup': 'scipp.DataGroup', + "scipp._scipp.core.DataArray": "scipp.DataArray", + "scipp._scipp.core.Dataset": "scipp.Dataset", + "scipp._scipp.core.DType": "scipp.DType", + "scipp._scipp.core.Unit": "scipp.Unit", + "scipp._scipp.core.Variable": "scipp.Variable", + "scipp.core.data_group.DataGroup": "scipp.DataGroup", } + # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # -source_suffix = ['.rst', '.md'] -html_sourcelink_suffix = '' # Avoid .ipynb.txt extensions in sources +source_suffix = [".rst", ".md"] +html_sourcelink_suffix = "" # Avoid .ipynb.txt extensions in sources # The master toctree document. -master_doc = 'index' +master_doc = "index" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # -# The short X.Y version. -version = nmx.__version__ -# The full version, including alpha/beta/rc tags. -release = nmx.__version__ +try: + release = get_version("essnmx") + version = ".".join(release.split(".")[:3]) # CalVer +except PackageNotFoundError: + logger.info( + "Warning: determining version from package metadata failed, falling back to " + "a dummy version number." + ) + release = version = "0.0.0-dev" warning_is_error = True @@ -125,10 +138,10 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '**.ipynb_checkpoints'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**.ipynb_checkpoints"] # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -192,14 +205,14 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] html_css_files = [] html_js_files = ["anaconda-icon.js"] # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. -htmlhelp_basename = 'essnmxdoc' +htmlhelp_basename = "essnmxdoc" # -- Options for Matplotlib in notebooks ---------------------------------- @@ -215,7 +228,7 @@ # In addition, there is no need to make plots in doctest as the documentation # build already tests if those plots can be made. # So we simply disable plots in doctests. -doctest_global_setup = ''' +doctest_global_setup = """ import numpy as np try: @@ -232,7 +245,7 @@ def do_not_plot(*args, **kwargs): except ImportError: # Scipp is not needed by docs if it is not installed. pass -''' +""" # Using normalize whitespace because many __str__ functions in scipp produce # extraneous empty lines and it would look strange to include them in the docs. @@ -247,5 +260,5 @@ def do_not_plot(*args, **kwargs): linkcheck_ignore = [ # Specific lines in Github blobs cannot be found by linkcheck. - r'https?://github\.com/.*?/blob/[a-f0-9]+/.+?#', + r"https?://github\.com/.*?/blob/[a-f0-9]+/.+?#", ] diff --git a/packages/essnmx/docs/developer/coding-conventions.md b/packages/essnmx/docs/developer/coding-conventions.md index b23c0eb4..4fafc18d 100644 --- a/packages/essnmx/docs/developer/coding-conventions.md +++ b/packages/essnmx/docs/developer/coding-conventions.md @@ -2,7 +2,7 @@ ## Code formatting -There are no explicit code formatting conventions since we use `black` to enforce a format. +There are no explicit code formatting conventions since we use `ruff` to enforce a format. ## Docstring format diff --git a/packages/essnmx/docs/developer/index.md b/packages/essnmx/docs/developer/index.md index 4910390a..d183a4ca 100644 --- a/packages/essnmx/docs/developer/index.md +++ b/packages/essnmx/docs/developer/index.md @@ -1,4 +1,4 @@ -# Developer documentation +# Development ```{include} ../../CONTRIBUTING.md ``` From 6e2f5b2781243724e304e40f7474ba076a1bf803 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 4 Jul 2024 14:16:33 +0200 Subject: [PATCH 206/403] Update module file. --- packages/essnmx/src/ess/nmx/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/__init__.py b/packages/essnmx/src/ess/nmx/__init__.py index dddbbaec..d481b0a5 100644 --- a/packages/essnmx/src/ess/nmx/__init__.py +++ b/packages/essnmx/src/ess/nmx/__init__.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +# ruff: noqa: E402, F401 -# flake8: noqa import importlib.metadata try: From 541259765bea5ed6799c6a27af1cf8059f1ad70e Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 4 Jul 2024 14:16:46 +0200 Subject: [PATCH 207/403] Update package configurations. --- packages/essnmx/conda/meta.yaml | 11 ++++++++ packages/essnmx/pyproject.toml | 45 +++++++++++++++++++++++++-------- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/packages/essnmx/conda/meta.yaml b/packages/essnmx/conda/meta.yaml index 40645fb8..1043461f 100644 --- a/packages/essnmx/conda/meta.yaml +++ b/packages/essnmx/conda/meta.yaml @@ -6,6 +6,11 @@ package: source: path: .. + +{% set pyproject = load_file_data('pyproject.toml') %} +{% set dependencies = pyproject.get('project', {}).get('dependencies', {}) %} + + requirements: build: - setuptools @@ -23,6 +28,12 @@ requirements: - gemmi - pandas + {# Conda does not allow spaces between package name and version, so remove them #} + {% for package in dependencies %} + - {% if package == "graphviz" %}python-graphviz{% else %}{{ package|replace(" ", "") }}{% endif %} + {% endfor %} + + test: imports: - ess.nmx diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 43c1efa9..d5384048 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -65,17 +65,43 @@ filterwarnings = [ "error", ] -[tool.bandit] -# Excluding tests because bandit doesn't like `assert`. -exclude_dirs = ["docs/conf.py", "tests"] +[tool.ruff] +line-length = 88 +extend-include = ["*.ipynb"] +extend-exclude = [ + ".*", "__pycache__", "build", "dist", "install", +] + +[tool.ruff.lint] +# See https://docs.astral.sh/ruff/rules/ +select = ["B", "C4", "DTZ", "E", "F", "G", "I", "PERF", "PGH", "PT", "PYI", "RUF", "S", "T20", "UP", "W"] +ignore = [ + # Conflict with ruff format, see + # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules + "COM812", "COM819", "D206", "D300", "E111", "E114", "E117", "ISC001", "ISC002", "Q000", "Q001", "Q002", "Q003", "W191", +] +fixable = ["I001", "B010"] +isort.known-first-party = ["essnmx"] +pydocstyle.convention = "numpy" -[tool.black] -skip-string-normalization = true +[tool.ruff.lint.per-file-ignores] +# those files have an increased risk of relying on import order +"__init__.py" = ["I"] +"tests/*" = [ + "S101", # asserts are fine in tests + "B018", # 'useless expressions' are ok because some tests just check for exceptions +] +"*.ipynb" = [ + "E501", # longer lines are sometimes more readable + "F403", # *-imports used with domain types + "F405", # linter may fail to find names because of *-imports + "I", # we don't collect imports at the top + "S101", # asserts are used for demonstration and are safe in notebooks + "T201", # printing is ok for demonstration purposes +] -[tool.isort] -skip_gitignore = true -profile = "black" -known_first_party = ["essnmx"] +[tool.ruff.format] +quote-style = "preserve" [tool.mypy] strict = true @@ -85,5 +111,4 @@ enable_error_code = [ "redundant-expr", "truthy-bool", ] -show_error_codes = true warn_unreachable = true From f7358ae96eff023153ba07027a64e195c3d10fc5 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 4 Jul 2024 14:17:16 +0200 Subject: [PATCH 208/403] Add arguments to tox commands. --- packages/essnmx/tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/tox.ini b/packages/essnmx/tox.ini index d65754ab..b4f9ce2a 100644 --- a/packages/essnmx/tox.ini +++ b/packages/essnmx/tox.ini @@ -10,14 +10,14 @@ commands = pytest {posargs} [testenv:nightly] deps = -r requirements/nightly.txt -commands = pytest +commands = pytest {posargs} [testenv:unpinned] description = Test with unpinned dependencies, as a user would install now. deps = -r requirements/basetest.txt essnmx -commands = pytest +commands = pytest {posargs} [testenv:docs] description = invoke sphinx-build to build the HTML docs From df6a9204ba3e62dda808288b196f7a7314fca81c Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 4 Jul 2024 14:17:26 +0200 Subject: [PATCH 209/403] Update requirements. --- packages/essnmx/requirements/base.txt | 14 +++++++------- packages/essnmx/requirements/ci.txt | 4 ++-- packages/essnmx/requirements/dev.txt | 10 +++++----- packages/essnmx/requirements/docs.txt | 12 +++++++----- packages/essnmx/requirements/make_base.py | 6 +----- packages/essnmx/requirements/mypy.txt | 2 +- packages/essnmx/requirements/nightly.txt | 12 ++++++------ 7 files changed, 29 insertions(+), 31 deletions(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 1895bdc3..0aa5ccda 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -5,7 +5,7 @@ # # pip-compile-multi # -certifi==2024.6.2 +certifi==2024.7.4 # via requests charset-normalizer==3.3.2 # via requests @@ -25,7 +25,7 @@ defusedxml==0.7.1 # via -r base.in fonttools==4.53.0 # via matplotlib -fsspec==2024.6.0 +fsspec==2024.6.1 # via dask gemmi==0.6.6 # via -r base.in @@ -35,13 +35,13 @@ h5py==3.11.0 # via scippnexus idna==3.7 # via requests -importlib-metadata==7.2.1 +importlib-metadata==8.0.0 # via dask kiwisolver==1.4.5 # via matplotlib locket==1.0.0 # via partd -matplotlib==3.9.0 +matplotlib==3.9.1 # via plopp networkx==3.3 # via cyclebane @@ -62,7 +62,7 @@ pandas==2.2.2 # via -r base.in partd==1.4.2 # via dask -pillow==10.3.0 +pillow==10.4.0 # via matplotlib platformdirs==4.2.2 # via pooch @@ -83,7 +83,7 @@ pyyaml==6.0.1 # via dask requests==2.32.3 # via pooch -sciline==24.6.1 +sciline==24.6.2 # via -r base.in scipp==24.6.0 # via @@ -91,7 +91,7 @@ scipp==24.6.0 # scippnexus scippnexus==24.6.0 # via -r base.in -scipy==1.13.1 +scipy==1.14.0 # via scippnexus six==1.16.0 # via python-dateutil diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index 0ef98e16..4d81cbd2 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -7,7 +7,7 @@ # cachetools==5.3.3 # via tox -certifi==2024.6.2 +certifi==2024.7.4 # via requests chardet==5.2.0 # via tox @@ -48,7 +48,7 @@ tomli==2.0.1 # via # pyproject-api # tox -tox==4.15.1 +tox==4.16.0 # via -r ci.in urllib3==2.2.2 # via requests diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index 077dbfc5..f0fd0a82 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -28,9 +28,9 @@ async-lru==2.0.4 # via jupyterlab cffi==1.16.0 # via argon2-cffi-bindings -copier==9.2.0 +copier==9.3.0 # via -r dev.in -dunamai==1.21.1 +dunamai==1.21.2 # via copier fqdn==1.5.1 # via jsonschema @@ -67,7 +67,7 @@ jupyter-server==2.14.1 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.2.2 +jupyterlab==4.2.3 # via -r dev.in jupyterlab-server==2.27.2 # via jupyterlab @@ -87,9 +87,9 @@ prometheus-client==0.20.0 # via jupyter-server pycparser==2.22 # via cffi -pydantic==2.7.4 +pydantic==2.8.2 # via copier -pydantic-core==2.18.4 +pydantic-core==2.20.1 # via pydantic python-json-logger==2.0.7 # via jupyter-events diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 437dd76d..52b1ff3e 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -30,7 +30,7 @@ comm==0.2.2 # via # ipykernel # ipywidgets -debugpy==1.8.1 +debugpy==1.8.2 # via ipykernel decorator==5.1.1 # via ipython @@ -50,9 +50,9 @@ imagesize==1.4.1 # via sphinx ipydatawidgets==4.3.5 # via pythreejs -ipykernel==6.29.4 +ipykernel==6.29.5 # via -r docs.in -ipython==8.25.0 +ipython==8.26.0 # via # -r docs.in # ipykernel @@ -106,6 +106,8 @@ mdurl==0.1.2 # via markdown-it-py mistune==3.0.2 # via nbconvert +mpltoolbox==24.5.1 + # via scippneutron myst-parser==3.0.1 # via -r docs.in nbclient==0.10.0 @@ -135,7 +137,7 @@ ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 # via stack-data -pydata-sphinx-theme==0.15.3 +pydata-sphinx-theme==0.15.4 # via -r docs.in pygments==2.18.0 # via @@ -158,7 +160,7 @@ rpds-py==0.18.1 # via # jsonschema # referencing -scippneutron==24.5.0 +scippneutron==24.6.0 # via -r docs.in snowballstemmer==2.2.0 # via sphinx diff --git a/packages/essnmx/requirements/make_base.py b/packages/essnmx/requirements/make_base.py index 3b1dbabc..68a17e84 100644 --- a/packages/essnmx/requirements/make_base.py +++ b/packages/essnmx/requirements/make_base.py @@ -1,10 +1,6 @@ -# SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2023 Scipp contributors (https://github.com/scipp) - import sys from argparse import ArgumentParser from pathlib import Path -from typing import List import tomli @@ -23,7 +19,7 @@ """ -def write_dependencies(dependency_name: str, dependencies: List[str]) -> None: +def write_dependencies(dependency_name: str, dependencies: list[str]) -> None: path = Path(f"{dependency_name}.in") if path.exists(): sections = path.read_text().split(CUSTOM_AUTO_SEPARATOR) diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index 3644c507..69cdcbc0 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -6,7 +6,7 @@ # pip-compile-multi # -r test.txt -mypy==1.10.0 +mypy==1.10.1 # via -r mypy.in mypy-extensions==1.0.0 # via mypy diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 2a8fc9da..9acbc135 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -6,7 +6,7 @@ # pip-compile-multi # -r basetest.txt -certifi==2024.6.2 +certifi==2024.7.4 # via requests charset-normalizer==3.3.2 # via requests @@ -26,7 +26,7 @@ defusedxml==0.7.1 # via -r nightly.in fonttools==4.53.0 # via matplotlib -fsspec==2024.6.0 +fsspec==2024.6.1 # via dask gemmi==0.6.6 # via -r nightly.in @@ -36,13 +36,13 @@ h5py==3.11.0 # via scippnexus idna==3.7 # via requests -importlib-metadata==7.2.1 +importlib-metadata==8.0.0 # via dask kiwisolver==1.4.5 # via matplotlib locket==1.0.0 # via partd -matplotlib==3.9.0 +matplotlib==3.9.1 # via plopp networkx==3.3 # via cyclebane @@ -58,7 +58,7 @@ pandas==2.2.2 # via -r nightly.in partd==1.4.2 # via dask -pillow==10.3.0 +pillow==10.4.0 # via matplotlib platformdirs==4.2.2 # via pooch @@ -87,7 +87,7 @@ scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-c # scippnexus scippnexus @ git+https://github.com/scipp/scippnexus@main # via -r nightly.in -scipy==1.13.1 +scipy==1.14.0 # via scippnexus six==1.16.0 # via python-dateutil From ae32cb0efa45c99f7d232b2f06d597f87758c0a5 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 4 Jul 2024 14:20:24 +0200 Subject: [PATCH 210/403] Update copier. --- packages/essnmx/.copier-answers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/.copier-answers.yml b/packages/essnmx/.copier-answers.yml index 16beda4a..b7f49d67 100644 --- a/packages/essnmx/.copier-answers.yml +++ b/packages/essnmx/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY -_commit: 6770edb +_commit: 6848c57 _src_path: gh:scipp/copier_template description: Data reduction for NMX at the European Spallation Source. max_python: '3.12' From 8d55aa22cc310ab56525935bb3fac99c7436d9e3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Thu, 4 Jul 2024 12:22:12 +0000 Subject: [PATCH 211/403] Apply automatic formatting --- packages/essnmx/src/ess/nmx/mcstas/xml.py | 9 +++------ packages/essnmx/tests/exporter_test.py | 1 - packages/essnmx/tests/loader_test.py | 3 +-- packages/essnmx/tests/mtz_io_test.py | 3 +-- packages/essnmx/tests/scaling_test.py | 1 - packages/essnmx/tests/workflow_test.py | 1 - 6 files changed, 5 insertions(+), 13 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas/xml.py b/packages/essnmx/src/ess/nmx/mcstas/xml.py index 3aa43aae..1c1b81ed 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas/xml.py @@ -33,14 +33,11 @@ class _XML(Protocol): tag: str attrib: dict[str, str] - def find(self, name: str) -> Optional['_XML']: - ... + def find(self, name: str) -> Optional['_XML']: ... - def __iter__(self) -> '_XML': - ... + def __iter__(self) -> '_XML': ... - def __next__(self) -> '_XML': - ... + def __next__(self) -> '_XML': ... def _check_and_unpack_if_only_one(xml_items: list[_XML], name: str) -> _XML: diff --git a/packages/essnmx/tests/exporter_test.py b/packages/essnmx/tests/exporter_test.py index 4e772f98..087c657e 100644 --- a/packages/essnmx/tests/exporter_test.py +++ b/packages/essnmx/tests/exporter_test.py @@ -5,7 +5,6 @@ import numpy as np import pytest import scipp as sc - from ess.nmx.nexus import export_as_nexus from ess.nmx.reduction import NMXReducedData diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 55b06d06..6cd11401 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -8,8 +8,6 @@ import sciline as sl import scipp as sc import scippnexus as snx -from scipp.testing import assert_allclose, assert_identical - from ess.nmx import default_parameters from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample from ess.nmx.mcstas.load import bank_names_to_detector_names @@ -21,6 +19,7 @@ FilePath, MaximumProbability, ) +from scipp.testing import assert_allclose, assert_identical sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) from mcstas_description_examples import ( # noqa: E402 diff --git a/packages/essnmx/tests/mtz_io_test.py b/packages/essnmx/tests/mtz_io_test.py index b882ce69..159bb502 100644 --- a/packages/essnmx/tests/mtz_io_test.py +++ b/packages/essnmx/tests/mtz_io_test.py @@ -5,11 +5,10 @@ import gemmi import pytest import scipp as sc - from ess.nmx import mtz_io from ess.nmx.data import get_small_mtz_samples -from ess.nmx.mtz_io import DEFAULT_SPACE_GROUP_DESC # P 21 21 21 from ess.nmx.mtz_io import ( + DEFAULT_SPACE_GROUP_DESC, # P 21 21 21 MtzDataFrame, MTZFileIndex, MTZFilePath, diff --git a/packages/essnmx/tests/scaling_test.py b/packages/essnmx/tests/scaling_test.py index 219603bc..499397b4 100644 --- a/packages/essnmx/tests/scaling_test.py +++ b/packages/essnmx/tests/scaling_test.py @@ -2,7 +2,6 @@ # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) import pytest import scipp as sc - from ess.nmx.scaling import ( ReferenceIntensities, estimate_scale_factor_per_hkl_asu_from_reference, diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index cf8e14d0..596dbb91 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -4,7 +4,6 @@ import pytest import sciline as sl import scipp as sc - from ess.nmx import default_parameters from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample from ess.nmx.mcstas.load import providers as load_providers From 1f5aee3967e0799b498b63c2dbe8c2ee7c27835b Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 4 Jul 2024 14:36:47 +0200 Subject: [PATCH 212/403] Remove unecessary setup.cfg file. --- packages/essnmx/setup.cfg | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 packages/essnmx/setup.cfg diff --git a/packages/essnmx/setup.cfg b/packages/essnmx/setup.cfg deleted file mode 100644 index 1ba190c5..00000000 --- a/packages/essnmx/setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[flake8] -# See https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#line-length -max-line-length = 88 -extend-ignore = E203 From bc25a50f4fc4fa791db55562e690a378d0ccf453 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 4 Jul 2024 14:29:03 +0200 Subject: [PATCH 213/403] Fix PT* --- packages/essnmx/tests/conftest.py | 2 +- packages/essnmx/tests/exporter_test.py | 4 ++-- packages/essnmx/tests/loader_test.py | 16 ++++++++-------- packages/essnmx/tests/mtz_io_test.py | 12 ++++++------ packages/essnmx/tests/scaling_test.py | 4 ++-- packages/essnmx/tests/workflow_test.py | 4 ++-- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/packages/essnmx/tests/conftest.py b/packages/essnmx/tests/conftest.py index 33d8e057..2077dcbd 100644 --- a/packages/essnmx/tests/conftest.py +++ b/packages/essnmx/tests/conftest.py @@ -8,6 +8,6 @@ import pytest -@pytest.fixture +@pytest.fixture()() def mcstas_2_deprecation_warning_context() -> partial[AbstractContextManager]: return partial(pytest.warns, DeprecationWarning, match="McStas") diff --git a/packages/essnmx/tests/exporter_test.py b/packages/essnmx/tests/exporter_test.py index 087c657e..aab78aad 100644 --- a/packages/essnmx/tests/exporter_test.py +++ b/packages/essnmx/tests/exporter_test.py @@ -9,7 +9,7 @@ from ess.nmx.reduction import NMXReducedData -@pytest.fixture +@pytest.fixture() def reduced_data() -> NMXReducedData: rng = np.random.default_rng(42) id_list = sc.array(dims=['event'], values=rng.integers(0, 12, size=100)) @@ -25,7 +25,7 @@ def reduced_data() -> NMXReducedData: return NMXReducedData( sc.DataGroup( - dict( + dict( # noqa: C408 counts=counts, proton_charge=sc.scalar(1.0, unit='counts'), crystal_rotation=sc.vector(value=[0.0, 20.0, 0.0], unit='deg'), diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 6cd11401..cd758f66 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -2,7 +2,7 @@ # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) import pathlib import sys -from typing import Generator +from collections.abc import Generator import pytest import sciline as sl @@ -22,7 +22,7 @@ from scipp.testing import assert_allclose, assert_identical sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) -from mcstas_description_examples import ( # noqa: E402 +from mcstas_description_examples import ( no_detectors, one_detector_no_filename, two_detectors_same_filename, @@ -64,14 +64,14 @@ def check_nmxdata_properties(dg: NMXData, fast_axis, slow_axis) -> None: @pytest.mark.parametrize( - 'detector_index, fast_axis, slow_axis', - ( + ('detector_index', 'fast_axis', 'slow_axis'), + [ # Expected values are provided by the IDS # based on the simulation settings of the sample file. (0, (1.0, 0.0, -0.01), (0.0, 1.0, 0.0)), (1, (-0.01, 0.0, -1.0), (0.0, 1.0, 0.0)), (2, (0.01, 0.0, 1.0), (0.0, 1.0, 0.0)), - ), + ], ) def test_file_reader_mcstas2( detector_index, fast_axis, slow_axis, mcstas_2_deprecation_warning_context @@ -114,14 +114,14 @@ def check_scalar_properties_mcstas_3(dg: NMXData): @pytest.mark.parametrize( - 'detector_index, fast_axis, slow_axis', - ( + ('detector_index', 'fast_axis', 'slow_axis'), + [ # Expected values are provided by the IDS # based on the simulation settings of the sample file. (0, (1.0, 0.0, -0.01), (0.0, 1.0, 0.0)), (1, (-0.01, 0.0, -1.0), (0.0, 1.0, 0.0)), (2, (0.01, 0.0, 1.0), (0.0, 1.0, 0.0)), - ), + ], ) def test_file_reader_mcstas3(detector_index, fast_axis, slow_axis) -> None: file_path = small_mcstas_3_sample() diff --git a/packages/essnmx/tests/mtz_io_test.py b/packages/essnmx/tests/mtz_io_test.py index 159bb502..4887d02c 100644 --- a/packages/essnmx/tests/mtz_io_test.py +++ b/packages/essnmx/tests/mtz_io_test.py @@ -34,7 +34,7 @@ def test_gemmi_mtz(file_path: pathlib.Path) -> None: assert len(mtz.columns[0]) == 100 # Number of samples, hard-coded value -@pytest.fixture +@pytest.fixture() def gemmi_mtz_object(file_path: pathlib.Path) -> gemmi.Mtz: return read_mtz_file(MTZFilePath(file_path)) @@ -64,7 +64,7 @@ def test_mtz_to_process_pandas_dataframe(gemmi_mtz_object: gemmi.Mtz) -> None: assert "hkl_asu" not in df.columns # It should be done on merged dataframes -@pytest.fixture +@pytest.fixture() def mtz_list() -> list[gemmi.Mtz]: return [ read_mtz_file(MTZFilePath(file_path)) for file_path in get_small_mtz_samples() @@ -78,7 +78,7 @@ def test_get_space_group_with_spacegroup_desc() -> None: ) -@pytest.fixture +@pytest.fixture() def conflicting_mtz_series( mtz_list: list[gemmi.Mtz], ) -> list[gemmi.Mtz]: @@ -103,14 +103,14 @@ def test_get_unique_space_group_raises_on_conflict( mtz_io.get_unique_space_group(*space_groups) -@pytest.fixture +@pytest.fixture() def merged_mtz_dataframe(mtz_list: list[gemmi.Mtz]) -> MtzDataFrame: """Tests if the merged data frame has the expected columns.""" reduced_mtz = [process_single_mtz_to_dataframe(mtz) for mtz in mtz_list] return mtz_io.merge_mtz_dataframes(*reduced_mtz) -@pytest.fixture +@pytest.fixture() def nmx_data_frame( mtz_list: list[gemmi.Mtz], merged_mtz_dataframe: MtzDataFrame, @@ -134,7 +134,7 @@ def test_process_merged_mtz_dataframe( assert "hkl_asu" in nmx_data_frame.columns -@pytest.fixture +@pytest.fixture() def nmx_data_array(nmx_data_frame: NMXMtzDataFrame) -> NMXMtzDataArray: return nmx_mtz_dataframe_to_scipp_dataarray(nmx_data_frame) diff --git a/packages/essnmx/tests/scaling_test.py b/packages/essnmx/tests/scaling_test.py index 499397b4..0c5fa34c 100644 --- a/packages/essnmx/tests/scaling_test.py +++ b/packages/essnmx/tests/scaling_test.py @@ -10,7 +10,7 @@ ) -@pytest.fixture +@pytest.fixture() def nmx_data_array() -> sc.DataArray: da = sc.DataArray( data=sc.array(dims=["row"], values=[1, 2, 3, 4, 5, 3.1, 3.2]), @@ -52,7 +52,7 @@ def test_get_reference_bin_middle(nmx_data_array: sc.DataArray) -> None: ) -@pytest.fixture +@pytest.fixture() def reference_bin(nmx_data_array: sc.DataArray) -> ReferenceIntensities: binned = nmx_data_array.bin({"wavelength": 6}) reference_wavelength = get_reference_wavelength(binned, reference_wavelength=None) diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index 596dbb91..6f38c040 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -22,7 +22,7 @@ def mcstas_file_path( return request.param() -@pytest.fixture +@pytest.fixture() def mcstas_workflow(mcstas_file_path: str) -> sl.Pipeline: return sl.Pipeline( [*load_providers, bin_time_of_arrival], @@ -34,7 +34,7 @@ def mcstas_workflow(mcstas_file_path: str) -> sl.Pipeline: ) -@pytest.fixture +@pytest.fixture() def multi_bank_mcstas_workflow(mcstas_workflow: sl.Pipeline) -> sl.Pipeline: pl = mcstas_workflow.copy() pl[NMXReducedData] = ( From 955bb861dfe53dbac186b795e5f95f0f969d67ba Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Thu, 4 Jul 2024 14:36:22 +0200 Subject: [PATCH 214/403] Fix all errors. --- packages/essnmx/docs/developer/test-dataset.ipynb | 13 ++++++++----- packages/essnmx/src/ess/nmx/mcstas/load.py | 3 +-- packages/essnmx/src/ess/nmx/mcstas/xml.py | 15 ++++++++------- packages/essnmx/src/ess/nmx/nexus.py | 11 +++++------ packages/essnmx/src/ess/nmx/scaling.py | 8 ++++---- packages/essnmx/tests/conftest.py | 2 +- 6 files changed, 27 insertions(+), 25 deletions(-) diff --git a/packages/essnmx/docs/developer/test-dataset.ipynb b/packages/essnmx/docs/developer/test-dataset.ipynb index 3811f7fd..96c7e252 100644 --- a/packages/essnmx/docs/developer/test-dataset.ipynb +++ b/packages/essnmx/docs/developer/test-dataset.ipynb @@ -48,13 +48,16 @@ " DEFAULT_STD_DEV_COLUMN_NAME, # SIGI\n", ")\n", "global_rng = np.random.default_rng(0)\n", - "HKL_CANDIDATES = tuple(zip(*[global_rng.integers(*HKL_RANGE, size=100) for _ in range(3)]))\n", + "HKL_CANDIDATES = tuple(\n", + " zip(*[global_rng.integers(*HKL_RANGE, size=100) for _ in range(3)], strict=False)\n", + ")\n", "\n", "def create_mtz_data_frame(random_seed: int) -> pd.DataFrame:\n", " rng = np.random.default_rng(random_seed)\n", - " intensities = np.sort(rng.normal(50, 20, size=10_000))[::-1] + (rng.uniform(\n", - " *INTENSITY_RANGE, size=10_000\n", - " )* rng.choice([0]*99 + [1], size=10_000))\n", + " intensities = np.sort(rng.normal(50, 20, size=10_000))[::-1] + (\n", + " rng.uniform(*INTENSITY_RANGE, size=10_000)\n", + " * rng.choice([0] * 99 + [1], size=10_000)\n", + " )\n", " std_devs = np.multiply(intensities, rng.uniform(0.1, 0.15, size=10_000))\n", " wavelengths = np.sort(rng.uniform(2.8, 3.2, size=10_000))[::-1]\n", "\n", @@ -65,7 +68,7 @@ " DEFAULT_WAVELENGTH_COLUMN_NAME: wavelengths,\n", " }\n", " )\n", - " \n", + "\n", " df[[\"H\", \"K\", \"L\"]] = pd.Series(\n", " rng.choice(HKL_CANDIDATES, size=10_000).tolist()\n", " ).to_list()\n", diff --git a/packages/essnmx/src/ess/nmx/mcstas/load.py b/packages/essnmx/src/ess/nmx/mcstas/load.py index 414a87f1..53d6c184 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/load.py +++ b/packages/essnmx/src/ess/nmx/mcstas/load.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) import re -from typing import Dict, List import scipp as sc import scippnexus as snx @@ -117,7 +116,7 @@ def proton_charge_from_event_data(da: EventData) -> ProtonCharge: return ProtonCharge(sc.scalar(1 / 10_000, unit=None) * da.bins.size().sum().data) -def bank_names_to_detector_names(description: str) -> Dict[str, List[str]]: +def bank_names_to_detector_names(description: str) -> dict[str, list[str]]: """Associates event data names with the names of the detectors where the events were detected""" diff --git a/packages/essnmx/src/ess/nmx/mcstas/xml.py b/packages/essnmx/src/ess/nmx/mcstas/xml.py index 1c1b81ed..f21188a7 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas/xml.py @@ -1,9 +1,10 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) # McStas instrument geometry xml description related functions. +from collections.abc import Iterable from dataclasses import dataclass from types import MappingProxyType -from typing import Iterable, Optional, Protocol, Tuple, TypeVar +from typing import Protocol, TypeVar import h5py import scipp as sc @@ -33,7 +34,7 @@ class _XML(Protocol): tag: str attrib: dict[str, str] - def find(self, name: str) -> Optional['_XML']: ... + def find(self, name: str) -> '_XML | None': ... def __iter__(self) -> '_XML': ... @@ -210,7 +211,7 @@ def num_fast_pixels_per_row(self) -> int: return self.num_x if self.fast_axis_name == 'x' else self.num_y -def _collect_detector_descriptions(tree: _XML) -> Tuple[DetectorDesc, ...]: +def _collect_detector_descriptions(tree: _XML) -> tuple[DetectorDesc, ...]: """Retrieve detector geometry descriptions from mcstas file.""" type_list = list(filter_by_tag(tree, 'type')) simulation_settings = SimulationSettings.from_xml(tree) @@ -245,7 +246,7 @@ class SampleDesc: name: str # From under position: sc.Variable - rotation_matrix: Optional[sc.Variable] + rotation_matrix: sc.Variable | None @classmethod def from_xml( @@ -307,7 +308,7 @@ def from_xml( ) -def _construct_pixel_ids(detector_descs: Tuple[DetectorDesc, ...]) -> sc.Variable: +def _construct_pixel_ids(detector_descs: tuple[DetectorDesc, ...]) -> sc.Variable: """Pixel IDs for all detectors.""" intervals = [ (desc.id_start, desc.id_start + desc.total_pixels) for desc in detector_descs @@ -340,7 +341,7 @@ def _pixel_positions( def _detector_pixel_positions( - detector_descs: Tuple[DetectorDesc, ...], sample: SampleDesc + detector_descs: tuple[DetectorDesc, ...], sample: SampleDesc ) -> sc.Variable: """Position of pixels of all detectors.""" positions = [ @@ -353,7 +354,7 @@ def _detector_pixel_positions( @dataclass class McStasInstrument: simulation_settings: SimulationSettings - detectors: Tuple[DetectorDesc, ...] + detectors: tuple[DetectorDesc, ...] source: SourceDesc sample: SampleDesc diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/nexus.py index 1944800c..0373db77 100644 --- a/packages/essnmx/src/ess/nmx/nexus.py +++ b/packages/essnmx/src/ess/nmx/nexus.py @@ -3,7 +3,6 @@ import io import pathlib from functools import partial -from typing import Optional, Union import h5py import scipp as sc @@ -14,11 +13,11 @@ def _create_dataset_from_var( root_entry: h5py.Group, var: sc.Variable, name: str, - long_name: Optional[str] = None, - compression: Optional[str] = None, - compression_opts: Optional[int] = None, + long_name: str | None = None, + compression: str | None = None, + compression_opts: int | None = None, ) -> h5py.Dataset: - compression_options = dict() + compression_options = {} if compression is not None: compression_options["compression"] = compression if compression_opts is not None: @@ -124,7 +123,7 @@ def _create_source_group(data: sc.DataGroup, nx_entry: h5py.Group) -> h5py.Group def export_as_nexus( - data: sc.DataGroup, output_file: Union[str, pathlib.Path, io.BytesIO] + data: sc.DataGroup, output_file: str | pathlib.Path | io.BytesIO ) -> None: """Export the reduced data to a NeXus file. diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index cf0383ed..5d135b6f 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -2,14 +2,14 @@ # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) from collections.abc import Callable, Mapping from dataclasses import dataclass -from typing import NewType, TypeVar, Union +from typing import NewType, TypeVar import scipp as sc from .mtz_io import NMXMtzDataArray # User defined or configurable types -WavelengthBins = NewType("WavelengthBins", Union[sc.Variable, int]) +WavelengthBins = NewType("WavelengthBins", sc.Variable | int) """User configurable wavelength binning""" ReferenceWavelength = NewType("ReferenceWavelength", sc.Variable | None) """The wavelength to select reference intensities.""" @@ -130,8 +130,8 @@ def get_reference_intensities( raise ValueError("Reference wavelength should be a scalar.") try: return binned["wavelength", reference_wavelength].values.copy(deep=False) - except IndexError: - raise IndexError(f"{reference_wavelength} out of range.") + except IndexError as err: + raise IndexError(f"{reference_wavelength} out of range.") from err def estimate_scale_factor_per_hkl_asu_from_reference( diff --git a/packages/essnmx/tests/conftest.py b/packages/essnmx/tests/conftest.py index 2077dcbd..cebcd9ef 100644 --- a/packages/essnmx/tests/conftest.py +++ b/packages/essnmx/tests/conftest.py @@ -8,6 +8,6 @@ import pytest -@pytest.fixture()() +@pytest.fixture() def mcstas_2_deprecation_warning_context() -> partial[AbstractContextManager]: return partial(pytest.warns, DeprecationWarning, match="McStas") From ccd28f5eb2ff2f682ddce44ba30d475dfef8c49d Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 23 Jul 2024 13:35:38 +0200 Subject: [PATCH 215/403] ci: update copier --- packages/essnmx/.copier-answers.yml | 2 +- packages/essnmx/.github/workflows/ci.yml | 2 ++ packages/essnmx/.github/workflows/docs.yml | 6 +++++- packages/essnmx/.github/workflows/nightly_at_main.yml | 1 + packages/essnmx/.github/workflows/nightly_at_release.yml | 1 + packages/essnmx/.github/workflows/test.yml | 3 +++ packages/essnmx/.github/workflows/unpinned.yml | 1 + packages/essnmx/.gitignore | 1 + packages/essnmx/requirements/make_base.py | 3 ++- 9 files changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/essnmx/.copier-answers.yml b/packages/essnmx/.copier-answers.yml index b7f49d67..ef43bf02 100644 --- a/packages/essnmx/.copier-answers.yml +++ b/packages/essnmx/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY -_commit: 6848c57 +_commit: 101e594 _src_path: gh:scipp/copier_template description: Data reduction for NMX at the European Spallation Source. max_python: '3.12' diff --git a/packages/essnmx/.github/workflows/ci.yml b/packages/essnmx/.github/workflows/ci.yml index 44266a23..8234bc9f 100644 --- a/packages/essnmx/.github/workflows/ci.yml +++ b/packages/essnmx/.github/workflows/ci.yml @@ -46,6 +46,7 @@ jobs: os-variant: ${{ matrix.os }} python-version: ${{ matrix.python.version }} tox-env: ${{ matrix.python.tox-env }} + secrets: inherit docs: needs: tests @@ -54,3 +55,4 @@ jobs: publish: false linkcheck: ${{ contains(matrix.variant.os, 'ubuntu') && github.ref == 'refs/heads/main' }} branch: ${{ github.head_ref == '' && github.ref_name || github.head_ref }} + secrets: inherit diff --git a/packages/essnmx/.github/workflows/docs.yml b/packages/essnmx/.github/workflows/docs.yml index 98aaf568..a5ea2b05 100644 --- a/packages/essnmx/.github/workflows/docs.yml +++ b/packages/essnmx/.github/workflows/docs.yml @@ -42,6 +42,10 @@ jobs: docs: name: Build documentation runs-on: 'ubuntu-22.04' + env: + ESS_PROTECTED_FILESTORE_USERNAME: ${{ secrets.ESS_PROTECTED_FILESTORE_USERNAME }} + ESS_PROTECTED_FILESTORE_PASSWORD: ${{ secrets.ESS_PROTECTED_FILESTORE_PASSWORD }} + steps: - run: sudo apt install --yes graphviz pandoc - uses: actions/checkout@v4 @@ -65,7 +69,7 @@ jobs: name: docs_html path: html/ - - uses: JamesIves/github-pages-deploy-action@v4.6.1 + - uses: JamesIves/github-pages-deploy-action@v4.6.3 if: ${{ inputs.publish }} with: branch: gh-pages diff --git a/packages/essnmx/.github/workflows/nightly_at_main.yml b/packages/essnmx/.github/workflows/nightly_at_main.yml index 08fdddd2..c2b9d33a 100644 --- a/packages/essnmx/.github/workflows/nightly_at_main.yml +++ b/packages/essnmx/.github/workflows/nightly_at_main.yml @@ -31,3 +31,4 @@ jobs: os-variant: ${{ matrix.os }} python-version: ${{ matrix.python.version }} tox-env: ${{ matrix.python.tox-env }} + secrets: inherit diff --git a/packages/essnmx/.github/workflows/nightly_at_release.yml b/packages/essnmx/.github/workflows/nightly_at_release.yml index 373c4546..3faa1c23 100644 --- a/packages/essnmx/.github/workflows/nightly_at_release.yml +++ b/packages/essnmx/.github/workflows/nightly_at_release.yml @@ -38,3 +38,4 @@ jobs: python-version: ${{ matrix.python.version }} tox-env: ${{ matrix.python.tox-env }} checkout_ref: ${{ needs.setup.outputs.release_tag }} + secrets: inherit diff --git a/packages/essnmx/.github/workflows/test.yml b/packages/essnmx/.github/workflows/test.yml index 5f56a069..373ea4a1 100644 --- a/packages/essnmx/.github/workflows/test.yml +++ b/packages/essnmx/.github/workflows/test.yml @@ -43,6 +43,9 @@ on: jobs: test: runs-on: ${{ inputs.os-variant }} + env: + ESS_PROTECTED_FILESTORE_USERNAME: ${{ secrets.ESS_PROTECTED_FILESTORE_USERNAME }} + ESS_PROTECTED_FILESTORE_PASSWORD: ${{ secrets.ESS_PROTECTED_FILESTORE_PASSWORD }} steps: - uses: actions/checkout@v4 diff --git a/packages/essnmx/.github/workflows/unpinned.yml b/packages/essnmx/.github/workflows/unpinned.yml index 46a84c1c..3f49f722 100644 --- a/packages/essnmx/.github/workflows/unpinned.yml +++ b/packages/essnmx/.github/workflows/unpinned.yml @@ -38,3 +38,4 @@ jobs: python-version: ${{ matrix.python.version }} tox-env: ${{ matrix.python.tox-env }} checkout_ref: ${{ needs.setup.outputs.release_tag }} + secrets: inherit diff --git a/packages/essnmx/.gitignore b/packages/essnmx/.gitignore index 0f0541bc..46c5ae85 100644 --- a/packages/essnmx/.gitignore +++ b/packages/essnmx/.gitignore @@ -20,6 +20,7 @@ __pycache__/ .pytest_cache .mypy_cache docs/generated/ +.ruff_cache # Editor settings .idea/ diff --git a/packages/essnmx/requirements/make_base.py b/packages/essnmx/requirements/make_base.py index 68a17e84..493ede16 100644 --- a/packages/essnmx/requirements/make_base.py +++ b/packages/essnmx/requirements/make_base.py @@ -55,7 +55,8 @@ def as_nightly(repo: str) -> str: version = f"cp{sys.version_info.major}{sys.version_info.minor}" base = "https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly" suffix = "manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - return "-".join([base, version, version, suffix]) + prefix = "scipp @ " + return prefix + "-".join([base, version, version, suffix]) return f"{repo} @ git+https://github.com/{org}/{repo}@main" From 7829f10ca1940a3a644e9c255e7e2c44c11369f5 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 23 Jul 2024 13:37:01 +0200 Subject: [PATCH 216/403] ci: update deps --- packages/essnmx/requirements/base.txt | 6 +++--- packages/essnmx/requirements/basetest.txt | 4 ++-- packages/essnmx/requirements/ci.txt | 2 +- packages/essnmx/requirements/dev.txt | 10 +++++----- packages/essnmx/requirements/docs.txt | 18 +++++++++--------- packages/essnmx/requirements/mypy.txt | 2 +- packages/essnmx/requirements/nightly.in | 2 +- packages/essnmx/requirements/nightly.txt | 8 ++++---- packages/essnmx/requirements/static.txt | 2 +- 9 files changed, 27 insertions(+), 27 deletions(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 0aa5ccda..0fce71fa 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -19,11 +19,11 @@ cyclebane==24.6.0 # via sciline cycler==0.12.1 # via matplotlib -dask==2024.6.2 +dask==2024.7.1 # via -r base.in defusedxml==0.7.1 # via -r base.in -fonttools==4.53.0 +fonttools==4.53.1 # via matplotlib fsspec==2024.6.1 # via dask @@ -45,7 +45,7 @@ matplotlib==3.9.1 # via plopp networkx==3.3 # via cyclebane -numpy==2.0.0 +numpy==2.0.1 # via # contourpy # h5py diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt index 4d95d3c9..01f19d18 100644 --- a/packages/essnmx/requirements/basetest.txt +++ b/packages/essnmx/requirements/basetest.txt @@ -5,7 +5,7 @@ # # pip-compile-multi # -exceptiongroup==1.2.1 +exceptiongroup==1.2.2 # via pytest iniconfig==2.0.0 # via pytest @@ -13,7 +13,7 @@ packaging==24.1 # via pytest pluggy==1.5.0 # via pytest -pytest==8.2.2 +pytest==8.3.1 # via -r basetest.in tomli==2.0.1 # via pytest diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index 4d81cbd2..4bad3aea 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -5,7 +5,7 @@ # # pip-compile-multi # -cachetools==5.3.3 +cachetools==5.4.0 # via tox certifi==2024.7.4 # via requests diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index f0fd0a82..f0b20ea1 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -28,7 +28,7 @@ async-lru==2.0.4 # via jupyterlab cffi==1.16.0 # via argon2-cffi-bindings -copier==9.3.0 +copier==9.3.1 # via -r dev.in dunamai==1.21.2 # via copier @@ -50,7 +50,7 @@ json5==0.9.25 # via jupyterlab-server jsonpointer==3.0.0 # via jsonschema -jsonschema[format-nongpl]==4.22.0 +jsonschema[format-nongpl]==4.23.0 # via # jupyter-events # jupyterlab-server @@ -59,7 +59,7 @@ jupyter-events==0.10.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab -jupyter-server==2.14.1 +jupyter-server==2.14.2 # via # jupyter-lsp # jupyterlab @@ -67,9 +67,9 @@ jupyter-server==2.14.1 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.2.3 +jupyterlab==4.2.4 # via -r dev.in -jupyterlab-server==2.27.2 +jupyterlab-server==2.27.3 # via jupyterlab notebook-shim==0.2.4 # via jupyterlab diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 52b1ff3e..8de80308 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -40,7 +40,7 @@ docutils==0.21.2 # nbsphinx # pydata-sphinx-theme # sphinx -exceptiongroup==1.2.1 +exceptiongroup==1.2.2 # via ipython executing==2.0.1 # via stack-data @@ -69,7 +69,7 @@ jinja2==3.1.4 # nbconvert # nbsphinx # sphinx -jsonschema==4.22.0 +jsonschema==4.23.0 # via nbformat jsonschema-specifications==2023.12.1 # via jsonschema @@ -135,7 +135,7 @@ psutil==6.0.0 # via ipykernel ptyprocess==0.7.0 # via pexpect -pure-eval==0.2.2 +pure-eval==0.2.3 # via stack-data pydata-sphinx-theme==0.15.4 # via -r docs.in @@ -156,17 +156,17 @@ referencing==0.35.1 # via # jsonschema # jsonschema-specifications -rpds-py==0.18.1 +rpds-py==0.19.0 # via # jsonschema # referencing -scippneutron==24.6.0 +scippneutron==24.7.0 # via -r docs.in snowballstemmer==2.2.0 # via sphinx soupsieve==2.5 # via beautifulsoup4 -sphinx==7.3.7 +sphinx==7.4.7 # via # -r docs.in # myst-parser @@ -175,7 +175,7 @@ sphinx==7.3.7 # sphinx-autodoc-typehints # sphinx-copybutton # sphinx-design -sphinx-autodoc-typehints==2.2.2 +sphinx-autodoc-typehints==2.2.3 # via -r docs.in sphinx-copybutton==0.5.2 # via -r docs.in @@ -185,11 +185,11 @@ sphinxcontrib-applehelp==1.0.8 # via sphinx sphinxcontrib-devhelp==1.0.6 # via sphinx -sphinxcontrib-htmlhelp==2.0.5 +sphinxcontrib-htmlhelp==2.0.6 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.7 +sphinxcontrib-qthelp==1.0.8 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index 69cdcbc0..b9863cd2 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -6,7 +6,7 @@ # pip-compile-multi # -r test.txt -mypy==1.10.1 +mypy==1.11.0 # via -r mypy.in mypy-extensions==1.0.0 # via mypy diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index 245b5831..6f38c4be 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -7,7 +7,7 @@ pooch pandas gemmi defusedxml -https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl sciline @ git+https://github.com/scipp/sciline@main scippnexus @ git+https://github.com/scipp/scippnexus@main plopp @ git+https://github.com/scipp/plopp@main diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 9acbc135..ad24d9a9 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:38452d7c1b37c7bcf4e0819d4f3cfcbe168c62ad +# SHA1:ddc214997f19196de8428e2c570655767569ae87 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -20,11 +20,11 @@ cyclebane==24.6.0 # via sciline cycler==0.12.1 # via matplotlib -dask==2024.6.2 +dask==2024.7.1 # via -r nightly.in defusedxml==0.7.1 # via -r nightly.in -fonttools==4.53.0 +fonttools==4.53.1 # via matplotlib fsspec==2024.6.1 # via dask @@ -46,7 +46,7 @@ matplotlib==3.9.1 # via plopp networkx==3.3 # via cyclebane -numpy==2.0.0 +numpy==2.0.1 # via # contourpy # h5py diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index bdf1ccc0..e106d602 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -11,7 +11,7 @@ distlib==0.3.8 # via virtualenv filelock==3.15.4 # via virtualenv -identify==2.5.36 +identify==2.6.0 # via pre-commit nodeenv==1.9.1 # via pre-commit From 357812dd683f5fe853c2a2d01982f4fc487b7939 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Fri, 30 Aug 2024 09:18:25 +0200 Subject: [PATCH 217/403] Cast number to float before making them to integer. --- packages/essnmx/src/ess/nmx/mcstas/xml.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas/xml.py b/packages/essnmx/src/ess/nmx/mcstas/xml.py index f21188a7..0294e339 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas/xml.py @@ -175,14 +175,16 @@ def from_xml( length_unit = simulation_settings.length_unit + # Type casting from str to float and then int to allow *e* notation + # For example, '1e4' -> 10000.0 -> 10_000 return cls( component_type=type_desc.attrib['name'], name=component.attrib['name'], - id_start=int(component.attrib['idstart']), + id_start=int(float(component.attrib['idstart'])), fast_axis_name=fast_axis_name, slow_axis_name=slow_axis_name, - num_x=int(type_desc.attrib['xpixels']), - num_y=int(type_desc.attrib['ypixels']), + num_x=int(float(type_desc.attrib['xpixels'])), + num_y=int(float(type_desc.attrib['ypixels'])), step_x=sc.scalar(float(type_desc.attrib['xstep']), unit=length_unit), step_y=sc.scalar(float(type_desc.attrib['ystep']), unit=length_unit), start_x=float(type_desc.attrib['xstart']), From 44bc2f37592f618330288529a4125c3cd1382ca4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 17:14:39 +0000 Subject: [PATCH 218/403] Bump scipp from 24.6.0 to 24.9.1 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 24.6.0 to 24.9.1. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/24.06.0...24.09.1) --- updated-dependencies: - dependency-name: scipp dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 0fce71fa..c139962e 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -35,8 +35,6 @@ h5py==3.11.0 # via scippnexus idna==3.7 # via requests -importlib-metadata==8.0.0 - # via dask kiwisolver==1.4.5 # via matplotlib locket==1.0.0 @@ -85,7 +83,7 @@ requests==2.32.3 # via pooch sciline==24.6.2 # via -r base.in -scipp==24.6.0 +scipp==24.9.1 # via # -r base.in # scippnexus @@ -103,5 +101,3 @@ tzdata==2024.1 # via pandas urllib3==2.2.2 # via requests -zipp==3.19.2 - # via importlib-metadata From 9aa168f7e8eec873159dbfa387890f83fa56c291 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Fri, 30 Aug 2024 14:13:50 +0200 Subject: [PATCH 219/403] Better error message for missing parameter in the file. --- packages/essnmx/src/ess/nmx/mcstas/load.py | 21 ++++++++++++++++++--- packages/essnmx/tests/loader_test.py | 22 +++++++++++++++++++++- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas/load.py b/packages/essnmx/src/ess/nmx/mcstas/load.py index 53d6c184..039dc520 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/load.py +++ b/packages/essnmx/src/ess/nmx/mcstas/load.py @@ -67,12 +67,27 @@ def load_raw_event_data( def load_crystal_rotation( file_path: FilePath, instrument: McStasInstrument ) -> CrystalRotation: - """Retrieve crystal rotation from the file.""" + """Retrieve crystal rotation from the file. + + Raises + ------ + KeyError + If the crystal rotation is not found in the file. + + """ with snx.File(file_path, 'r') as file: - return sc.vector( - value=[file[f"entry1/simulation/Param/XtalPhi{key}"][...] for key in "XYZ"], + param_keys = tuple(f"entry1/simulation/Param/XtalPhi{key}" for key in "XYZ") + if not all(key in file for key in param_keys): + raise KeyError( + f"Crystal rotations [{', '.join(param_keys)}] not found in file." + ) + + return CrystalRotation( + sc.vector( + value=[file[param_key][...] for param_key in param_keys], unit=instrument.simulation_settings.angle_unit, ) + ) def event_weights_from_probability( diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index cd758f66..1e8cca9c 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -10,7 +10,7 @@ import scippnexus as snx from ess.nmx import default_parameters from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample -from ess.nmx.mcstas.load import bank_names_to_detector_names +from ess.nmx.mcstas.load import bank_names_to_detector_names, load_crystal_rotation from ess.nmx.mcstas.load import providers as loader_providers from ess.nmx.reduction import NMXData from ess.nmx.types import ( @@ -192,6 +192,26 @@ def test_file_reader_mcstas_additional_fields(tmp_mcstas_file: pathlib.Path) -> assert isinstance(dg, sc.DataGroup) +@pytest.fixture() +def rotation_mission_tmp_file(tmp_mcstas_file: pathlib.Path) -> pathlib.Path: + import h5py + + param_keys = tuple(f"entry1/simulation/Param/XtalPhi{key}" for key in "XYZ") + + # Remove the rotation parameters from the file. + with h5py.File(tmp_mcstas_file, 'a') as file: + for key in param_keys: + del file[key] + + return tmp_mcstas_file + + +def test_missing_rotation(rotation_mission_tmp_file: FilePath) -> None: + with pytest.raises(KeyError, match="XtalPhiX"): + load_crystal_rotation(rotation_mission_tmp_file, None) + # McStasInstrument is not used due to error in the file. + + def test_bank_names_to_detector_names_two_detectors(): res = bank_names_to_detector_names(two_detectors_two_filenames) assert len(res) == 2 From d8cb24aa2959d6ba589e14fcb8f67f744e0d0ba9 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Fri, 30 Aug 2024 14:29:12 +0200 Subject: [PATCH 220/403] Fix wrong indentation. --- packages/essnmx/src/ess/nmx/mcstas/load.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas/load.py b/packages/essnmx/src/ess/nmx/mcstas/load.py index 039dc520..02ed1799 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/load.py +++ b/packages/essnmx/src/ess/nmx/mcstas/load.py @@ -81,13 +81,12 @@ def load_crystal_rotation( raise KeyError( f"Crystal rotations [{', '.join(param_keys)}] not found in file." ) - - return CrystalRotation( - sc.vector( - value=[file[param_key][...] for param_key in param_keys], - unit=instrument.simulation_settings.angle_unit, + return CrystalRotation( + sc.vector( + value=[file[param_key][...] for param_key in param_keys], + unit=instrument.simulation_settings.angle_unit, + ) ) - ) def event_weights_from_probability( From c8361b5ee59e700712c08e86512d18bf7c2fcb46 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Fri, 27 Sep 2024 16:23:17 +0200 Subject: [PATCH 221/403] Add rotated detector pixel offset based on the starting position. --- packages/essnmx/src/ess/nmx/mcstas/xml.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas/xml.py b/packages/essnmx/src/ess/nmx/mcstas/xml.py index 0294e339..c872e681 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas/xml.py @@ -338,8 +338,11 @@ def _pixel_positions( return ( (pixel_n_slow * slow_axis_steps) + (pixel_n_fast * fast_axis_steps) - + position_offset - ) + + detector.rotation_matrix + * sc.vector( + [detector.start_x, detector.start_y, 0.0], unit=position_offset.unit + ) # Detector pixel offset should also be rotated first. + ) + position_offset def _detector_pixel_positions( From 7e0917cedd9e3bde5c0fde6bf8e64ff01a30887d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 17:58:09 +0000 Subject: [PATCH 222/403] Bump scipp from 24.9.1 to 24.11.0 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 24.9.1 to 24.11.0. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/24.09.1...24.11.0) --- updated-dependencies: - dependency-name: scipp dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index c139962e..9f8a2654 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -83,7 +83,7 @@ requests==2.32.3 # via pooch sciline==24.6.2 # via -r base.in -scipp==24.9.1 +scipp==24.11.0 # via # -r base.in # scippnexus From 9bdbd14e5786ed905bd223e659d2fcf7cac11e12 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Nov 2024 17:53:53 +0000 Subject: [PATCH 223/403] Bump scipp from 24.11.0 to 24.11.1 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 24.11.0 to 24.11.1. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/24.11.0...24.11.1) --- updated-dependencies: - dependency-name: scipp dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 9f8a2654..08691c42 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -83,7 +83,7 @@ requests==2.32.3 # via pooch sciline==24.6.2 # via -r base.in -scipp==24.11.0 +scipp==24.11.1 # via # -r base.in # scippnexus From 7d8145a6680226039b5035f2f73235de850e18d0 Mon Sep 17 00:00:00 2001 From: Mridul Seth Date: Fri, 22 Nov 2024 13:19:17 +0100 Subject: [PATCH 224/403] copier update, py313, tox deps --- packages/essnmx/.copier-answers.yml | 4 +- packages/essnmx/.github/workflows/ci.yml | 12 +-- packages/essnmx/.github/workflows/docs.yml | 6 +- .../.github/workflows/nightly_at_main.yml | 6 +- .../.github/workflows/nightly_at_release.yml | 8 +- packages/essnmx/.github/workflows/release.yml | 10 +-- packages/essnmx/.github/workflows/test.yml | 19 ++++- .../essnmx/.github/workflows/unpinned.yml | 8 +- packages/essnmx/.gitignore | 5 ++ packages/essnmx/.pre-commit-config.yaml | 16 +++- packages/essnmx/.python-version | 1 + packages/essnmx/conda/meta.yaml | 9 ++- packages/essnmx/docs/conf.py | 3 + packages/essnmx/pyproject.toml | 18 ++++- packages/essnmx/requirements/base.txt | 62 ++++++++-------- packages/essnmx/requirements/basetest.in | 6 ++ packages/essnmx/requirements/basetest.txt | 6 +- packages/essnmx/requirements/ci.txt | 28 +++---- packages/essnmx/requirements/dev.txt | 30 ++++---- packages/essnmx/requirements/docs.txt | 72 +++++++++--------- packages/essnmx/requirements/make_base.py | 8 +- packages/essnmx/requirements/mypy.txt | 2 +- packages/essnmx/requirements/nightly.in | 3 +- packages/essnmx/requirements/nightly.txt | 73 +++++++++++-------- packages/essnmx/requirements/static.txt | 14 ++-- packages/essnmx/requirements/wheels.txt | 8 +- packages/essnmx/src/ess/nmx/__init__.py | 2 +- packages/essnmx/tests/package_test.py | 13 ++++ 28 files changed, 277 insertions(+), 175 deletions(-) create mode 100644 packages/essnmx/.python-version diff --git a/packages/essnmx/.copier-answers.yml b/packages/essnmx/.copier-answers.yml index ef43bf02..f758cf37 100644 --- a/packages/essnmx/.copier-answers.yml +++ b/packages/essnmx/.copier-answers.yml @@ -1,8 +1,8 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY -_commit: 101e594 +_commit: aa5dc5e _src_path: gh:scipp/copier_template description: Data reduction for NMX at the European Spallation Source. -max_python: '3.12' +max_python: '3.13' min_python: '3.10' namespace_package: ess nightly_deps: scipp,sciline,scippnexus,plopp diff --git a/packages/essnmx/.github/workflows/ci.yml b/packages/essnmx/.github/workflows/ci.yml index 8234bc9f..b273e684 100644 --- a/packages/essnmx/.github/workflows/ci.yml +++ b/packages/essnmx/.github/workflows/ci.yml @@ -10,7 +10,7 @@ on: jobs: formatting: name: Formatting and static analysis - runs-on: 'ubuntu-22.04' + runs-on: 'ubuntu-24.04' outputs: min_python: ${{ steps.vars.outputs.min_python }} min_tox_env: ${{ steps.vars.outputs.min_tox_env }} @@ -19,15 +19,15 @@ jobs: - name: Get Python version for other CI jobs id: vars run: | - echo "min_python=$(cat .github/workflows/python-version-ci)" >> $GITHUB_OUTPUT - echo "min_tox_env=py$(cat .github/workflows/python-version-ci | sed 's/\.//g')" >> $GITHUB_OUTPUT + echo "min_python=$(< .github/workflows/python-version-ci)" >> "$GITHUB_OUTPUT" + echo "min_tox_env=py$(sed 's/\.//g' < .github/workflows/python-version-ci)" >> "$GITHUB_OUTPUT" - uses: actions/setup-python@v5 with: python-version-file: '.github/workflows/python-version-ci' - uses: pre-commit/action@v3.0.1 with: extra_args: --all-files - - uses: pre-commit-ci/lite-action@v1.0.2 + - uses: pre-commit-ci/lite-action@v1.1.0 if: always() with: msg: Apply automatic formatting @@ -37,7 +37,7 @@ jobs: needs: formatting strategy: matrix: - os: ['ubuntu-22.04'] + os: ['ubuntu-24.04'] python: - version: '${{needs.formatting.outputs.min_python}}' tox-env: '${{needs.formatting.outputs.min_tox_env}}' @@ -53,6 +53,6 @@ jobs: uses: ./.github/workflows/docs.yml with: publish: false - linkcheck: ${{ contains(matrix.variant.os, 'ubuntu') && github.ref == 'refs/heads/main' }} + linkcheck: ${{ github.ref == 'refs/heads/main' }} branch: ${{ github.head_ref == '' && github.ref_name || github.head_ref }} secrets: inherit diff --git a/packages/essnmx/.github/workflows/docs.yml b/packages/essnmx/.github/workflows/docs.yml index a5ea2b05..72659cbd 100644 --- a/packages/essnmx/.github/workflows/docs.yml +++ b/packages/essnmx/.github/workflows/docs.yml @@ -41,7 +41,7 @@ env: jobs: docs: name: Build documentation - runs-on: 'ubuntu-22.04' + runs-on: 'ubuntu-24.04' env: ESS_PROTECTED_FILESTORE_USERNAME: ${{ secrets.ESS_PROTECTED_FILESTORE_USERNAME }} ESS_PROTECTED_FILESTORE_PASSWORD: ${{ secrets.ESS_PROTECTED_FILESTORE_PASSWORD }} @@ -58,7 +58,7 @@ jobs: python-version-file: '.github/workflows/python-version-ci' - run: python -m pip install --upgrade pip - run: python -m pip install -r requirements/ci.txt - - run: tox -e releasedocs -- ${VERSION} + - run: tox -e releasedocs -- "${VERSION}" if: ${{ inputs.version != '' }} - run: tox -e docs if: ${{ inputs.version == '' }} @@ -69,7 +69,7 @@ jobs: name: docs_html path: html/ - - uses: JamesIves/github-pages-deploy-action@v4.6.3 + - uses: JamesIves/github-pages-deploy-action@v4.6.9 if: ${{ inputs.publish }} with: branch: gh-pages diff --git a/packages/essnmx/.github/workflows/nightly_at_main.yml b/packages/essnmx/.github/workflows/nightly_at_main.yml index c2b9d33a..20e9ee4a 100644 --- a/packages/essnmx/.github/workflows/nightly_at_main.yml +++ b/packages/essnmx/.github/workflows/nightly_at_main.yml @@ -8,21 +8,21 @@ on: jobs: setup: name: Setup variables - runs-on: 'ubuntu-22.04' + runs-on: 'ubuntu-24.04' outputs: min_python: ${{ steps.vars.outputs.min_python }} steps: - uses: actions/checkout@v4 - name: Get Python version for other CI jobs id: vars - run: echo "min_python=$(cat .github/workflows/python-version-ci)" >> $GITHUB_OUTPUT + run: echo "min_python=$(< .github/workflows/python-version-ci)" >> "$GITHUB_OUTPUT" tests: name: Tests needs: setup strategy: matrix: - os: ['ubuntu-22.04'] + os: ['ubuntu-24.04'] python: - version: '${{needs.setup.outputs.min_python}}' tox-env: 'nightly' diff --git a/packages/essnmx/.github/workflows/nightly_at_release.yml b/packages/essnmx/.github/workflows/nightly_at_release.yml index 3faa1c23..14b75213 100644 --- a/packages/essnmx/.github/workflows/nightly_at_release.yml +++ b/packages/essnmx/.github/workflows/nightly_at_release.yml @@ -8,7 +8,7 @@ on: jobs: setup: name: Setup variables - runs-on: 'ubuntu-22.04' + runs-on: 'ubuntu-24.04' outputs: min_python: ${{ steps.vars.outputs.min_python }} release_tag: ${{ steps.release.outputs.release_tag }} @@ -18,17 +18,17 @@ jobs: fetch-depth: 0 # history required so we can determine latest release tag - name: Get last release tag from git id: release - run: echo "release_tag=$(git describe --tags --abbrev=0 --match '[0-9]*.[0-9]*.[0-9]*')" >> $GITHUB_OUTPUT + run: echo "release_tag=$(git describe --tags --abbrev=0 --match '[0-9]*.[0-9]*.[0-9]*')" >> "$GITHUB_OUTPUT" - name: Get Python version for other CI jobs id: vars - run: echo "min_python=$(cat .github/workflows/python-version-ci)" >> $GITHUB_OUTPUT + run: echo "min_python=$(cat .github/workflows/python-version-ci)" >> "$GITHUB_OUTPUT" tests: name: Tests needs: setup strategy: matrix: - os: ['ubuntu-22.04'] + os: ['ubuntu-24.04'] python: - version: '${{needs.setup.outputs.min_python}}' tox-env: 'nightly' diff --git a/packages/essnmx/.github/workflows/release.yml b/packages/essnmx/.github/workflows/release.yml index d1317a76..fe8ef23d 100644 --- a/packages/essnmx/.github/workflows/release.yml +++ b/packages/essnmx/.github/workflows/release.yml @@ -12,7 +12,7 @@ defaults: jobs: build_conda: name: Conda build - runs-on: 'ubuntu-22.04' + runs-on: 'ubuntu-24.04' steps: - uses: actions/checkout@v4 @@ -35,7 +35,7 @@ jobs: build_wheels: name: Wheels - runs-on: 'ubuntu-22.04' + runs-on: 'ubuntu-24.04' steps: - uses: actions/checkout@v4 @@ -61,7 +61,7 @@ jobs: upload_pypi: name: Deploy PyPI needs: [build_wheels, build_conda] - runs-on: 'ubuntu-22.04' + runs-on: 'ubuntu-24.04' environment: release permissions: id-token: write @@ -73,7 +73,7 @@ jobs: upload_conda: name: Deploy Conda needs: [build_wheels, build_conda] - runs-on: 'ubuntu-22.04' + runs-on: 'ubuntu-24.04' if: github.event_name == 'release' && github.event.action == 'published' steps: @@ -97,7 +97,7 @@ jobs: assets: name: Upload docs needs: docs - runs-on: 'ubuntu-22.04' + runs-on: 'ubuntu-24.04' permissions: contents: write # This is needed so that the action can upload the asset steps: diff --git a/packages/essnmx/.github/workflows/test.yml b/packages/essnmx/.github/workflows/test.yml index 373ea4a1..ca40b253 100644 --- a/packages/essnmx/.github/workflows/test.yml +++ b/packages/essnmx/.github/workflows/test.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: os-variant: - default: 'ubuntu-22.04' + default: 'ubuntu-24.04' type: string python-version: type: string @@ -23,7 +23,7 @@ on: workflow_call: inputs: os-variant: - default: 'ubuntu-22.04' + default: 'ubuntu-24.04' type: string python-version: type: string @@ -41,6 +41,21 @@ on: type: string jobs: + package-test: + runs-on: ${{ inputs.os-variant }} + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.checkout_ref }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - run: python -m pip install --upgrade pip + - run: python -m pip install . + - run: python tests/package_test.py + name: Run package tests + test: runs-on: ${{ inputs.os-variant }} env: diff --git a/packages/essnmx/.github/workflows/unpinned.yml b/packages/essnmx/.github/workflows/unpinned.yml index 3f49f722..ff03faa1 100644 --- a/packages/essnmx/.github/workflows/unpinned.yml +++ b/packages/essnmx/.github/workflows/unpinned.yml @@ -8,7 +8,7 @@ on: jobs: setup: name: Setup variables - runs-on: 'ubuntu-22.04' + runs-on: 'ubuntu-24.04' outputs: min_python: ${{ steps.vars.outputs.min_python }} release_tag: ${{ steps.release.outputs.release_tag }} @@ -18,17 +18,17 @@ jobs: fetch-depth: 0 # history required so we can determine latest release tag - name: Get last release tag from git id: release - run: echo "release_tag=$(git describe --tags --abbrev=0 --match '[0-9]*.[0-9]*.[0-9]*')" >> $GITHUB_OUTPUT + run: echo "release_tag=$(git describe --tags --abbrev=0 --match '[0-9]*.[0-9]*.[0-9]*')" >> "$GITHUB_OUTPUT" - name: Get Python version for other CI jobs id: vars - run: echo "min_python=$(cat .github/workflows/python-version-ci)" >> $GITHUB_OUTPUT + run: echo "min_python=$(cat .github/workflows/python-version-ci)" >> "$GITHUB_OUTPUT" tests: name: Tests needs: setup strategy: matrix: - os: ['ubuntu-22.04'] + os: ['ubuntu-24.04'] python: - version: '${{needs.setup.outputs.min_python}}' tox-env: 'unpinned' diff --git a/packages/essnmx/.gitignore b/packages/essnmx/.gitignore index 46c5ae85..3e02ecce 100644 --- a/packages/essnmx/.gitignore +++ b/packages/essnmx/.gitignore @@ -4,11 +4,13 @@ dist html .tox *.egg-info +uv.lock # we lock dependencies with pip-compile, not uv *.sw? # Environments venv +.venv # Caches .clangd/ @@ -39,3 +41,6 @@ docs/generated/ *.cif *.rcif *.ort +*.zip +*.sqw +*.nxspe diff --git a/packages/essnmx/.pre-commit-config.yaml b/packages/essnmx/.pre-commit-config.yaml index 4442b1b9..f83362c6 100644 --- a/packages/essnmx/.pre-commit-config.yaml +++ b/packages/essnmx/.pre-commit-config.yaml @@ -1,8 +1,10 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 hooks: - id: check-added-large-files + - id: check-case-conflict + - id: check-illegal-windows-names - id: check-json exclude: asv.conf.json - id: check-merge-conflict @@ -14,14 +16,14 @@ repos: args: [ --markdown-linebreak-ext=md ] exclude: '\.svg' - repo: https://github.com/kynan/nbstripout - rev: 0.6.0 + rev: 0.7.1 hooks: - id: nbstripout types: [ "jupyter" ] args: [ "--drop-empty-cells", "--extra-keys 'metadata.language_info.version cell.metadata.jp-MarkdownHeadingCollapsed cell.metadata.pycharm'" ] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.3 + rev: v0.6.9 hooks: - id: ruff args: [ --fix ] @@ -29,7 +31,7 @@ repos: - id: ruff-format types_or: [ python, pyi ] - repo: https://github.com/codespell-project/codespell - rev: v2.2.6 + rev: v2.3.0 hooks: - id: codespell additional_dependencies: @@ -44,3 +46,9 @@ repos: - id: rst-directive-colons - id: rst-inline-touching-normal - id: text-unicode-replacement-char + - repo: https://github.com/rhysd/actionlint + rev: v1.7.3 + hooks: + - id: actionlint + # Disable because of false-positive SC2046 + args: ["-shellcheck="] diff --git a/packages/essnmx/.python-version b/packages/essnmx/.python-version new file mode 100644 index 00000000..c8cfe395 --- /dev/null +++ b/packages/essnmx/.python-version @@ -0,0 +1 @@ +3.10 diff --git a/packages/essnmx/conda/meta.yaml b/packages/essnmx/conda/meta.yaml index 1043461f..08c4160d 100644 --- a/packages/essnmx/conda/meta.yaml +++ b/packages/essnmx/conda/meta.yaml @@ -9,6 +9,7 @@ source: {% set pyproject = load_file_data('pyproject.toml') %} {% set dependencies = pyproject.get('project', {}).get('dependencies', {}) %} +{% set test_dependencies = pyproject.get('project', {}).get('optional-dependencies', {}).get('test', {}) %} requirements: @@ -38,7 +39,13 @@ test: imports: - ess.nmx requires: - - pytest + + {# Conda does not allow spaces between package name and version, so remove them #} + {% for package in test_dependencies %} + - {% if package == "graphviz" %}python-graphviz{% else %}{{ package|replace(" ", "") }}{% endif %} + {% endfor %} + + source_files: - pyproject.toml - tests/ diff --git a/packages/essnmx/docs/conf.py b/packages/essnmx/docs/conf.py index adc0f235..5dd75d9d 100644 --- a/packages/essnmx/docs/conf.py +++ b/packages/essnmx/docs/conf.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) + import doctest import os import sys diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index d5384048..ef143600 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -21,6 +21,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Scientific/Engineering", "Typing :: Typed", ] @@ -44,6 +45,11 @@ dependencies = [ dynamic = ["version"] +[project.optional-dependencies] +test = [ + "pytest", +] + [project.urls] "Bug Tracker" = "https://github.com/scipp/essnmx/issues" "Documentation" = "https://scipp.github.io/essnmx" @@ -80,8 +86,8 @@ ignore = [ # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules "COM812", "COM819", "D206", "D300", "E111", "E114", "E117", "ISC001", "ISC002", "Q000", "Q001", "Q002", "Q003", "W191", ] -fixable = ["I001", "B010"] -isort.known-first-party = ["essnmx"] +fixable = ["B010", "I001", "PT001"] +isort.known-first-party = ["ess.nmx"] pydocstyle.convention = "numpy" [tool.ruff.lint.per-file-ignores] @@ -112,3 +118,11 @@ enable_error_code = [ "truthy-bool", ] warn_unreachable = true + +[tool.codespell] +ignore-words-list = [ + # Codespell wants "socioeconomic" which seems to be the standard spelling. + # But we use the word in our code of conduct which is the contributor covenant. + # Let's not modify it if we don't have to. + "socio-economic", +] diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 08691c42..67d7d683 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -5,45 +5,47 @@ # # pip-compile-multi # -certifi==2024.7.4 +certifi==2024.8.30 # via requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via dask -cloudpickle==3.0.0 +cloudpickle==3.1.0 # via dask -contourpy==1.2.1 +contourpy==1.3.1 # via matplotlib -cyclebane==24.6.0 +cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -dask==2024.7.1 +dask==2024.11.2 # via -r base.in defusedxml==0.7.1 # via -r base.in -fonttools==4.53.1 +fonttools==4.55.0 # via matplotlib -fsspec==2024.6.1 +fsspec==2024.10.0 # via dask -gemmi==0.6.6 +gemmi==0.6.7 # via -r base.in graphviz==0.20.3 # via -r base.in -h5py==3.11.0 +h5py==3.12.1 # via scippnexus -idna==3.7 +idna==3.10 # via requests -kiwisolver==1.4.5 +importlib-metadata==8.5.0 + # via dask +kiwisolver==1.4.7 # via matplotlib locket==1.0.0 # via partd -matplotlib==3.9.1 +matplotlib==3.9.2 # via plopp -networkx==3.3 +networkx==3.4.2 # via cyclebane -numpy==2.0.1 +numpy==2.1.3 # via # contourpy # h5py @@ -51,53 +53,55 @@ numpy==2.0.1 # pandas # scipp # scipy -packaging==24.1 +packaging==24.2 # via # dask # matplotlib # pooch -pandas==2.2.2 +pandas==2.2.3 # via -r base.in partd==1.4.2 # via dask -pillow==10.4.0 +pillow==11.0.0 # via matplotlib -platformdirs==4.2.2 +platformdirs==4.3.6 # via pooch -plopp==24.6.0 +plopp==24.10.0 # via -r base.in pooch==1.8.2 # via -r base.in -pyparsing==3.1.2 +pyparsing==3.2.0 # via matplotlib python-dateutil==2.9.0.post0 # via # matplotlib # pandas # scippnexus -pytz==2024.1 +pytz==2024.2 # via pandas -pyyaml==6.0.1 +pyyaml==6.0.2 # via dask requests==2.32.3 # via pooch -sciline==24.6.2 +sciline==24.10.0 # via -r base.in scipp==24.11.1 # via # -r base.in # scippnexus -scippnexus==24.6.0 +scippnexus==24.11.0 # via -r base.in -scipy==1.14.0 +scipy==1.14.1 # via scippnexus six==1.16.0 # via python-dateutil -toolz==0.12.1 +toolz==1.0.0 # via # dask # partd -tzdata==2024.1 +tzdata==2024.2 # via pandas -urllib3==2.2.2 +urllib3==2.2.3 # via requests +zipp==3.21.0 + # via importlib-metadata diff --git a/packages/essnmx/requirements/basetest.in b/packages/essnmx/requirements/basetest.in index e4a48b29..5b3942ea 100644 --- a/packages/essnmx/requirements/basetest.in +++ b/packages/essnmx/requirements/basetest.in @@ -1,4 +1,10 @@ # Dependencies that are only used by tests. # Do not make an environment from this file, use test.txt instead! +# Add more dependencies in the ``test`` list +# under ``[project.optional-dependencies]`` section, in ``pyproject.toml`` +# Anything above "--- END OF CUSTOM SECTION ---" +# will not be touched by ``make_base.py`` +# --- END OF CUSTOM SECTION --- +# The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! pytest diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt index 01f19d18..66970e04 100644 --- a/packages/essnmx/requirements/basetest.txt +++ b/packages/essnmx/requirements/basetest.txt @@ -9,11 +9,11 @@ exceptiongroup==1.2.2 # via pytest iniconfig==2.0.0 # via pytest -packaging==24.1 +packaging==24.2 # via pytest pluggy==1.5.0 # via pytest -pytest==8.3.1 +pytest==8.3.3 # via -r basetest.in -tomli==2.0.1 +tomli==2.1.0 # via pytest diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index 4bad3aea..14921a29 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -5,19 +5,19 @@ # # pip-compile-multi # -cachetools==5.4.0 +cachetools==5.5.0 # via tox -certifi==2024.7.4 +certifi==2024.8.30 # via requests chardet==5.2.0 # via tox -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests colorama==0.4.6 # via tox -distlib==0.3.8 +distlib==0.3.9 # via virtualenv -filelock==3.15.4 +filelock==3.16.1 # via # tox # virtualenv @@ -25,32 +25,34 @@ gitdb==4.0.11 # via gitpython gitpython==3.1.43 # via -r ci.in -idna==3.7 +idna==3.10 # via requests -packaging==24.1 +packaging==24.2 # via # -r ci.in # pyproject-api # tox -platformdirs==4.2.2 +platformdirs==4.3.6 # via # tox # virtualenv pluggy==1.5.0 # via tox -pyproject-api==1.7.1 +pyproject-api==1.8.0 # via tox requests==2.32.3 # via -r ci.in smmap==5.0.1 # via gitdb -tomli==2.0.1 +tomli==2.1.0 # via # pyproject-api # tox -tox==4.16.0 +tox==4.23.2 # via -r ci.in -urllib3==2.2.2 +typing-extensions==4.12.2 + # via tox +urllib3==2.2.3 # via requests -virtualenv==20.26.3 +virtualenv==20.27.1 # via tox diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index f0b20ea1..f9cab17c 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -14,7 +14,7 @@ -r wheels.txt annotated-types==0.7.0 # via pydantic -anyio==4.4.0 +anyio==4.6.2.post1 # via # httpx # jupyter-server @@ -26,11 +26,11 @@ arrow==1.3.0 # via isoduration async-lru==2.0.4 # via jupyterlab -cffi==1.16.0 +cffi==1.17.1 # via argon2-cffi-bindings -copier==9.3.1 +copier==9.4.1 # via -r dev.in -dunamai==1.21.2 +dunamai==1.23.0 # via copier fqdn==1.5.1 # via jsonschema @@ -38,15 +38,15 @@ funcy==2.0 # via copier h11==0.14.0 # via httpcore -httpcore==1.0.5 +httpcore==1.0.7 # via httpx -httpx==0.27.0 +httpx==0.27.2 # via jupyterlab isoduration==20.11.0 # via jsonschema jinja2-ansible-filters==1.3.2 # via copier -json5==0.9.25 +json5==0.9.28 # via jupyterlab-server jsonpointer==3.0.0 # via jsonschema @@ -67,7 +67,7 @@ jupyter-server==2.14.2 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.2.4 +jupyterlab==4.3.1 # via -r dev.in jupyterlab-server==2.27.3 # via jupyterlab @@ -81,15 +81,15 @@ pip-compile-multi==2.6.4 # via -r dev.in pip-tools==7.4.1 # via pip-compile-multi -plumbum==1.8.3 +plumbum==1.9.0 # via copier -prometheus-client==0.20.0 +prometheus-client==0.21.0 # via jupyter-server pycparser==2.22 # via cffi -pydantic==2.8.2 +pydantic==2.10.1 # via copier -pydantic-core==2.20.1 +pydantic-core==2.27.1 # via pydantic python-json-logger==2.0.7 # via jupyter-events @@ -115,15 +115,15 @@ terminado==0.18.1 # jupyter-server-terminals toposort==1.10 # via pip-compile-multi -types-python-dateutil==2.9.0.20240316 +types-python-dateutil==2.9.0.20241003 # via arrow uri-template==1.3.0 # via jsonschema -webcolors==24.6.0 +webcolors==24.11.1 # via jsonschema websocket-client==1.8.0 # via jupyter-server -wheel==0.43.0 +wheel==0.45.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 8de80308..5dd4a562 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -8,15 +8,17 @@ -r base.txt accessible-pygments==0.0.5 # via pydata-sphinx-theme -alabaster==0.7.16 +alabaster==1.0.0 # via sphinx +appnope==0.1.4 + # via ipykernel asttokens==2.4.1 # via stack-data -attrs==23.2.0 +attrs==24.2.0 # via # jsonschema # referencing -babel==2.15.0 +babel==2.16.0 # via # pydata-sphinx-theme # sphinx @@ -24,13 +26,13 @@ beautifulsoup4==4.12.3 # via # nbconvert # pydata-sphinx-theme -bleach==6.1.0 +bleach==6.2.0 # via nbconvert comm==0.2.2 # via # ipykernel # ipywidgets -debugpy==1.8.2 +debugpy==1.8.9 # via ipykernel decorator==5.1.1 # via ipython @@ -42,7 +44,7 @@ docutils==0.21.2 # sphinx exceptiongroup==1.2.2 # via ipython -executing==2.0.1 +executing==2.1.0 # via stack-data fastjsonschema==2.20.0 # via nbformat @@ -52,16 +54,16 @@ ipydatawidgets==4.3.5 # via pythreejs ipykernel==6.29.5 # via -r docs.in -ipython==8.26.0 +ipython==8.29.0 # via # -r docs.in # ipykernel # ipywidgets -ipywidgets==8.1.3 +ipywidgets==8.1.5 # via # ipydatawidgets # pythreejs -jedi==0.19.1 +jedi==0.19.2 # via ipython jinja2==3.1.4 # via @@ -71,9 +73,9 @@ jinja2==3.1.4 # sphinx jsonschema==4.23.0 # via nbformat -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 # via jsonschema -jupyter-client==8.6.2 +jupyter-client==8.6.3 # via # ipykernel # nbclient @@ -86,13 +88,13 @@ jupyter-core==5.7.2 # nbformat jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-widgets==3.0.11 +jupyterlab-widgets==3.0.13 # via ipywidgets markdown-it-py==3.0.0 # via # mdit-py-plugins # myst-parser -markupsafe==2.1.5 +markupsafe==3.0.2 # via # jinja2 # nbconvert @@ -100,7 +102,7 @@ matplotlib-inline==0.1.7 # via # ipykernel # ipython -mdit-py-plugins==0.4.1 +mdit-py-plugins==0.4.2 # via myst-parser mdurl==0.1.2 # via markdown-it-py @@ -108,7 +110,7 @@ mistune==3.0.2 # via nbconvert mpltoolbox==24.5.1 # via scippneutron -myst-parser==3.0.1 +myst-parser==4.0.0 # via -r docs.in nbclient==0.10.0 # via nbconvert @@ -119,7 +121,7 @@ nbformat==5.10.4 # nbclient # nbconvert # nbsphinx -nbsphinx==0.9.4 +nbsphinx==0.9.5 # via -r docs.in nest-asyncio==1.6.0 # via ipykernel @@ -129,15 +131,15 @@ parso==0.8.4 # via jedi pexpect==4.9.0 # via ipython -prompt-toolkit==3.0.47 +prompt-toolkit==3.0.48 # via ipython -psutil==6.0.0 +psutil==6.1.0 # via ipykernel ptyprocess==0.7.0 # via pexpect pure-eval==0.2.3 # via stack-data -pydata-sphinx-theme==0.15.4 +pydata-sphinx-theme==0.16.0 # via -r docs.in pygments==2.18.0 # via @@ -148,7 +150,7 @@ pygments==2.18.0 # sphinx pythreejs==2.4.2 # via -r docs.in -pyzmq==26.0.3 +pyzmq==26.2.0 # via # ipykernel # jupyter-client @@ -156,17 +158,17 @@ referencing==0.35.1 # via # jsonschema # jsonschema-specifications -rpds-py==0.19.0 +rpds-py==0.21.0 # via # jsonschema # referencing -scippneutron==24.7.0 +scippneutron==24.11.0 # via -r docs.in snowballstemmer==2.2.0 # via sphinx -soupsieve==2.5 +soupsieve==2.6 # via beautifulsoup4 -sphinx==7.4.7 +sphinx==8.1.3 # via # -r docs.in # myst-parser @@ -175,31 +177,31 @@ sphinx==7.4.7 # sphinx-autodoc-typehints # sphinx-copybutton # sphinx-design -sphinx-autodoc-typehints==2.2.3 +sphinx-autodoc-typehints==2.5.0 # via -r docs.in sphinx-copybutton==0.5.2 # via -r docs.in -sphinx-design==0.6.0 +sphinx-design==0.6.1 # via -r docs.in -sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-applehelp==2.0.0 # via sphinx -sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-devhelp==2.0.0 # via sphinx -sphinxcontrib-htmlhelp==2.0.6 +sphinxcontrib-htmlhelp==2.1.0 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.8 +sphinxcontrib-qthelp==2.0.0 # via sphinx -sphinxcontrib-serializinghtml==1.1.10 +sphinxcontrib-serializinghtml==2.0.0 # via sphinx stack-data==0.6.3 # via ipython -tinycss2==1.3.0 +tinycss2==1.4.0 # via nbconvert -tomli==2.0.1 +tomli==2.1.0 # via sphinx -tornado==6.4.1 +tornado==6.4.2 # via # ipykernel # jupyter-client @@ -230,5 +232,5 @@ webencodings==0.5.1 # via # bleach # tinycss2 -widgetsnbextension==4.0.11 +widgetsnbextension==4.0.13 # via ipywidgets diff --git a/packages/essnmx/requirements/make_base.py b/packages/essnmx/requirements/make_base.py index 493ede16..3c70a5cc 100644 --- a/packages/essnmx/requirements/make_base.py +++ b/packages/essnmx/requirements/make_base.py @@ -42,8 +42,14 @@ def write_dependencies(dependency_name: str, dependencies: list[str]) -> None: if dependencies is None: raise RuntimeError("No dependencies found in pyproject.toml") dependencies = [dep.strip().strip('"') for dep in dependencies] + test_dependencies = ( + pyproject["project"].get("optional-dependencies", {}).get("test", []) + ) + test_dependencies = [dep.strip().strip('"') for dep in test_dependencies] + write_dependencies("base", dependencies) +write_dependencies("basetest", test_dependencies) def as_nightly(repo: str) -> str: @@ -61,7 +67,7 @@ def as_nightly(repo: str) -> str: nightly = tuple(args.nightly.split(",") if args.nightly else []) -nightly_dependencies = [dep for dep in dependencies if not dep.startswith(nightly)] +nightly_dependencies = [dep for dep in dependencies + test_dependencies if not dep.startswith(nightly)] nightly_dependencies += [as_nightly(arg) for arg in nightly] write_dependencies("nightly", nightly_dependencies) diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index b9863cd2..6ffc9ba6 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -6,7 +6,7 @@ # pip-compile-multi # -r test.txt -mypy==1.11.0 +mypy==1.13.0 # via -r mypy.in mypy-extensions==1.0.0 # via mypy diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index 6f38c4be..bca11276 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -1,4 +1,4 @@ --r basetest.in + # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask @@ -7,6 +7,7 @@ pooch pandas gemmi defusedxml +pytest scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl sciline @ git+https://github.com/scipp/sciline@main scippnexus @ git+https://github.com/scipp/scippnexus@main diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index ad24d9a9..daf1fd6e 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,52 +1,55 @@ -# SHA1:ddc214997f19196de8428e2c570655767569ae87 +# SHA1:5567d188ad845d7f7faf225ea3ed89fbc989745b # # This file is autogenerated by pip-compile-multi # To update, run: # # pip-compile-multi # --r basetest.txt -certifi==2024.7.4 +certifi==2024.8.30 # via requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via dask -cloudpickle==3.0.0 +cloudpickle==3.1.0 # via dask -contourpy==1.2.1 +contourpy==1.3.1 # via matplotlib -cyclebane==24.6.0 +cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -dask==2024.7.1 +dask==2024.11.2 # via -r nightly.in defusedxml==0.7.1 # via -r nightly.in -fonttools==4.53.1 +exceptiongroup==1.2.2 + # via pytest +fonttools==4.55.0 # via matplotlib -fsspec==2024.6.1 +fsspec==2024.10.0 # via dask -gemmi==0.6.6 +gemmi==0.6.7 # via -r nightly.in graphviz==0.20.3 # via -r nightly.in -h5py==3.11.0 +h5py==3.12.1 # via scippnexus -idna==3.7 +idna==3.10 # via requests -importlib-metadata==8.0.0 +importlib-metadata==8.5.0 # via dask -kiwisolver==1.4.5 +iniconfig==2.0.0 + # via pytest +kiwisolver==1.4.7 # via matplotlib locket==1.0.0 # via partd -matplotlib==3.9.1 +matplotlib==3.9.2 # via plopp -networkx==3.3 +networkx==3.4.2 # via cyclebane -numpy==2.0.1 +numpy==2.1.3 # via # contourpy # h5py @@ -54,28 +57,38 @@ numpy==2.0.1 # pandas # scipp # scipy -pandas==2.2.2 +packaging==24.2 + # via + # dask + # matplotlib + # pooch + # pytest +pandas==2.2.3 # via -r nightly.in partd==1.4.2 # via dask -pillow==10.4.0 +pillow==11.0.0 # via matplotlib -platformdirs==4.2.2 +platformdirs==4.3.6 # via pooch plopp @ git+https://github.com/scipp/plopp@main # via -r nightly.in +pluggy==1.5.0 + # via pytest pooch==1.8.2 # via -r nightly.in -pyparsing==3.1.2 +pyparsing==3.2.0 # via matplotlib +pytest==8.3.3 + # via -r nightly.in python-dateutil==2.9.0.post0 # via # matplotlib # pandas # scippnexus -pytz==2024.1 +pytz==2024.2 # via pandas -pyyaml==6.0.1 +pyyaml==6.0.2 # via dask requests==2.32.3 # via pooch @@ -87,17 +100,19 @@ scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-c # scippnexus scippnexus @ git+https://github.com/scipp/scippnexus@main # via -r nightly.in -scipy==1.14.0 +scipy==1.14.1 # via scippnexus six==1.16.0 # via python-dateutil -toolz==0.12.1 +tomli==2.1.0 + # via pytest +toolz==1.0.0 # via # dask # partd -tzdata==2024.1 +tzdata==2024.2 # via pandas -urllib3==2.2.2 +urllib3==2.2.3 # via requests -zipp==3.19.2 +zipp==3.21.0 # via importlib-metadata diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index e106d602..e107d915 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -7,19 +7,19 @@ # cfgv==3.4.0 # via pre-commit -distlib==0.3.8 +distlib==0.3.9 # via virtualenv -filelock==3.15.4 +filelock==3.16.1 # via virtualenv -identify==2.6.0 +identify==2.6.2 # via pre-commit nodeenv==1.9.1 # via pre-commit -platformdirs==4.2.2 +platformdirs==4.3.6 # via virtualenv -pre-commit==3.7.1 +pre-commit==4.0.1 # via -r static.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via pre-commit -virtualenv==20.26.3 +virtualenv==20.27.1 # via pre-commit diff --git a/packages/essnmx/requirements/wheels.txt b/packages/essnmx/requirements/wheels.txt index a1fa46e2..6a1c0600 100644 --- a/packages/essnmx/requirements/wheels.txt +++ b/packages/essnmx/requirements/wheels.txt @@ -5,11 +5,11 @@ # # pip-compile-multi # -build==1.2.1 +build==1.2.2.post1 # via -r wheels.in -packaging==24.1 +packaging==24.2 # via build -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via build -tomli==2.0.1 +tomli==2.1.0 # via build diff --git a/packages/essnmx/src/ess/nmx/__init__.py b/packages/essnmx/src/ess/nmx/__init__.py index d481b0a5..76f6c76e 100644 --- a/packages/essnmx/src/ess/nmx/__init__.py +++ b/packages/essnmx/src/ess/nmx/__init__.py @@ -5,7 +5,7 @@ import importlib.metadata try: - __version__ = importlib.metadata.version(__package__ or __name__) + __version__ = importlib.metadata.version("essnmx") except importlib.metadata.PackageNotFoundError: __version__ = "0.0.0" diff --git a/packages/essnmx/tests/package_test.py b/packages/essnmx/tests/package_test.py index 5e6fc243..0bf3d352 100644 --- a/packages/essnmx/tests/package_test.py +++ b/packages/essnmx/tests/package_test.py @@ -1,7 +1,20 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) + +"""Tests of package integrity. + +Note that additional imports need to be added for repositories that +contain multiple packages. +""" + from ess import nmx as pkg def test_has_version(): assert hasattr(pkg, '__version__') + + +# This is for CI package tests. They need to run tests with minimal dependencies, +# that is, without installing pytest. This code does not affect pytest. +if __name__ == '__main__': + test_has_version() From f69363484383f0182bc3b39ace027de44c377567 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Fri, 22 Nov 2024 12:24:36 +0000 Subject: [PATCH 225/403] Apply automatic formatting --- packages/essnmx/requirements/make_base.py | 4 +++- packages/essnmx/tests/conftest.py | 2 +- packages/essnmx/tests/exporter_test.py | 3 ++- packages/essnmx/tests/loader_test.py | 5 +++-- packages/essnmx/tests/mtz_io_test.py | 13 +++++++------ packages/essnmx/tests/scaling_test.py | 5 +++-- packages/essnmx/tests/workflow_test.py | 5 +++-- 7 files changed, 22 insertions(+), 15 deletions(-) diff --git a/packages/essnmx/requirements/make_base.py b/packages/essnmx/requirements/make_base.py index 3c70a5cc..01cd4587 100644 --- a/packages/essnmx/requirements/make_base.py +++ b/packages/essnmx/requirements/make_base.py @@ -67,7 +67,9 @@ def as_nightly(repo: str) -> str: nightly = tuple(args.nightly.split(",") if args.nightly else []) -nightly_dependencies = [dep for dep in dependencies + test_dependencies if not dep.startswith(nightly)] +nightly_dependencies = [ + dep for dep in dependencies + test_dependencies if not dep.startswith(nightly) +] nightly_dependencies += [as_nightly(arg) for arg in nightly] write_dependencies("nightly", nightly_dependencies) diff --git a/packages/essnmx/tests/conftest.py b/packages/essnmx/tests/conftest.py index cebcd9ef..33d8e057 100644 --- a/packages/essnmx/tests/conftest.py +++ b/packages/essnmx/tests/conftest.py @@ -8,6 +8,6 @@ import pytest -@pytest.fixture() +@pytest.fixture def mcstas_2_deprecation_warning_context() -> partial[AbstractContextManager]: return partial(pytest.warns, DeprecationWarning, match="McStas") diff --git a/packages/essnmx/tests/exporter_test.py b/packages/essnmx/tests/exporter_test.py index aab78aad..80b8f495 100644 --- a/packages/essnmx/tests/exporter_test.py +++ b/packages/essnmx/tests/exporter_test.py @@ -5,11 +5,12 @@ import numpy as np import pytest import scipp as sc + from ess.nmx.nexus import export_as_nexus from ess.nmx.reduction import NMXReducedData -@pytest.fixture() +@pytest.fixture def reduced_data() -> NMXReducedData: rng = np.random.default_rng(42) id_list = sc.array(dims=['event'], values=rng.integers(0, 12, size=100)) diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 1e8cca9c..d3dadc40 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -8,6 +8,8 @@ import sciline as sl import scipp as sc import scippnexus as snx +from scipp.testing import assert_allclose, assert_identical + from ess.nmx import default_parameters from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample from ess.nmx.mcstas.load import bank_names_to_detector_names, load_crystal_rotation @@ -19,7 +21,6 @@ FilePath, MaximumProbability, ) -from scipp.testing import assert_allclose, assert_identical sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) from mcstas_description_examples import ( @@ -192,7 +193,7 @@ def test_file_reader_mcstas_additional_fields(tmp_mcstas_file: pathlib.Path) -> assert isinstance(dg, sc.DataGroup) -@pytest.fixture() +@pytest.fixture def rotation_mission_tmp_file(tmp_mcstas_file: pathlib.Path) -> pathlib.Path: import h5py diff --git a/packages/essnmx/tests/mtz_io_test.py b/packages/essnmx/tests/mtz_io_test.py index 4887d02c..33981359 100644 --- a/packages/essnmx/tests/mtz_io_test.py +++ b/packages/essnmx/tests/mtz_io_test.py @@ -5,6 +5,7 @@ import gemmi import pytest import scipp as sc + from ess.nmx import mtz_io from ess.nmx.data import get_small_mtz_samples from ess.nmx.mtz_io import ( @@ -34,7 +35,7 @@ def test_gemmi_mtz(file_path: pathlib.Path) -> None: assert len(mtz.columns[0]) == 100 # Number of samples, hard-coded value -@pytest.fixture() +@pytest.fixture def gemmi_mtz_object(file_path: pathlib.Path) -> gemmi.Mtz: return read_mtz_file(MTZFilePath(file_path)) @@ -64,7 +65,7 @@ def test_mtz_to_process_pandas_dataframe(gemmi_mtz_object: gemmi.Mtz) -> None: assert "hkl_asu" not in df.columns # It should be done on merged dataframes -@pytest.fixture() +@pytest.fixture def mtz_list() -> list[gemmi.Mtz]: return [ read_mtz_file(MTZFilePath(file_path)) for file_path in get_small_mtz_samples() @@ -78,7 +79,7 @@ def test_get_space_group_with_spacegroup_desc() -> None: ) -@pytest.fixture() +@pytest.fixture def conflicting_mtz_series( mtz_list: list[gemmi.Mtz], ) -> list[gemmi.Mtz]: @@ -103,14 +104,14 @@ def test_get_unique_space_group_raises_on_conflict( mtz_io.get_unique_space_group(*space_groups) -@pytest.fixture() +@pytest.fixture def merged_mtz_dataframe(mtz_list: list[gemmi.Mtz]) -> MtzDataFrame: """Tests if the merged data frame has the expected columns.""" reduced_mtz = [process_single_mtz_to_dataframe(mtz) for mtz in mtz_list] return mtz_io.merge_mtz_dataframes(*reduced_mtz) -@pytest.fixture() +@pytest.fixture def nmx_data_frame( mtz_list: list[gemmi.Mtz], merged_mtz_dataframe: MtzDataFrame, @@ -134,7 +135,7 @@ def test_process_merged_mtz_dataframe( assert "hkl_asu" in nmx_data_frame.columns -@pytest.fixture() +@pytest.fixture def nmx_data_array(nmx_data_frame: NMXMtzDataFrame) -> NMXMtzDataArray: return nmx_mtz_dataframe_to_scipp_dataarray(nmx_data_frame) diff --git a/packages/essnmx/tests/scaling_test.py b/packages/essnmx/tests/scaling_test.py index 0c5fa34c..219603bc 100644 --- a/packages/essnmx/tests/scaling_test.py +++ b/packages/essnmx/tests/scaling_test.py @@ -2,6 +2,7 @@ # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) import pytest import scipp as sc + from ess.nmx.scaling import ( ReferenceIntensities, estimate_scale_factor_per_hkl_asu_from_reference, @@ -10,7 +11,7 @@ ) -@pytest.fixture() +@pytest.fixture def nmx_data_array() -> sc.DataArray: da = sc.DataArray( data=sc.array(dims=["row"], values=[1, 2, 3, 4, 5, 3.1, 3.2]), @@ -52,7 +53,7 @@ def test_get_reference_bin_middle(nmx_data_array: sc.DataArray) -> None: ) -@pytest.fixture() +@pytest.fixture def reference_bin(nmx_data_array: sc.DataArray) -> ReferenceIntensities: binned = nmx_data_array.bin({"wavelength": 6}) reference_wavelength = get_reference_wavelength(binned, reference_wavelength=None) diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index 6f38c040..cf8e14d0 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -4,6 +4,7 @@ import pytest import sciline as sl import scipp as sc + from ess.nmx import default_parameters from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample from ess.nmx.mcstas.load import providers as load_providers @@ -22,7 +23,7 @@ def mcstas_file_path( return request.param() -@pytest.fixture() +@pytest.fixture def mcstas_workflow(mcstas_file_path: str) -> sl.Pipeline: return sl.Pipeline( [*load_providers, bin_time_of_arrival], @@ -34,7 +35,7 @@ def mcstas_workflow(mcstas_file_path: str) -> sl.Pipeline: ) -@pytest.fixture() +@pytest.fixture def multi_bank_mcstas_workflow(mcstas_workflow: sl.Pipeline) -> sl.Pipeline: pl = mcstas_workflow.copy() pl[NMXReducedData] = ( From cd9e61903877b1d33704ac2c4a833786a5126f65 Mon Sep 17 00:00:00 2001 From: Mridul Seth Date: Fri, 22 Nov 2024 13:28:03 +0100 Subject: [PATCH 226/403] fix pre-commit --- packages/essnmx/src/ess/nmx/scaling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 5d135b6f..837ebcb3 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -139,7 +139,7 @@ def estimate_scale_factor_per_hkl_asu_from_reference( ) -> EstimatedScaleFactor: """Calculate the estimated scale factor per ``hkl_asu``. - The estimated scale factor is calculatd as the average + The estimated scale factor is calculated as the average of the inverse of the non-empty reference intensities. It is part of the calculation of estimated scaled intensities From 5891313fa3db9cfb58c8c8949af937fa8351946c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 29 Nov 2024 17:49:52 +0000 Subject: [PATCH 227/403] Bump scipp from 24.11.1 to 24.11.2 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 24.11.1 to 24.11.2. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/24.11.1...24.11.2) --- updated-dependencies: - dependency-name: scipp dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 67d7d683..9a5625cb 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -85,7 +85,7 @@ requests==2.32.3 # via pooch sciline==24.10.0 # via -r base.in -scipp==24.11.1 +scipp==24.11.2 # via # -r base.in # scippnexus From 17c9657c70e30bf7ac56491225832d67cb4d409c Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Mon, 13 Jan 2025 15:20:01 +0100 Subject: [PATCH 228/403] fix: remove dummy row if present --- packages/essnmx/src/ess/nmx/mcstas/load.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/essnmx/src/ess/nmx/mcstas/load.py b/packages/essnmx/src/ess/nmx/mcstas/load.py index 02ed1799..b7942aec 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/load.py +++ b/packages/essnmx/src/ess/nmx/mcstas/load.py @@ -48,6 +48,10 @@ def load_raw_event_data( root = f["entry1/data"] (bank_name,) = (name for name in root.keys() if bank_name in name) data = root[bank_name]["events"][()].rename_dims({'dim_0': 'event'}) + if (data.values[0] == 0).all(): + # McStas can add an extra event line containing 0,0,0,0,0,0 + # This line should not be included so we skip it. + data = data["event", 1:] return sc.DataArray( coords={ 'id': sc.array( From 3beb7f0eaafb9673d75a8923d7a6d0cbe26d6562 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Jan 2025 17:21:06 +0000 Subject: [PATCH 229/403] Bump scipp from 24.11.2 to 25.1.0 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 24.11.2 to 25.1.0. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/24.11.2...25.01.0) --- updated-dependencies: - dependency-name: scipp dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 9a5625cb..5d6f7fcd 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -85,7 +85,7 @@ requests==2.32.3 # via pooch sciline==24.10.0 # via -r base.in -scipp==24.11.2 +scipp==25.1.0 # via # -r base.in # scippnexus From 0e7fd84fc76849e508f11c979a17f51312ce8e49 Mon Sep 17 00:00:00 2001 From: jokasimr Date: Fri, 24 Jan 2025 15:29:43 +0100 Subject: [PATCH 230/403] fix: use float64 to avoid truncation (#100) * fix: use float64 to avoid truncation * explanation --- packages/essnmx/src/ess/nmx/scaling.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index 837ebcb3..c76ae0d8 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -359,6 +359,10 @@ def polyval_wavelength( """ + # We need to use float64 precision because + # the curve fit routine depends on finite difference + # estimates of the derivative. + wavelength = wavelength.to(dtype='float64') out = sc.zeros_like(wavelength) out.unit = out_unit xk = sc.ones_like(wavelength) From 8419bbd823078c55873a232da82af39f1a657717 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 4 Feb 2025 17:47:03 +0000 Subject: [PATCH 231/403] Bump scipp from 25.1.0 to 25.2.0 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 25.1.0 to 25.2.0. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/25.01.0...25.02.0) --- updated-dependencies: - dependency-name: scipp dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 5d6f7fcd..ce305c8b 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -85,7 +85,7 @@ requests==2.32.3 # via pooch sciline==24.10.0 # via -r base.in -scipp==25.1.0 +scipp==25.2.0 # via # -r base.in # scippnexus From 1bf0da0c80af55a92ad7e5b98dcc148b113d18d7 Mon Sep 17 00:00:00 2001 From: Mridul Seth Date: Wed, 12 Feb 2025 17:00:07 +0100 Subject: [PATCH 232/403] copier update --- packages/essnmx/.copier-answers.yml | 4 +- packages/essnmx/.github/workflows/docs.yml | 2 +- .../workflows/weekly_windows_macos.yml | 42 +++++++++++++++++++ packages/essnmx/.gitignore | 3 +- packages/essnmx/.pre-commit-config.yaml | 2 +- packages/essnmx/LICENSE | 2 +- packages/essnmx/docs/conf.py | 8 ++-- packages/essnmx/pyproject.toml | 3 +- packages/essnmx/requirements/make_base.py | 15 ++++--- packages/essnmx/src/ess/nmx/__init__.py | 8 ++-- packages/essnmx/tests/package_test.py | 2 +- packages/essnmx/tox.ini | 5 ++- 12 files changed, 72 insertions(+), 24 deletions(-) create mode 100644 packages/essnmx/.github/workflows/weekly_windows_macos.yml diff --git a/packages/essnmx/.copier-answers.yml b/packages/essnmx/.copier-answers.yml index f758cf37..18cbbc8f 100644 --- a/packages/essnmx/.copier-answers.yml +++ b/packages/essnmx/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY -_commit: aa5dc5e +_commit: 5c4fd02 _src_path: gh:scipp/copier_template description: Data reduction for NMX at the European Spallation Source. max_python: '3.13' @@ -10,4 +10,4 @@ orgname: scipp prettyname: ESSnmx projectname: essnmx related_projects: Scipp,Sciline,Plopp,ScippNexus -year: 2024 +year: 2025 diff --git a/packages/essnmx/.github/workflows/docs.yml b/packages/essnmx/.github/workflows/docs.yml index 72659cbd..a90bcf10 100644 --- a/packages/essnmx/.github/workflows/docs.yml +++ b/packages/essnmx/.github/workflows/docs.yml @@ -69,7 +69,7 @@ jobs: name: docs_html path: html/ - - uses: JamesIves/github-pages-deploy-action@v4.6.9 + - uses: JamesIves/github-pages-deploy-action@v4.7.2 if: ${{ inputs.publish }} with: branch: gh-pages diff --git a/packages/essnmx/.github/workflows/weekly_windows_macos.yml b/packages/essnmx/.github/workflows/weekly_windows_macos.yml new file mode 100644 index 00000000..1544d7f9 --- /dev/null +++ b/packages/essnmx/.github/workflows/weekly_windows_macos.yml @@ -0,0 +1,42 @@ +name: Windows and MacOS weekly tests + +on: + workflow_dispatch: + schedule: + - cron: '0 2 * * 1' + +jobs: + pytox: + name: Python and Tox env + runs-on: 'ubuntu-24.04' + outputs: + min_python: ${{ steps.vars.outputs.min_python }} + min_tox_env: ${{ steps.vars.outputs.min_tox_env }} + steps: + - uses: actions/checkout@v4 + - name: Get Python version for other CI jobs + id: vars + run: | + echo "min_python=$(cat .github/workflows/python-version-ci)" >> $GITHUB_OUTPUT + echo "min_tox_env=py$(cat .github/workflows/python-version-ci | sed 's/\.//g')" >> $GITHUB_OUTPUT + tests: + name: Tests + needs: pytox + strategy: + matrix: + os: ['macos-latest', 'windows-latest'] + python: + - version: '${{needs.pytox.outputs.min_python}}' + tox-env: '${{needs.pytox.outputs.min_tox_env}}' + uses: ./.github/workflows/test.yml + with: + os-variant: ${{ matrix.os }} + python-version: ${{ matrix.python.version }} + tox-env: ${{ matrix.python.tox-env }} + + docs: + needs: tests + uses: ./.github/workflows/docs.yml + with: + publish: false + branch: ${{ github.head_ref == '' && github.ref_name || github.head_ref }} diff --git a/packages/essnmx/.gitignore b/packages/essnmx/.gitignore index 3e02ecce..340e6499 100644 --- a/packages/essnmx/.gitignore +++ b/packages/essnmx/.gitignore @@ -4,7 +4,8 @@ dist html .tox *.egg-info -uv.lock # we lock dependencies with pip-compile, not uv +# we lock dependencies with pip-compile, not uv +uv.lock *.sw? diff --git a/packages/essnmx/.pre-commit-config.yaml b/packages/essnmx/.pre-commit-config.yaml index f83362c6..0f3f9a95 100644 --- a/packages/essnmx/.pre-commit-config.yaml +++ b/packages/essnmx/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: args: [ "--drop-empty-cells", "--extra-keys 'metadata.language_info.version cell.metadata.jp-MarkdownHeadingCollapsed cell.metadata.pycharm'" ] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.9 + rev: v0.8.0 hooks: - id: ruff args: [ --fix ] diff --git a/packages/essnmx/LICENSE b/packages/essnmx/LICENSE index 54b3cf4d..7d62083d 100644 --- a/packages/essnmx/LICENSE +++ b/packages/essnmx/LICENSE @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2024, Scipp contributors (https://github.com/scipp) +Copyright (c) 2025, Scipp contributors (https://github.com/scipp) All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/packages/essnmx/docs/conf.py b/packages/essnmx/docs/conf.py index 5dd75d9d..1601faf3 100644 --- a/packages/essnmx/docs/conf.py +++ b/packages/essnmx/docs/conf.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +# Copyright (c) 2025 Scipp contributors (https://github.com/scipp) import doctest import os @@ -14,9 +14,9 @@ logger = logging.getLogger(__name__) # General information about the project. -project = "ESSnmx" -copyright = "2024 Scipp contributors" -author = "Scipp contributors" +project = 'ESSnmx' +copyright = '2025 Scipp contributors' +author = 'Scipp contributors' html_show_sourcelink = True diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index ef143600..91755912 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -86,13 +86,12 @@ ignore = [ # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules "COM812", "COM819", "D206", "D300", "E111", "E114", "E117", "ISC001", "ISC002", "Q000", "Q001", "Q002", "Q003", "W191", ] -fixable = ["B010", "I001", "PT001"] +fixable = ["B010", "I001", "PT001", "RUF022"] isort.known-first-party = ["ess.nmx"] pydocstyle.convention = "numpy" [tool.ruff.lint.per-file-ignores] # those files have an increased risk of relying on import order -"__init__.py" = ["I"] "tests/*" = [ "S101", # asserts are fine in tests "B018", # 'useless expressions' are ok because some tests just check for exceptions diff --git a/packages/essnmx/requirements/make_base.py b/packages/essnmx/requirements/make_base.py index 01cd4587..2cda547f 100644 --- a/packages/essnmx/requirements/make_base.py +++ b/packages/essnmx/requirements/make_base.py @@ -1,4 +1,3 @@ -import sys from argparse import ArgumentParser from pathlib import Path @@ -58,11 +57,15 @@ def as_nightly(repo: str) -> str: else: org = "scipp" if repo == "scipp": - version = f"cp{sys.version_info.major}{sys.version_info.minor}" - base = "https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly" - suffix = "manylinux_2_17_x86_64.manylinux2014_x86_64.whl" - prefix = "scipp @ " - return prefix + "-".join([base, version, version, suffix]) + # With the standard pip resolver index-url takes precedence over + # extra-index-url but with uv it's reversed, so if we move to tox-uv + # this needs to be reversed. + return ( + "scipp\n" + "--index-url=https://pypi.anaconda.org/scipp-nightly-wheels/simple/\n" + "--extra-index-url=https://pypi.org/simple\n" + "--pre" + ) return f"{repo} @ git+https://github.com/{org}/{repo}@main" diff --git a/packages/essnmx/src/ess/nmx/__init__.py b/packages/essnmx/src/ess/nmx/__init__.py index 76f6c76e..82d668bb 100644 --- a/packages/essnmx/src/ess/nmx/__init__.py +++ b/packages/essnmx/src/ess/nmx/__init__.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) -# ruff: noqa: E402, F401 +# Copyright (c) 2025 Scipp contributors (https://github.com/scipp) +# ruff: noqa: E402, F401, I import importlib.metadata @@ -20,8 +20,8 @@ del MaximumProbability __all__ = [ - "small_mcstas_3_sample", - "NMXReducedData", "NMXData", + "NMXReducedData", "default_parameters", + "small_mcstas_3_sample", ] diff --git a/packages/essnmx/tests/package_test.py b/packages/essnmx/tests/package_test.py index 0bf3d352..8e2cb7de 100644 --- a/packages/essnmx/tests/package_test.py +++ b/packages/essnmx/tests/package_test.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +# Copyright (c) 2025 Scipp contributors (https://github.com/scipp) """Tests of package integrity. diff --git a/packages/essnmx/tox.ini b/packages/essnmx/tox.ini index b4f9ce2a..e1e1cbd0 100644 --- a/packages/essnmx/tox.ini +++ b/packages/essnmx/tox.ini @@ -10,6 +10,9 @@ commands = pytest {posargs} [testenv:nightly] deps = -r requirements/nightly.txt +setenv = + PIP_INDEX_URL = https://pypi.anaconda.org/scipp-nightly-wheels/simple + PIP_EXTRA_INDEX_URL = https://pypi.org/simple commands = pytest {posargs} [testenv:unpinned] @@ -64,4 +67,4 @@ deps = skip_install = true changedir = requirements commands = python ./make_base.py --nightly scipp,sciline,scippnexus,plopp - pip-compile-multi -d . --backtracking + pip-compile-multi -d . --backtracking --annotate-index From 70a8978b317da370da79f667fedcb4aa5ca6013c Mon Sep 17 00:00:00 2001 From: Mridul Seth Date: Wed, 12 Feb 2025 17:09:14 +0100 Subject: [PATCH 233/403] tox -e deps --- packages/essnmx/requirements/base.txt | 40 ++++++++--------- packages/essnmx/requirements/basetest.txt | 4 +- packages/essnmx/requirements/ci.txt | 24 +++++----- packages/essnmx/requirements/dev.txt | 32 +++++++------ packages/essnmx/requirements/docs.txt | 55 ++++++++++++----------- packages/essnmx/requirements/mypy.txt | 2 +- packages/essnmx/requirements/nightly.in | 5 ++- packages/essnmx/requirements/nightly.txt | 54 ++++++++++++---------- packages/essnmx/requirements/static.txt | 8 ++-- packages/essnmx/requirements/wheels.txt | 2 +- 10 files changed, 118 insertions(+), 108 deletions(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index ce305c8b..d896da5e 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -5,13 +5,13 @@ # # pip-compile-multi # -certifi==2024.8.30 +certifi==2025.1.31 # via requests -charset-normalizer==3.4.0 +charset-normalizer==3.4.1 # via requests -click==8.1.7 +click==8.1.8 # via dask -cloudpickle==3.1.0 +cloudpickle==3.1.1 # via dask contourpy==1.3.1 # via matplotlib @@ -19,15 +19,15 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -dask==2024.11.2 +dask==2025.1.0 # via -r base.in defusedxml==0.7.1 # via -r base.in -fonttools==4.55.0 +fonttools==4.56.0 # via matplotlib -fsspec==2024.10.0 +fsspec==2025.2.0 # via dask -gemmi==0.6.7 +gemmi==0.7.0 # via -r base.in graphviz==0.20.3 # via -r base.in @@ -35,17 +35,17 @@ h5py==3.12.1 # via scippnexus idna==3.10 # via requests -importlib-metadata==8.5.0 +importlib-metadata==8.6.1 # via dask -kiwisolver==1.4.7 +kiwisolver==1.4.8 # via matplotlib locket==1.0.0 # via partd -matplotlib==3.9.2 +matplotlib==3.10.0 # via plopp networkx==3.4.2 # via cyclebane -numpy==2.1.3 +numpy==2.2.2 # via # contourpy # h5py @@ -62,7 +62,7 @@ pandas==2.2.3 # via -r base.in partd==1.4.2 # via dask -pillow==11.0.0 +pillow==11.1.0 # via matplotlib platformdirs==4.3.6 # via pooch @@ -70,14 +70,14 @@ plopp==24.10.0 # via -r base.in pooch==1.8.2 # via -r base.in -pyparsing==3.2.0 +pyparsing==3.2.1 # via matplotlib python-dateutil==2.9.0.post0 # via # matplotlib # pandas # scippnexus -pytz==2024.2 +pytz==2025.1 # via pandas pyyaml==6.0.2 # via dask @@ -89,19 +89,19 @@ scipp==25.2.0 # via # -r base.in # scippnexus -scippnexus==24.11.0 +scippnexus==24.11.1 # via -r base.in -scipy==1.14.1 +scipy==1.15.1 # via scippnexus -six==1.16.0 +six==1.17.0 # via python-dateutil toolz==1.0.0 # via # dask # partd -tzdata==2024.2 +tzdata==2025.1 # via pandas -urllib3==2.2.3 +urllib3==2.3.0 # via requests zipp==3.21.0 # via importlib-metadata diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt index 66970e04..e1f778ef 100644 --- a/packages/essnmx/requirements/basetest.txt +++ b/packages/essnmx/requirements/basetest.txt @@ -13,7 +13,7 @@ packaging==24.2 # via pytest pluggy==1.5.0 # via pytest -pytest==8.3.3 +pytest==8.3.4 # via -r basetest.in -tomli==2.1.0 +tomli==2.2.1 # via pytest diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index 14921a29..078cca56 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -5,25 +5,25 @@ # # pip-compile-multi # -cachetools==5.5.0 +cachetools==5.5.1 # via tox -certifi==2024.8.30 +certifi==2025.1.31 # via requests chardet==5.2.0 # via tox -charset-normalizer==3.4.0 +charset-normalizer==3.4.1 # via requests colorama==0.4.6 # via tox distlib==0.3.9 # via virtualenv -filelock==3.16.1 +filelock==3.17.0 # via # tox # virtualenv -gitdb==4.0.11 +gitdb==4.0.12 # via gitpython -gitpython==3.1.43 +gitpython==3.1.44 # via -r ci.in idna==3.10 # via requests @@ -38,21 +38,21 @@ platformdirs==4.3.6 # virtualenv pluggy==1.5.0 # via tox -pyproject-api==1.8.0 +pyproject-api==1.9.0 # via tox requests==2.32.3 # via -r ci.in -smmap==5.0.1 +smmap==5.0.2 # via gitdb -tomli==2.1.0 +tomli==2.2.1 # via # pyproject-api # tox -tox==4.23.2 +tox==4.24.1 # via -r ci.in typing-extensions==4.12.2 # via tox -urllib3==2.2.3 +urllib3==2.3.0 # via requests -virtualenv==20.27.1 +virtualenv==20.29.2 # via tox diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index f9cab17c..814d4763 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -14,7 +14,7 @@ -r wheels.txt annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.8.0 # via # httpx # jupyter-server @@ -40,13 +40,13 @@ h11==0.14.0 # via httpcore httpcore==1.0.7 # via httpx -httpx==0.27.2 +httpx==0.28.1 # via jupyterlab isoduration==20.11.0 # via jsonschema jinja2-ansible-filters==1.3.2 # via copier -json5==0.9.28 +json5==0.10.0 # via jupyterlab-server jsonpointer==3.0.0 # via jsonschema @@ -55,11 +55,11 @@ jsonschema[format-nongpl]==4.23.0 # jupyter-events # jupyterlab-server # nbformat -jupyter-events==0.10.0 +jupyter-events==0.12.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab -jupyter-server==2.14.2 +jupyter-server==2.15.0 # via # jupyter-lsp # jupyterlab @@ -67,7 +67,7 @@ jupyter-server==2.14.2 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.3.1 +jupyterlab==4.3.5 # via -r dev.in jupyterlab-server==2.27.3 # via jupyterlab @@ -77,23 +77,23 @@ overrides==7.7.0 # via jupyter-server pathspec==0.12.1 # via copier -pip-compile-multi==2.6.4 +pip-compile-multi==2.7.1 # via -r dev.in pip-tools==7.4.1 # via pip-compile-multi plumbum==1.9.0 # via copier -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via jupyter-server pycparser==2.22 # via cffi -pydantic==2.10.1 +pydantic==2.10.6 # via copier -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic -python-json-logger==2.0.7 +python-json-logger==3.2.1 # via jupyter-events -questionary==1.10.0 +questionary==2.1.0 # via copier rfc3339-validator==0.1.4 # via @@ -106,16 +106,14 @@ rfc3986-validator==0.1.1 send2trash==1.8.3 # via jupyter-server sniffio==1.3.1 - # via - # anyio - # httpx + # via anyio terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals toposort==1.10 # via pip-compile-multi -types-python-dateutil==2.9.0.20241003 +types-python-dateutil==2.9.0.20241206 # via arrow uri-template==1.3.0 # via jsonschema @@ -123,7 +121,7 @@ webcolors==24.11.1 # via jsonschema websocket-client==1.8.0 # via jupyter-server -wheel==0.45.0 +wheel==0.45.1 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 5dd4a562..aa04ca18 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -12,27 +12,27 @@ alabaster==1.0.0 # via sphinx appnope==0.1.4 # via ipykernel -asttokens==2.4.1 +asttokens==3.0.0 # via stack-data -attrs==24.2.0 +attrs==25.1.0 # via # jsonschema # referencing -babel==2.16.0 +babel==2.17.0 # via # pydata-sphinx-theme # sphinx -beautifulsoup4==4.12.3 +beautifulsoup4==4.13.3 # via # nbconvert # pydata-sphinx-theme -bleach==6.2.0 +bleach[css]==6.2.0 # via nbconvert comm==0.2.2 # via # ipykernel # ipywidgets -debugpy==1.8.9 +debugpy==1.8.12 # via ipykernel decorator==5.1.1 # via ipython @@ -44,9 +44,9 @@ docutils==0.21.2 # sphinx exceptiongroup==1.2.2 # via ipython -executing==2.1.0 +executing==2.2.0 # via stack-data -fastjsonschema==2.20.0 +fastjsonschema==2.21.1 # via nbformat imagesize==1.4.1 # via sphinx @@ -54,7 +54,7 @@ ipydatawidgets==4.3.5 # via pythreejs ipykernel==6.29.5 # via -r docs.in -ipython==8.29.0 +ipython==8.32.0 # via # -r docs.in # ipykernel @@ -65,7 +65,7 @@ ipywidgets==8.1.5 # pythreejs jedi==0.19.2 # via ipython -jinja2==3.1.4 +jinja2==3.1.5 # via # myst-parser # nbconvert @@ -106,22 +106,22 @@ mdit-py-plugins==0.4.2 # via myst-parser mdurl==0.1.2 # via markdown-it-py -mistune==3.0.2 +mistune==3.1.1 # via nbconvert mpltoolbox==24.5.1 # via scippneutron -myst-parser==4.0.0 +myst-parser==4.0.1 # via -r docs.in -nbclient==0.10.0 +nbclient==0.10.2 # via nbconvert -nbconvert==7.16.4 +nbconvert==7.16.6 # via nbsphinx nbformat==5.10.4 # via # nbclient # nbconvert # nbsphinx -nbsphinx==0.9.5 +nbsphinx==0.9.6 # via -r docs.in nest-asyncio==1.6.0 # via ipykernel @@ -131,17 +131,17 @@ parso==0.8.4 # via jedi pexpect==4.9.0 # via ipython -prompt-toolkit==3.0.48 +prompt-toolkit==3.0.50 # via ipython -psutil==6.1.0 +psutil==6.1.1 # via ipykernel ptyprocess==0.7.0 # via pexpect pure-eval==0.2.3 # via stack-data -pydata-sphinx-theme==0.16.0 +pydata-sphinx-theme==0.16.1 # via -r docs.in -pygments==2.18.0 +pygments==2.19.1 # via # accessible-pygments # ipython @@ -150,19 +150,19 @@ pygments==2.18.0 # sphinx pythreejs==2.4.2 # via -r docs.in -pyzmq==26.2.0 +pyzmq==26.2.1 # via # ipykernel # jupyter-client -referencing==0.35.1 +referencing==0.36.2 # via # jsonschema # jsonschema-specifications -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing -scippneutron==24.11.0 +scippneutron==25.1.0 # via -r docs.in snowballstemmer==2.2.0 # via sphinx @@ -177,7 +177,7 @@ sphinx==8.1.3 # sphinx-autodoc-typehints # sphinx-copybutton # sphinx-design -sphinx-autodoc-typehints==2.5.0 +sphinx-autodoc-typehints==3.0.1 # via -r docs.in sphinx-copybutton==0.5.2 # via -r docs.in @@ -198,8 +198,8 @@ sphinxcontrib-serializinghtml==2.0.0 stack-data==0.6.3 # via ipython tinycss2==1.4.0 - # via nbconvert -tomli==2.1.0 + # via bleach +tomli==2.2.1 # via sphinx tornado==6.4.2 # via @@ -224,8 +224,11 @@ traittypes==0.2.1 # via ipydatawidgets typing-extensions==4.12.2 # via + # beautifulsoup4 # ipython + # mistune # pydata-sphinx-theme + # referencing wcwidth==0.2.13 # via prompt-toolkit webencodings==0.5.1 diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index 6ffc9ba6..7f020f7a 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -6,7 +6,7 @@ # pip-compile-multi # -r test.txt -mypy==1.13.0 +mypy==1.15.0 # via -r mypy.in mypy-extensions==1.0.0 # via mypy diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index bca11276..f34be063 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -8,7 +8,10 @@ pandas gemmi defusedxml pytest -scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +scipp +--index-url=https://pypi.anaconda.org/scipp-nightly-wheels/simple/ +--extra-index-url=https://pypi.org/simple +--pre sciline @ git+https://github.com/scipp/sciline@main scippnexus @ git+https://github.com/scipp/scippnexus@main plopp @ git+https://github.com/scipp/plopp@main diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index daf1fd6e..d6420af3 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,17 +1,20 @@ -# SHA1:5567d188ad845d7f7faf225ea3ed89fbc989745b +# SHA1:38e72cba5ebcd6086e53af32007e3d9b1e5f12bf # # This file is autogenerated by pip-compile-multi # To update, run: # # pip-compile-multi # -certifi==2024.8.30 +--index-url https://pypi.anaconda.org/scipp-nightly-wheels/simple/ +--extra-index-url https://pypi.org/simple + +certifi==2025.1.31 # via requests -charset-normalizer==3.4.0 +charset-normalizer==3.4.1 # via requests -click==8.1.7 +click==8.1.8 # via dask -cloudpickle==3.1.0 +cloudpickle==3.1.1 # via dask contourpy==1.3.1 # via matplotlib @@ -19,17 +22,17 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -dask==2024.11.2 +dask==2025.1.0 # via -r nightly.in -defusedxml==0.7.1 +defusedxml==0.8.0rc2 # via -r nightly.in exceptiongroup==1.2.2 # via pytest -fonttools==4.55.0 +fonttools==4.56.0 # via matplotlib -fsspec==2024.10.0 +fsspec==2025.2.0 # via dask -gemmi==0.6.7 +gemmi==0.7.0 # via -r nightly.in graphviz==0.20.3 # via -r nightly.in @@ -37,19 +40,21 @@ h5py==3.12.1 # via scippnexus idna==3.10 # via requests -importlib-metadata==8.5.0 +importlib-metadata==8.6.1 # via dask iniconfig==2.0.0 # via pytest -kiwisolver==1.4.7 +kiwisolver==1.4.8 # via matplotlib +lazy-loader==0.4 + # via plopp locket==1.0.0 # via partd -matplotlib==3.9.2 +matplotlib==3.10.0 # via plopp networkx==3.4.2 # via cyclebane -numpy==2.1.3 +numpy==2.2.2 # via # contourpy # h5py @@ -60,6 +65,7 @@ numpy==2.1.3 packaging==24.2 # via # dask + # lazy-loader # matplotlib # pooch # pytest @@ -67,7 +73,7 @@ pandas==2.2.3 # via -r nightly.in partd==1.4.2 # via dask -pillow==11.0.0 +pillow==11.1.0 # via matplotlib platformdirs==4.3.6 # via pooch @@ -77,16 +83,16 @@ pluggy==1.5.0 # via pytest pooch==1.8.2 # via -r nightly.in -pyparsing==3.2.0 +pyparsing==3.2.1 # via matplotlib -pytest==8.3.3 +pytest==8.3.4 # via -r nightly.in python-dateutil==2.9.0.post0 # via # matplotlib # pandas # scippnexus -pytz==2024.2 +pytz==2025.1 # via pandas pyyaml==6.0.2 # via dask @@ -94,25 +100,25 @@ requests==2.32.3 # via pooch sciline @ git+https://github.com/scipp/sciline@main # via -r nightly.in -scipp @ https://github.com/scipp/scipp/releases/download/nightly/scipp-nightly-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +scipp==100.0.0.dev0 # via # -r nightly.in # scippnexus scippnexus @ git+https://github.com/scipp/scippnexus@main # via -r nightly.in -scipy==1.14.1 +scipy==1.15.1 # via scippnexus -six==1.16.0 +six==1.17.0 # via python-dateutil -tomli==2.1.0 +tomli==2.2.1 # via pytest toolz==1.0.0 # via # dask # partd -tzdata==2024.2 +tzdata==2025.1 # via pandas -urllib3==2.2.3 +urllib3==2.3.0 # via requests zipp==3.21.0 # via importlib-metadata diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index e107d915..a0569bda 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -9,17 +9,17 @@ cfgv==3.4.0 # via pre-commit distlib==0.3.9 # via virtualenv -filelock==3.16.1 +filelock==3.17.0 # via virtualenv -identify==2.6.2 +identify==2.6.7 # via pre-commit nodeenv==1.9.1 # via pre-commit platformdirs==4.3.6 # via virtualenv -pre-commit==4.0.1 +pre-commit==4.1.0 # via -r static.in pyyaml==6.0.2 # via pre-commit -virtualenv==20.27.1 +virtualenv==20.29.2 # via pre-commit diff --git a/packages/essnmx/requirements/wheels.txt b/packages/essnmx/requirements/wheels.txt index 6a1c0600..bfae20bf 100644 --- a/packages/essnmx/requirements/wheels.txt +++ b/packages/essnmx/requirements/wheels.txt @@ -11,5 +11,5 @@ packaging==24.2 # via build pyproject-hooks==1.2.0 # via build -tomli==2.1.0 +tomli==2.2.1 # via build From 5e88ebe165c1ab1ad7eb98dacec233ba6fe3c145 Mon Sep 17 00:00:00 2001 From: Mridul Seth Date: Wed, 12 Feb 2025 17:09:50 +0100 Subject: [PATCH 234/403] lint --- packages/essnmx/src/ess/nmx/data/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/data/__init__.py b/packages/essnmx/src/ess/nmx/data/__init__.py index 12cf4889..f8803dbb 100644 --- a/packages/essnmx/src/ess/nmx/data/__init__.py +++ b/packages/essnmx/src/ess/nmx/data/__init__.py @@ -6,7 +6,7 @@ _version = "0" -__all__ = ["small_mcstas_2_sample", "small_mcstas_3_sample", "get_path"] +__all__ = ["get_path", "small_mcstas_2_sample", "small_mcstas_3_sample"] def _make_pooch() -> pooch.Pooch: From 1c11046634f26b9fae1cc515ee1a2a8e87248ae1 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 25 Feb 2025 14:08:57 +0100 Subject: [PATCH 235/403] Add warning for the to-be-deprecated interface. --- packages/essnmx/src/ess/nmx/nexus.py | 9 +++++++++ packages/essnmx/tests/exporter_test.py | 5 ++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/nexus.py index 0373db77..81da9037 100644 --- a/packages/essnmx/src/ess/nmx/nexus.py +++ b/packages/essnmx/src/ess/nmx/nexus.py @@ -129,6 +129,15 @@ def export_as_nexus( Currently exporting step is not expected to be part of sciline pipelines. """ + import warnings + + warnings.warn( + DeprecationWarning( + "Exporting to custom NeXus format will be deprecated in the near future." + "Please use ``export_as_nxlauetof`` instead." + ), + stacklevel=1, + ) with h5py.File(output_file, "w") as f: f.attrs["default"] = "NMX_data" nx_entry = _create_root_data_entry(f) diff --git a/packages/essnmx/tests/exporter_test.py b/packages/essnmx/tests/exporter_test.py index 80b8f495..b2efb78b 100644 --- a/packages/essnmx/tests/exporter_test.py +++ b/packages/essnmx/tests/exporter_test.py @@ -70,7 +70,10 @@ def test_mcstas_reduction_export_to_bytestream(reduced_data: NMXReducedData) -> ] with io.BytesIO() as bio: - export_as_nexus(reduced_data, bio) + with pytest.warns( + DeprecationWarning, match='Please use ``export_as_nxlauetof`` instead.' + ): + export_as_nexus(reduced_data, bio) with h5py.File(bio, 'r') as f: assert 'NMX_data' in f nmx_data: h5py.Group = f.require_group('NMX_data') From 04d70bbe788bf9beb1dcfd8f1e7f07e5bc24df9f Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Tue, 25 Feb 2025 16:31:08 +0100 Subject: [PATCH 236/403] Update stacklevel for warning --- packages/essnmx/src/ess/nmx/nexus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/nexus.py index 81da9037..bf6b509d 100644 --- a/packages/essnmx/src/ess/nmx/nexus.py +++ b/packages/essnmx/src/ess/nmx/nexus.py @@ -136,7 +136,7 @@ def export_as_nexus( "Exporting to custom NeXus format will be deprecated in the near future." "Please use ``export_as_nxlauetof`` instead." ), - stacklevel=1, + stacklevel=2, ) with h5py.File(output_file, "w") as f: f.attrs["default"] = "NMX_data" From f985c423dc3657cfc0a1f5ce832a1c8149a8aade Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Wed, 26 Feb 2025 11:06:30 +0100 Subject: [PATCH 237/403] Rename MaximumProbability to MaximumCounts and add MaximumProbability. (#108) * Rename MaximumProbability to MaximumCounts and add MaximumProbability. * Expose scalefactor to be exported. * Update type name. --- packages/essnmx/docs/examples/workflow.ipynb | 6 +-- packages/essnmx/src/ess/nmx/__init__.py | 6 +-- packages/essnmx/src/ess/nmx/mcstas/load.py | 41 +++++++++++++++++--- packages/essnmx/src/ess/nmx/types.py | 8 +++- packages/essnmx/tests/loader_test.py | 9 +---- 5 files changed, 50 insertions(+), 20 deletions(-) diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index 3edd44a4..df80cb0a 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -29,7 +29,7 @@ "wf = McStasWorkflow()\n", "# Replace with the path to your own file\n", "wf[FilePath] = small_mcstas_3_sample()\n", - "wf[MaximumProbability] = 10000\n", + "wf[MaximumCounts] = 10000\n", "wf[TimeBinSteps] = 50" ] }, @@ -172,7 +172,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "nmx-dev-310", "language": "python", "name": "python3" }, @@ -186,7 +186,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/packages/essnmx/src/ess/nmx/__init__.py b/packages/essnmx/src/ess/nmx/__init__.py index 82d668bb..2fbe4c31 100644 --- a/packages/essnmx/src/ess/nmx/__init__.py +++ b/packages/essnmx/src/ess/nmx/__init__.py @@ -13,11 +13,11 @@ from .data import small_mcstas_3_sample from .reduction import NMXData, NMXReducedData -from .types import MaximumProbability +from .types import MaximumCounts -default_parameters = {MaximumProbability: 10000} +default_parameters = {MaximumCounts: 10000} -del MaximumProbability +del MaximumCounts __all__ = [ "NMXData", diff --git a/packages/essnmx/src/ess/nmx/mcstas/load.py b/packages/essnmx/src/ess/nmx/mcstas/load.py index b7942aec..28c451d3 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/load.py +++ b/packages/essnmx/src/ess/nmx/mcstas/load.py @@ -13,7 +13,9 @@ DetectorName, EventData, FilePath, + MaximumCounts, MaximumProbability, + McStasWeight2CountScaleFactor, ProtonCharge, RawEventData, ) @@ -93,24 +95,49 @@ def load_crystal_rotation( ) +def maximum_probability(da: RawEventData) -> MaximumProbability: + """Find the maximum probability in the data.""" + return MaximumProbability(da.data.max()) + + +def mcstas_weight_to_probability_scalefactor( + max_counts: MaximumCounts, max_probability: MaximumProbability +) -> McStasWeight2CountScaleFactor: + """Calculate the scale factor to convert McStas weights to counts. + + max_counts * (probabilities / max_probability) + + Parameters + ---------- + max_counts: + The maximum number of counts after scaling the event counts. + + max_probability: + The maximum probability to scale the weights. + + """ + return McStasWeight2CountScaleFactor( + sc.scalar(max_counts, unit="counts") / max_probability + ) + + def event_weights_from_probability( - da: RawEventData, - max_probability: MaximumProbability, + da: RawEventData, scale_factor: McStasWeight2CountScaleFactor ) -> EventData: """Create event weights by scaling probability data. - event_weights = max_probability * (probabilities / max(probabilities)) + event_weights = max_counts * (probabilities / max_probability) Parameters ---------- da: The probabilities of the events - max_probability: - The maximum probability to scale the weights. + scale_factor: + The scale factor to convert McStas weights to counts """ - return sc.scalar(max_probability, unit='counts') * da / da.max() + return EventData(da * scale_factor) def proton_charge_from_event_data(da: EventData) -> ProtonCharge: @@ -185,6 +212,8 @@ def load_mcstas( detector_name_from_index, load_event_data_bank_name, load_raw_event_data, + maximum_probability, + mcstas_weight_to_probability_scalefactor, event_weights_from_probability, proton_charge_from_event_data, load_crystal_rotation, diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index c4906377..8e7fcd8d 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -15,9 +15,15 @@ """Prefix identifying the event data array containing the events from the selected detector""" -MaximumProbability = NewType("MaximumProbability", int) +MaximumCounts = NewType("MaximumCounts", int) """Maximum number of counts after scaling the event counts""" +MaximumProbability = NewType("MaximumProbability", sc.Variable) +"""Maximum probability to scale the McStas event counts""" + +McStasWeight2CountScaleFactor = NewType("McStasWeight2CountScaleFactor", sc.Variable) +"""Scale factor to convert McStas weights to counts""" + RawEventData = NewType("RawEventData", sc.DataArray) """DataArray containing the event counts read from the McStas file, has coordinates 'id' and 't' """ diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index d3dadc40..90c75533 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -15,12 +15,7 @@ from ess.nmx.mcstas.load import bank_names_to_detector_names, load_crystal_rotation from ess.nmx.mcstas.load import providers as loader_providers from ess.nmx.reduction import NMXData -from ess.nmx.types import ( - DetectorBankPrefix, - DetectorIndex, - FilePath, - MaximumProbability, -) +from ess.nmx.types import DetectorBankPrefix, DetectorIndex, FilePath, MaximumCounts sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) from mcstas_description_examples import ( @@ -54,7 +49,7 @@ def check_nmxdata_properties(dg: NMXData, fast_axis, slow_axis) -> None: # Check maximum value of weights. assert_allclose( dg['weights'].max().data, - sc.scalar(default_parameters[MaximumProbability], unit='counts', dtype=float), + sc.scalar(default_parameters[MaximumCounts], unit='counts', dtype=float), atol=sc.scalar(1e-10, unit='counts'), rtol=sc.scalar(1e-8), ) From a5b68f860ba4644e7ac100f3a1154b5ed1fa2c9a Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Wed, 26 Feb 2025 14:55:23 +0100 Subject: [PATCH 238/403] Update workflow to be able to process chunk by chunk. (#109) * Rename MaximumProbability to MaximumCounts and add MaximumProbability. * Expose scalefactor to be exported. * Update type name. * Update workflow to have more granular steps. * Move scaling to later step. * Remove unecessary argument. * Fix iter chunk slices. --- packages/essnmx/docs/examples/workflow.ipynb | 22 +- packages/essnmx/src/ess/nmx/__init__.py | 8 +- .../essnmx/src/ess/nmx/mcstas/__init__.py | 16 +- packages/essnmx/src/ess/nmx/mcstas/load.py | 200 +++++++++++------- packages/essnmx/src/ess/nmx/mcstas/xml.py | 5 +- packages/essnmx/src/ess/nmx/reduction.py | 75 +++++-- packages/essnmx/src/ess/nmx/types.py | 21 +- packages/essnmx/tests/exporter_test.py | 10 +- packages/essnmx/tests/loader_test.py | 40 ++-- packages/essnmx/tests/workflow_test.py | 50 ++++- 10 files changed, 293 insertions(+), 154 deletions(-) diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/examples/workflow.ipynb index df80cb0a..6fb39732 100644 --- a/packages/essnmx/docs/examples/workflow.ipynb +++ b/packages/essnmx/docs/examples/workflow.ipynb @@ -23,7 +23,7 @@ "from ess.nmx.data import small_mcstas_3_sample\n", "\n", "from ess.nmx.types import *\n", - "from ess.nmx.reduction import NMXData, NMXReducedData, merge_panels\n", + "from ess.nmx.reduction import merge_panels\n", "from ess.nmx.nexus import export_as_nexus\n", "\n", "wf = McStasWorkflow()\n", @@ -64,8 +64,8 @@ "source": [ "# DetectorIndex selects what detector panels to include in the run\n", "# in this case we select all three panels.\n", - "wf[NMXReducedData] = (\n", - " wf[NMXReducedData]\n", + "wf[NMXReducedDataGroup] = (\n", + " wf[NMXReducedDataGroup]\n", " .map({DetectorIndex: sc.arange('panel', 3, unit=None)})\n", " .reduce(index=\"panel\", func=merge_panels)\n", ")" @@ -84,7 +84,7 @@ "metadata": {}, "outputs": [], "source": [ - "wf.visualize(NMXReducedData, graph_attr={\"rankdir\": \"TD\"}, compact=True)" + "wf.visualize(NMXReducedDataGroup, graph_attr={\"rankdir\": \"TD\"}, compact=True)" ] }, { @@ -102,8 +102,8 @@ "source": [ "from cyclebane.graph import NodeName, IndexValues\n", "\n", - "# Event data grouped by pixel id for each of the selected detectors\n", - "targets = [NodeName(NMXData, IndexValues((\"panel\",), (i,))) for i in range(3)]\n", + "# Data from all selected detectors binned by panel, pixel and timeslice\n", + "targets = [NodeName(NMXReducedDataGroup, IndexValues((\"panel\",), (i,))) for i in range(3)]\n", "dg = merge_panels(*wf.compute(targets).values())\n", "dg" ] @@ -114,9 +114,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Data from all selected detectors binned by panel, pixel and timeslice\n", - "binned_dg = wf.compute(NMXReducedData)\n", - "binned_dg" + "dg['counts']" ] }, { @@ -136,7 +134,7 @@ "metadata": {}, "outputs": [], "source": [ - "export_as_nexus(binned_dg, \"test.nxs\")" + "export_as_nexus(dg, \"test.nxs\")" ] }, { @@ -162,10 +160,10 @@ "source": [ "import scippneutron as scn\n", "\n", - "da = dg[\"weights\"]\n", + "da = dg[\"counts\"]\n", "da.coords[\"position\"] = dg[\"position\"]\n", "# Plot one out of 100 pixels to reduce size of docs output\n", - "view = scn.instrument_view(da[\"id\", ::100].hist(), pixel_size=0.0075)\n", + "view = scn.instrument_view(da[\"id\", ::100].sum('t'), pixel_size=0.0075)\n", "view" ] } diff --git a/packages/essnmx/src/ess/nmx/__init__.py b/packages/essnmx/src/ess/nmx/__init__.py index 2fbe4c31..57586b2e 100644 --- a/packages/essnmx/src/ess/nmx/__init__.py +++ b/packages/essnmx/src/ess/nmx/__init__.py @@ -12,16 +12,16 @@ del importlib from .data import small_mcstas_3_sample -from .reduction import NMXData, NMXReducedData -from .types import MaximumCounts +from .reduction import NMXReducedDataGroup +from .types import MaximumCounts, NMXRawEventCountsDataGroup default_parameters = {MaximumCounts: 10000} del MaximumCounts __all__ = [ - "NMXData", - "NMXReducedData", + "NMXRawEventCountsDataGroup", + "NMXReducedDataGroup", "default_parameters", "small_mcstas_3_sample", ] diff --git a/packages/essnmx/src/ess/nmx/mcstas/__init__.py b/packages/essnmx/src/ess/nmx/mcstas/__init__.py index 24397a5c..c40c8057 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/__init__.py +++ b/packages/essnmx/src/ess/nmx/mcstas/__init__.py @@ -5,11 +5,23 @@ def McStasWorkflow(): import sciline as sl - from ess.nmx.reduction import bin_time_of_arrival + from ess.nmx.reduction import ( + format_nmx_reduced_data, + proton_charge_from_event_counts, + raw_event_probability_to_counts, + reduce_raw_event_probability, + ) from .load import providers as loader_providers from .xml import read_mcstas_geometry_xml return sl.Pipeline( - (*loader_providers, read_mcstas_geometry_xml, bin_time_of_arrival) + ( + *loader_providers, + read_mcstas_geometry_xml, + proton_charge_from_event_counts, + reduce_raw_event_probability, + raw_event_probability_to_counts, + format_nmx_reduced_data, + ) ) diff --git a/packages/essnmx/src/ess/nmx/mcstas/load.py b/packages/essnmx/src/ess/nmx/mcstas/load.py index 28c451d3..c1d67f58 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/load.py +++ b/packages/essnmx/src/ess/nmx/mcstas/load.py @@ -1,23 +1,23 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) import re +from collections.abc import Generator import scipp as sc import scippnexus as snx -from ..reduction import NMXData from ..types import ( CrystalRotation, DetectorBankPrefix, DetectorIndex, DetectorName, - EventData, FilePath, MaximumCounts, MaximumProbability, McStasWeight2CountScaleFactor, - ProtonCharge, - RawEventData, + NMXRawEventCountsDataGroup, + PixelIds, + RawEventProbability, ) from .xml import McStasInstrument, read_mcstas_geometry_xml @@ -34,40 +34,114 @@ def load_event_data_bank_name( description = file['entry1/instrument/description'][()] for bank_name, det_names in bank_names_to_detector_names(description).items(): if detector_name in det_names: - return bank_name.partition('.')[0] + return DetectorBankPrefix(bank_name.partition('.')[0]) + raise KeyError( + f"{DetectorBankPrefix.__name__} cannot be found for " + f"{DetectorName.__name__} from the file {FilePath.__name__}" + ) + + +def _exclude_zero_events(data: sc.Variable) -> sc.Variable: + """Exclude events with zero counts from the data. + + McStas can add an extra event line containing 0,0,0,0,0,0 + This line should not be included so we skip it. + """ + if (data.values[0] == 0).all(): + data = data["event", 1:] + else: + data = data + return data + + +def _wrap_raw_event_data(data: sc.Variable) -> RawEventProbability: + data = data.rename_dims({'dim_0': 'event'}) + data = _exclude_zero_events(data) + event_da = sc.DataArray( + coords={ + 'id': sc.array( + dims=['event'], + values=data['dim_1', 4].values, + dtype='int64', + unit=None, + ), + 't': sc.array(dims=['event'], values=data['dim_1', 5].values, unit='s'), + }, + data=sc.array(dims=['event'], values=data['dim_1', 0].values, unit='counts'), + ) + return RawEventProbability(event_da) def load_raw_event_data( + file_path: FilePath, *, detector_name: DetectorName, bank_prefix: DetectorBankPrefix +) -> RawEventProbability: + """Retrieve events from the nexus file. + + Parameters + ---------- + file_path: + Path to the nexus file + detector_name: + Name of the detector to load + bank_prefix: + Prefix identifying the event data array containing the events of the detector + If None, the bank name is determined automatically from the detector name. + + """ + if bank_prefix is None: + bank_prefix = load_event_data_bank_name(detector_name, file_path) + bank_name = f'{bank_prefix}_dat_list_p_x_y_n_id_t' + with snx.File(file_path, 'r') as f: + root = f["entry1/data"] + (bank_name,) = (name for name in root.keys() if bank_name in name) + data = root[bank_name]["events"][()] + return _wrap_raw_event_data(data) + + +def raw_event_data_chunk_generator( file_path: FilePath, - bank_prefix: DetectorBankPrefix, + *, detector_name: DetectorName, - instrument: McStasInstrument, -) -> RawEventData: - """Retrieve events from the nexus file.""" - coords = instrument.to_coords(detector_name) + bank_prefix: DetectorBankPrefix | None = None, + chunk_size: int = 0, # Number of rows to read at a time +) -> Generator[RawEventProbability, None, None]: + """Chunk events from the nexus file. + + Parameters + ---------- + file_path: + Path to the nexus file + detector_name: + Name of the detector to load + pixel_ids: + Pixel ids to generate the data array with the events + chunk_size: + Number of rows to read at a time. + If 0, chunk slice is determined automatically by the ``iter_chunks``. + Note that it only works if the dataset is already chunked. + + """ + # Find the data bank name associated with the detector + bank_prefix = load_event_data_bank_name( + detector_name=detector_name, file_path=file_path + ) bank_name = f'{bank_prefix}_dat_list_p_x_y_n_id_t' with snx.File(file_path, 'r') as f: root = f["entry1/data"] (bank_name,) = (name for name in root.keys() if bank_name in name) - data = root[bank_name]["events"][()].rename_dims({'dim_0': 'event'}) - if (data.values[0] == 0).all(): - # McStas can add an extra event line containing 0,0,0,0,0,0 - # This line should not be included so we skip it. - data = data["event", 1:] - return sc.DataArray( - coords={ - 'id': sc.array( - dims=['event'], - values=data['dim_1', 4].values, - dtype='int64', - unit=None, - ), - 't': sc.array(dims=['event'], values=data['dim_1', 5].values, unit='s'), - }, - data=sc.array( - dims=['event'], values=data['dim_1', 0].values, unit='counts' - ), - ).group(coords.pop('pixel_id')) + + with snx.File(file_path, 'r') as f: + root = f["entry1/data"] + dset = root[bank_name]["events"] + if chunk_size == 0: + for data_slice in dset.dataset.iter_chunks(): + dim_0_slice, _ = data_slice # dim_0_slice, dim_1_slice + yield _wrap_raw_event_data(dset["dim_0", dim_0_slice]) + else: + num_events = dset.shape[0] + for start in range(0, num_events, chunk_size): + data = dset["dim_0", start : start + chunk_size] + yield _wrap_raw_event_data(data) def load_crystal_rotation( @@ -95,7 +169,7 @@ def load_crystal_rotation( ) -def maximum_probability(da: RawEventData) -> MaximumProbability: +def maximum_probability(da: RawEventProbability) -> MaximumProbability: """Find the maximum probability in the data.""" return MaximumProbability(da.data.max()) @@ -112,53 +186,14 @@ def mcstas_weight_to_probability_scalefactor( max_counts: The maximum number of counts after scaling the event counts. - max_probability: - The maximum probability to scale the weights. - - """ - return McStasWeight2CountScaleFactor( - sc.scalar(max_counts, unit="counts") / max_probability - ) - - -def event_weights_from_probability( - da: RawEventData, scale_factor: McStasWeight2CountScaleFactor -) -> EventData: - """Create event weights by scaling probability data. - - event_weights = max_counts * (probabilities / max_probability) - - Parameters - ---------- - da: - The probabilities of the events - scale_factor: The scale factor to convert McStas weights to counts """ - return EventData(da * scale_factor) - - -def proton_charge_from_event_data(da: EventData) -> ProtonCharge: - """Make up the proton charge from the event data array. - - Proton charge is proportional to the number of neutrons, - which is proportional to the number of events. - The scale factor is manually chosen based on previous results - to be convenient for data manipulation in the next steps. - It is derived this way since - the protons are not part of McStas simulation, - and the number of neutrons is not included in the result. - - Parameters - ---------- - event_da: - The event data - """ - # Arbitrary number to scale the proton charge - return ProtonCharge(sc.scalar(1 / 10_000, unit=None) * da.bins.size().sum().data) + return McStasWeight2CountScaleFactor( + sc.scalar(max_counts, unit="counts") / max_probability + ) def bank_names_to_detector_names(description: str) -> dict[str, list[str]]: @@ -189,24 +224,30 @@ def bank_names_to_detector_names(description: str) -> dict[str, list[str]]: def load_mcstas( *, - da: EventData, - proton_charge: ProtonCharge, + da: RawEventProbability, crystal_rotation: CrystalRotation, detector_name: DetectorName, instrument: McStasInstrument, -) -> NMXData: +) -> NMXRawEventCountsDataGroup: coords = instrument.to_coords(detector_name) - coords.pop('pixel_id') - return NMXData( + return NMXRawEventCountsDataGroup( sc.DataGroup( weights=da, - proton_charge=proton_charge, crystal_rotation=crystal_rotation, + name=sc.scalar(detector_name), + pixel_id=instrument.pixel_ids(detector_name), **coords, ) ) +def retrieve_pixel_ids( + instrument: McStasInstrument, detector_name: DetectorName +) -> PixelIds: + """Retrieve the pixel IDs for a given detector.""" + return PixelIds(instrument.pixel_ids(detector_name)) + + providers = ( read_mcstas_geometry_xml, detector_name_from_index, @@ -214,8 +255,7 @@ def load_mcstas( load_raw_event_data, maximum_probability, mcstas_weight_to_probability_scalefactor, - event_weights_from_probability, - proton_charge_from_event_data, + retrieve_pixel_ids, load_crystal_rotation, load_mcstas, ) diff --git a/packages/essnmx/src/ess/nmx/mcstas/xml.py b/packages/essnmx/src/ess/nmx/mcstas/xml.py index c872e681..3014220d 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas/xml.py @@ -379,6 +379,10 @@ def from_xml(cls, tree: _XML) -> 'McStasInstrument': ), ) + def pixel_ids(self, *det_names: str) -> sc.Variable: + detectors = tuple(det for det in self.detectors if det.name in det_names) + return _construct_pixel_ids(detectors) + def to_coords(self, *det_names: str) -> dict[str, sc.Variable]: """Extract coordinates from the McStas instrument description. @@ -393,7 +397,6 @@ def to_coords(self, *det_names: str) -> dict[str, sc.Variable]: fast_axes = [det.fast_axis for det in detectors] origins = [self.sample.position_from_sample(det.position) for det in detectors] return { - 'pixel_id': _construct_pixel_ids(detectors), 'fast_axis': sc.concat(fast_axes, 'panel'), 'slow_axis': sc.concat(slow_axes, 'panel'), 'origin_position': sc.concat(origins, 'panel'), diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 1bb3df4f..5d54323d 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -1,29 +1,74 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) -from typing import NewType - import scipp as sc from .mcstas.xml import McStasInstrument -from .types import DetectorName, TimeBinSteps +from .types import ( + CrystalRotation, + DetectorName, + McStasWeight2CountScaleFactor, + NMXReducedCounts, + NMXReducedDataGroup, + NMXReducedProbability, + PixelIds, + ProtonCharge, + RawEventProbability, + TimeBinSteps, +) + + +def proton_charge_from_event_counts(da: NMXReducedCounts) -> ProtonCharge: + """Make up the proton charge from the event counts. + + Proton charge is proportional to the number of neutrons, + which is proportional to the number of events. + The scale factor is manually chosen based on previous results + to be convenient for data manipulation in the next steps. + It is derived this way since + the protons are not part of McStas simulation, + and the number of neutrons is not included in the result. + + Parameters + ---------- + event_da: + The event data + + """ + # Arbitrary number to scale the proton charge + return ProtonCharge(sc.scalar(1 / 10_000, unit='dimensionless') * da.data.sum()) + -NMXData = NewType("NMXData", sc.DataGroup) -NMXReducedData = NewType("NMXReducedData", sc.DataGroup) +def reduce_raw_event_probability( + da: RawEventProbability, pixel_ids: PixelIds, time_bin_step: TimeBinSteps +) -> NMXReducedProbability: + return NMXReducedProbability(da.group(pixel_ids).hist(t=time_bin_step)) -def bin_time_of_arrival( - nmx_data: NMXData, +def raw_event_probability_to_counts( + da: NMXReducedProbability, + scale_factor: McStasWeight2CountScaleFactor, +) -> NMXReducedCounts: + return NMXReducedCounts(da * scale_factor) + + +def format_nmx_reduced_data( + da: NMXReducedCounts, detector_name: DetectorName, instrument: McStasInstrument, - time_bin_step: TimeBinSteps, -) -> NMXReducedData: + proton_charge: ProtonCharge, + crystal_rotation: CrystalRotation, +) -> NMXReducedDataGroup: """Bin time of arrival data into ``time_bin_step`` bins.""" - - counts = nmx_data.pop('weights').hist(t=time_bin_step) new_coords = instrument.to_coords(detector_name) - new_coords.pop('pixel_id') - return NMXReducedData(sc.DataGroup(counts=counts, **{**nmx_data, **new_coords})) + return NMXReducedDataGroup( + sc.DataGroup( + counts=da, + proton_charge=proton_charge, + crystal_rotation=crystal_rotation, + **new_coords, + ) + ) def _concat_or_same( @@ -39,12 +84,12 @@ def _concat_or_same( return sc.concat(obj, dim) -def merge_panels(*panel: NMXReducedData) -> NMXReducedData: +def merge_panels(*panel: NMXReducedDataGroup) -> NMXReducedDataGroup: """Merge a list of panels by concatenating along the 'panel' dimension.""" keys = panel[0].keys() if not all(p.keys() == keys for p in panel): raise ValueError("All panels must have the same keys.") - return NMXReducedData( + return NMXReducedDataGroup( sc.DataGroup( {key: _concat_or_same([p[key] for p in panel], 'panel') for key in keys} ) diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 8e7fcd8d..a79921f6 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -24,12 +24,13 @@ McStasWeight2CountScaleFactor = NewType("McStasWeight2CountScaleFactor", sc.Variable) """Scale factor to convert McStas weights to counts""" -RawEventData = NewType("RawEventData", sc.DataArray) -"""DataArray containing the event counts read from the McStas file, + +RawEventProbability = NewType("RawEventProbability", sc.DataArray) +"""DataArray containing the event probabilities read from the McStas file, has coordinates 'id' and 't' """ -EventData = NewType("EventData", sc.DataArray) -"""The scaled RawEventData""" +NMXRawEventCountsDataGroup = NewType("NMXRawEventCountsDataGroup", sc.DataGroup) +"""DataGroup containing the RawEventData and other metadata""" ProtonCharge = NewType("ProtonCharge", sc.Variable) """The proton charge signal""" @@ -42,3 +43,15 @@ TimeBinSteps = NewType("TimeBinSteps", int) """Number of bins in the binning of the time coordinate""" + +PixelIds = NewType("PixelIds", sc.Variable) +"""The pixel ids of the detector""" + +NMXReducedProbability = NewType("NMXReducedProbability", sc.DataArray) +"""Histogram of time-of-arrival and pixel-id.""" + +NMXReducedCounts = NewType("NMXReducedCounts", sc.DataArray) +"""Histogram of time-of-arrival and pixel-id.""" + +NMXReducedDataGroup = NewType("NMXReducedDataGroup", sc.DataGroup) +"""Histogram of time-of-arrival and pixel-id, with additional metadata.""" diff --git a/packages/essnmx/tests/exporter_test.py b/packages/essnmx/tests/exporter_test.py index b2efb78b..5b519c38 100644 --- a/packages/essnmx/tests/exporter_test.py +++ b/packages/essnmx/tests/exporter_test.py @@ -7,11 +7,11 @@ import scipp as sc from ess.nmx.nexus import export_as_nexus -from ess.nmx.reduction import NMXReducedData +from ess.nmx.reduction import NMXReducedDataGroup @pytest.fixture -def reduced_data() -> NMXReducedData: +def reduced_data() -> NMXReducedDataGroup: rng = np.random.default_rng(42) id_list = sc.array(dims=['event'], values=rng.integers(0, 12, size=100)) t_list = sc.array(dims=['event'], values=rng.random(size=100, dtype=float)) @@ -24,7 +24,7 @@ def reduced_data() -> NMXReducedData: .hist(t=10) ) - return NMXReducedData( + return NMXReducedDataGroup( sc.DataGroup( dict( # noqa: C408 counts=counts, @@ -54,7 +54,9 @@ def reduced_data() -> NMXReducedData: ) -def test_mcstas_reduction_export_to_bytestream(reduced_data: NMXReducedData) -> None: +def test_mcstas_reduction_export_to_bytestream( + reduced_data: NMXReducedDataGroup, +) -> None: """Test export method.""" import h5py import numpy as np diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 90c75533..ccd306eb 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -14,8 +14,12 @@ from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample from ess.nmx.mcstas.load import bank_names_to_detector_names, load_crystal_rotation from ess.nmx.mcstas.load import providers as loader_providers -from ess.nmx.reduction import NMXData -from ess.nmx.types import DetectorBankPrefix, DetectorIndex, FilePath, MaximumCounts +from ess.nmx.types import ( + DetectorBankPrefix, + DetectorIndex, + FilePath, + NMXRawEventCountsDataGroup, +) sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) from mcstas_description_examples import ( @@ -26,15 +30,11 @@ ) -def check_scalar_properties_mcstas_2(dg: NMXData): +def check_scalar_properties_mcstas_2(dg: NMXRawEventCountsDataGroup): """Test helper for NMXData loaded from McStas 2. Expected numbers are hard-coded based on the sample file. """ - assert_identical( - dg['proton_charge'], - sc.scalar(1e-4 * dg['weights'].bins.size().sum().data.values, unit=None), - ) assert_identical(dg['crystal_rotation'], sc.vector([20, 0, 90], unit='deg')) assert_identical(dg['sample_position'], sc.vector(value=[0, 0, 0], unit='m')) assert_identical( @@ -43,16 +43,10 @@ def check_scalar_properties_mcstas_2(dg: NMXData): assert dg['sample_name'] == sc.scalar("sampleMantid") -def check_nmxdata_properties(dg: NMXData, fast_axis, slow_axis) -> None: +def check_nmxdata_properties( + dg: NMXRawEventCountsDataGroup, fast_axis, slow_axis +) -> None: assert isinstance(dg, sc.DataGroup) - assert dg.shape == ((1280, 1280)[0] * (1280, 1280)[1], 1) - # Check maximum value of weights. - assert_allclose( - dg['weights'].max().data, - sc.scalar(default_parameters[MaximumCounts], unit='counts', dtype=float), - atol=sc.scalar(1e-10, unit='counts'), - rtol=sc.scalar(1e-8), - ) assert_allclose( sc.squeeze(dg['fast_axis'], 'panel'), fast_axis, atol=sc.scalar(0.005) ) @@ -86,21 +80,17 @@ def test_file_reader_mcstas2( **default_parameters, }, ) - dg = pl.compute(NMXData) + dg = pl.compute(NMXRawEventCountsDataGroup) check_scalar_properties_mcstas_2(dg) check_nmxdata_properties(dg, fast_axis, slow_axis) -def check_scalar_properties_mcstas_3(dg: NMXData): +def check_scalar_properties_mcstas_3(dg: NMXRawEventCountsDataGroup): """Test helper for NMXData loaded from McStas 3. Expected numbers are hard-coded based on the sample file. """ - assert_identical( - dg['proton_charge'], - sc.scalar(1e-4 * dg['weights'].bins.size().sum().data.values, unit=None), - ) assert_identical(dg['crystal_rotation'], sc.vector([0, 0, 0], unit='deg')) assert_identical(dg['sample_position'], sc.vector(value=[0, 0, 0], unit='m')) assert_identical( @@ -130,7 +120,7 @@ def test_file_reader_mcstas3(detector_index, fast_axis, slow_axis) -> None: **default_parameters, }, ) - dg, bank = pl.compute((NMXData, DetectorBankPrefix)).values() + dg, bank = pl.compute((NMXRawEventCountsDataGroup, DetectorBankPrefix)).values() entry_path = f"entry1/data/{bank}_dat_list_p_x_y_n_id_t" with snx.File(file_path) as file: @@ -138,7 +128,7 @@ def test_file_reader_mcstas3(detector_index, fast_axis, slow_axis) -> None: data_length = raw_data.sizes['dim_0'] check_scalar_properties_mcstas_3(dg) - assert dg['weights'].bins.size().sum().value == data_length + assert dg['weights'].sizes['event'] == data_length check_nmxdata_properties(dg, sc.vector(fast_axis), sc.vector(slow_axis)) @@ -183,7 +173,7 @@ def test_file_reader_mcstas_additional_fields(tmp_mcstas_file: pathlib.Path) -> **default_parameters, }, ) - dg = pl.compute(NMXData) + dg = pl.compute(NMXRawEventCountsDataGroup) assert isinstance(dg, sc.DataGroup) diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index cf8e14d0..f0c5c8a4 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -8,8 +8,21 @@ from ess.nmx import default_parameters from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample from ess.nmx.mcstas.load import providers as load_providers -from ess.nmx.reduction import NMXData, NMXReducedData, bin_time_of_arrival, merge_panels -from ess.nmx.types import DetectorIndex, FilePath, TimeBinSteps +from ess.nmx.reduction import ( + NMXReducedDataGroup, + format_nmx_reduced_data, + merge_panels, + proton_charge_from_event_counts, + raw_event_probability_to_counts, + reduce_raw_event_probability, +) +from ess.nmx.types import ( + DetectorIndex, + FilePath, + MaximumCounts, + NMXRawEventCountsDataGroup, + TimeBinSteps, +) @pytest.fixture(params=[small_mcstas_2_sample, small_mcstas_3_sample]) @@ -26,7 +39,13 @@ def mcstas_file_path( @pytest.fixture def mcstas_workflow(mcstas_file_path: str) -> sl.Pipeline: return sl.Pipeline( - [*load_providers, bin_time_of_arrival], + [ + *load_providers, + reduce_raw_event_probability, + proton_charge_from_event_counts, + raw_event_probability_to_counts, + format_nmx_reduced_data, + ], params={ FilePath: mcstas_file_path, TimeBinSteps: 50, @@ -38,8 +57,8 @@ def mcstas_workflow(mcstas_file_path: str) -> sl.Pipeline: @pytest.fixture def multi_bank_mcstas_workflow(mcstas_workflow: sl.Pipeline) -> sl.Pipeline: pl = mcstas_workflow.copy() - pl[NMXReducedData] = ( - pl[NMXReducedData] + pl[NMXReducedDataGroup] = ( + pl[NMXReducedDataGroup] .map(pd.DataFrame({DetectorIndex: range(3)}).rename_axis('panel')) .reduce(index='panel', func=merge_panels) ) @@ -53,13 +72,30 @@ def test_pipeline_builder(mcstas_workflow: sl.Pipeline, mcstas_file_path: str) - def test_pipeline_mcstas_loader(mcstas_workflow: sl.Pipeline) -> None: """Test if the loader graph is complete.""" mcstas_workflow[DetectorIndex] = 0 - nmx_data = mcstas_workflow.compute(NMXData) + nmx_data = mcstas_workflow.compute(NMXRawEventCountsDataGroup) assert isinstance(nmx_data, sc.DataGroup) assert nmx_data.sizes['id'] == 1280 * 1280 def test_pipeline_mcstas_reduction(multi_bank_mcstas_workflow: sl.Pipeline) -> None: """Test if the loader graph is complete.""" - nmx_reduced_data = multi_bank_mcstas_workflow.compute(NMXReducedData) + from scipp.testing import assert_allclose, assert_identical + + nmx_reduced_data = multi_bank_mcstas_workflow.compute(NMXReducedDataGroup) + assert nmx_reduced_data.shape == (3, (1280, 1280)[0] * (1280, 1280)[1], 50) + # Panel, Pixels, Time bins assert isinstance(nmx_reduced_data, sc.DataGroup) + + # Check maximum value of weights. + assert_allclose( + nmx_reduced_data['counts'].max().data, + sc.scalar(default_parameters[MaximumCounts], unit='counts', dtype=float), + atol=sc.scalar(1e-10, unit='counts'), + rtol=sc.scalar(1e-8), + ) + assert_identical( + nmx_reduced_data['proton_charge'], + sc.scalar(1e-4, unit='dimensionless') + * nmx_reduced_data['counts'].data.sum('id').sum('t'), + ) assert nmx_reduced_data.sizes['t'] == 50 From 094a29a932f3915f4acf260f15c518bd4d8235b5 Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Wed, 26 Feb 2025 14:55:41 +0100 Subject: [PATCH 239/403] Add essreduce as a dependency to use stream processor * Rename MaximumProbability to MaximumCounts and add MaximumProbability. * Expose scalefactor to be exported. * Update type name. * Update workflow to have more granular steps. * Move scaling to later step. * Remove unecessary argument. * Fix iter chunk slices. * Add essreduce as a dependency. --- packages/essnmx/pyproject.toml | 1 + packages/essnmx/requirements/base.in | 1 + packages/essnmx/requirements/base.txt | 71 +++++++++++++++++++----- packages/essnmx/requirements/ci.txt | 2 +- packages/essnmx/requirements/dev.txt | 8 +-- packages/essnmx/requirements/docs.txt | 19 ++----- packages/essnmx/requirements/mypy.txt | 2 - packages/essnmx/requirements/nightly.in | 1 + packages/essnmx/requirements/nightly.txt | 68 ++++++++++++++++++----- packages/essnmx/requirements/static.txt | 2 +- 10 files changed, 123 insertions(+), 52 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 91755912..9559362b 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -32,6 +32,7 @@ requires-python = ">=3.10" # Make sure to list one dependency per line. dependencies = [ "dask", + "essreduce>=24.02.3", "graphviz", "plopp", "sciline>=24.06.0", diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index f746e638..5b8a9a1a 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -3,6 +3,7 @@ # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask +essreduce>=24.02.3 graphviz plopp sciline>=24.06.0 diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index d896da5e..4c060f5d 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,10 +1,12 @@ -# SHA1:bea70253436877e109e43f0839bf995cf4bbc84f +# SHA1:ddb5c021f8aab4013482100474cd29d0000f612d # # This file is autogenerated by pip-compile-multi # To update, run: # # pip-compile-multi # +annotated-types==0.7.0 + # via pydantic certifi==2025.1.31 # via requests charset-normalizer==3.4.1 @@ -19,10 +21,16 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -dask==2025.1.0 +dask==2025.2.0 # via -r base.in defusedxml==0.7.1 # via -r base.in +dnspython==2.7.0 + # via email-validator +email-validator==2.2.0 + # via scippneutron +essreduce==25.2.4 + # via -r base.in fonttools==4.56.0 # via matplotlib fsspec==2025.2.0 @@ -31,31 +39,46 @@ gemmi==0.7.0 # via -r base.in graphviz==0.20.3 # via -r base.in -h5py==3.12.1 - # via scippnexus +h5py==3.13.0 + # via + # scippneutron + # scippnexus idna==3.10 - # via requests + # via + # email-validator + # requests importlib-metadata==8.6.1 # via dask kiwisolver==1.4.8 # via matplotlib +lazy-loader==0.4 + # via + # plopp + # scippneutron locket==1.0.0 # via partd matplotlib==3.10.0 - # via plopp + # via + # mpltoolbox + # plopp +mpltoolbox==24.5.1 + # via scippneutron networkx==3.4.2 # via cyclebane -numpy==2.2.2 +numpy==2.2.3 # via # contourpy # h5py # matplotlib + # mpltoolbox # pandas # scipp + # scippneutron # scipy packaging==24.2 # via # dask + # lazy-loader # matplotlib # pooch pandas==2.2.3 @@ -66,16 +89,23 @@ pillow==11.1.0 # via matplotlib platformdirs==4.3.6 # via pooch -plopp==24.10.0 - # via -r base.in +plopp==25.2.0 + # via + # -r base.in + # scippneutron pooch==1.8.2 # via -r base.in +pydantic==2.10.6 + # via scippneutron +pydantic-core==2.27.2 + # via pydantic pyparsing==3.2.1 # via matplotlib python-dateutil==2.9.0.post0 # via # matplotlib # pandas + # scippneutron # scippnexus pytz==2025.1 # via pandas @@ -84,21 +114,36 @@ pyyaml==6.0.2 requests==2.32.3 # via pooch sciline==24.10.0 - # via -r base.in + # via + # -r base.in + # essreduce scipp==25.2.0 # via # -r base.in + # essreduce + # scippneutron # scippnexus +scippneutron==25.2.1 + # via essreduce scippnexus==24.11.1 - # via -r base.in -scipy==1.15.1 - # via scippnexus + # via + # -r base.in + # essreduce + # scippneutron +scipy==1.15.2 + # via + # scippneutron + # scippnexus six==1.17.0 # via python-dateutil toolz==1.0.0 # via # dask # partd +typing-extensions==4.12.2 + # via + # pydantic + # pydantic-core tzdata==2025.1 # via pandas urllib3==2.3.0 diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index 078cca56..1faaceaf 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -5,7 +5,7 @@ # # pip-compile-multi # -cachetools==5.5.1 +cachetools==5.5.2 # via tox certifi==2025.1.31 # via requests diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index 814d4763..b9014c97 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -12,8 +12,6 @@ -r static.txt -r test.txt -r wheels.txt -annotated-types==0.7.0 - # via pydantic anyio==4.8.0 # via # httpx @@ -28,7 +26,7 @@ async-lru==2.0.4 # via jupyterlab cffi==1.17.1 # via argon2-cffi-bindings -copier==9.4.1 +copier==9.5.0 # via -r dev.in dunamai==1.23.0 # via copier @@ -87,10 +85,6 @@ prometheus-client==0.21.1 # via jupyter-server pycparser==2.22 # via cffi -pydantic==2.10.6 - # via copier -pydantic-core==2.27.2 - # via pydantic python-json-logger==3.2.1 # via jupyter-events questionary==2.1.0 diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index aa04ca18..11002120 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -34,7 +34,7 @@ comm==0.2.2 # ipywidgets debugpy==1.8.12 # via ipykernel -decorator==5.1.1 +decorator==5.2.1 # via ipython docutils==0.21.2 # via @@ -106,10 +106,8 @@ mdit-py-plugins==0.4.2 # via myst-parser mdurl==0.1.2 # via markdown-it-py -mistune==3.1.1 +mistune==3.1.2 # via nbconvert -mpltoolbox==24.5.1 - # via scippneutron myst-parser==4.0.1 # via -r docs.in nbclient==0.10.2 @@ -133,7 +131,7 @@ pexpect==4.9.0 # via ipython prompt-toolkit==3.0.50 # via ipython -psutil==6.1.1 +psutil==7.0.0 # via ipykernel ptyprocess==0.7.0 # via pexpect @@ -158,12 +156,10 @@ referencing==0.36.2 # via # jsonschema # jsonschema-specifications -rpds-py==0.22.3 +rpds-py==0.23.1 # via # jsonschema # referencing -scippneutron==25.1.0 - # via -r docs.in snowballstemmer==2.2.0 # via sphinx soupsieve==2.6 @@ -222,13 +218,6 @@ traitlets==5.14.3 # traittypes traittypes==0.2.1 # via ipydatawidgets -typing-extensions==4.12.2 - # via - # beautifulsoup4 - # ipython - # mistune - # pydata-sphinx-theme - # referencing wcwidth==0.2.13 # via prompt-toolkit webencodings==0.5.1 diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index 7f020f7a..61d88db1 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -10,5 +10,3 @@ mypy==1.15.0 # via -r mypy.in mypy-extensions==1.0.0 # via mypy -typing-extensions==4.12.2 - # via mypy diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index f34be063..003773ba 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -2,6 +2,7 @@ # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask +essreduce>=24.02.3 graphviz pooch pandas diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index d6420af3..376e00e8 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:38e72cba5ebcd6086e53af32007e3d9b1e5f12bf +# SHA1:a1aeab214632eed331a1c2075a27b248a26d4755 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -8,6 +8,8 @@ --index-url https://pypi.anaconda.org/scipp-nightly-wheels/simple/ --extra-index-url https://pypi.org/simple +annotated-types==0.7.0 + # via pydantic certifi==2025.1.31 # via requests charset-normalizer==3.4.1 @@ -22,10 +24,16 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -dask==2025.1.0 +dask==2025.2.0 # via -r nightly.in defusedxml==0.8.0rc2 # via -r nightly.in +dnspython==2.7.0 + # via email-validator +email-validator==2.2.0 + # via scippneutron +essreduce==25.2.4 + # via -r nightly.in exceptiongroup==1.2.2 # via pytest fonttools==4.56.0 @@ -36,10 +44,14 @@ gemmi==0.7.0 # via -r nightly.in graphviz==0.20.3 # via -r nightly.in -h5py==3.12.1 - # via scippnexus +h5py==3.13.0 + # via + # scippneutron + # scippnexus idna==3.10 - # via requests + # via + # email-validator + # requests importlib-metadata==8.6.1 # via dask iniconfig==2.0.0 @@ -47,20 +59,28 @@ iniconfig==2.0.0 kiwisolver==1.4.8 # via matplotlib lazy-loader==0.4 - # via plopp + # via + # plopp + # scippneutron locket==1.0.0 # via partd matplotlib==3.10.0 - # via plopp + # via + # mpltoolbox + # plopp +mpltoolbox==24.5.1 + # via scippneutron networkx==3.4.2 # via cyclebane -numpy==2.2.2 +numpy==2.2.3 # via # contourpy # h5py # matplotlib + # mpltoolbox # pandas # scipp + # scippneutron # scipy packaging==24.2 # via @@ -78,11 +98,17 @@ pillow==11.1.0 platformdirs==4.3.6 # via pooch plopp @ git+https://github.com/scipp/plopp@main - # via -r nightly.in + # via + # -r nightly.in + # scippneutron pluggy==1.5.0 # via pytest pooch==1.8.2 # via -r nightly.in +pydantic==2.11.0a2 + # via scippneutron +pydantic-core==2.29.0 + # via pydantic pyparsing==3.2.1 # via matplotlib pytest==8.3.4 @@ -91,6 +117,7 @@ python-dateutil==2.9.0.post0 # via # matplotlib # pandas + # scippneutron # scippnexus pytz==2025.1 # via pandas @@ -99,15 +126,26 @@ pyyaml==6.0.2 requests==2.32.3 # via pooch sciline @ git+https://github.com/scipp/sciline@main - # via -r nightly.in + # via + # -r nightly.in + # essreduce scipp==100.0.0.dev0 # via # -r nightly.in + # essreduce + # scippneutron # scippnexus +scippneutron==25.2.1 + # via essreduce scippnexus @ git+https://github.com/scipp/scippnexus@main - # via -r nightly.in -scipy==1.15.1 - # via scippnexus + # via + # -r nightly.in + # essreduce + # scippneutron +scipy==1.15.2 + # via + # scippneutron + # scippnexus six==1.17.0 # via python-dateutil tomli==2.2.1 @@ -116,6 +154,10 @@ toolz==1.0.0 # via # dask # partd +typing-extensions==4.12.2 + # via + # pydantic + # pydantic-core tzdata==2025.1 # via pandas urllib3==2.3.0 diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index a0569bda..098c4f5a 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -11,7 +11,7 @@ distlib==0.3.9 # via virtualenv filelock==3.17.0 # via virtualenv -identify==2.6.7 +identify==2.6.8 # via pre-commit nodeenv==1.9.1 # via pre-commit From 95e43bd5c06c62ef759b43aabea8f3949530210f Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Wed, 26 Feb 2025 15:04:56 +0100 Subject: [PATCH 240/403] Chunk data processing example. * Rename MaximumProbability to MaximumCounts and add MaximumProbability. * Expose scalefactor to be exported. * Update type name. * Update workflow to have more granular steps. * Move scaling to later step. * Remove unecessary argument. * Fix iter chunk slices. * Add essreduce as a dependency. * Add example of chunk workflow. * Use python min. * Use min instead of concatenating. * Warn if chunk size is too small. --- .../essnmx/docs/examples/workflow_chunk.ipynb | 257 ++++++++++++++++++ packages/essnmx/src/ess/nmx/mcstas/load.py | 25 +- packages/essnmx/src/ess/nmx/streaming.py | 89 ++++++ 3 files changed, 370 insertions(+), 1 deletion(-) create mode 100644 packages/essnmx/docs/examples/workflow_chunk.ipynb create mode 100644 packages/essnmx/src/ess/nmx/streaming.py diff --git a/packages/essnmx/docs/examples/workflow_chunk.ipynb b/packages/essnmx/docs/examples/workflow_chunk.ipynb new file mode 100644 index 00000000..1618b070 --- /dev/null +++ b/packages/essnmx/docs/examples/workflow_chunk.ipynb @@ -0,0 +1,257 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Chunk Workflow\n", + "In this example, we will process McStas events chunk by chunk, panel by panel." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Build Base Workflow" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ess.nmx.mcstas import McStasWorkflow\n", + "from ess.nmx.data import small_mcstas_3_sample\n", + "from ess.nmx.types import *\n", + "\n", + "wf = McStasWorkflow()\n", + "# Replace with the path to your own file\n", + "wf[FilePath] = small_mcstas_3_sample()\n", + "wf[MaximumCounts] = 10_000\n", + "wf[TimeBinSteps] = 50\n", + "wf.visualize(NMXReducedDataGroup, graph_attr={\"rankdir\": \"TD\"}, compact=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute Maximum Probabiliity\n", + "\n", + "`McStasWeight2CountScaleFactor` should not be different from chunk to chunk.\n", + "\n", + "Therefore we need to compute `McStasWeight2CoutScaleFactor` before we compute `NMXReducedDataGroup`.\n", + "\n", + "It can be done by `ess.reduce.streaming.StreamProcessor`.\n", + "\n", + "In this example, `MaximumProbability` will be renewed every time a chunk is added to the streaming processor.\n", + "\n", + "`MaxAccumulator` remembers the previous maximum value and compute new maximum value with the new chunk.\n", + "\n", + "``raw_event_data_chunk_generator`` yields a chunk of raw event probability from mcstas h5 file.\n", + "\n", + "This example below process the data chunk by chunk with size: ``CHUNK_SIZE``." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from functools import partial\n", + "from ess.reduce.streaming import StreamProcessor\n", + "from ess.nmx.streaming import MaxAccumulator\n", + "\n", + "# Stream processor building helper\n", + "scalefactor_stream_processor = partial(\n", + " StreamProcessor,\n", + " dynamic_keys=(RawEventProbability,),\n", + " target_keys=(McStasWeight2CountScaleFactor,),\n", + " accumulators={MaximumProbability: MaxAccumulator},\n", + ")\n", + "scalefactor_wf = wf.copy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "scalefactor_wf.visualize(McStasWeight2CountScaleFactor, graph_attr={\"rankdir\": \"TD\"}, compact=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ess.nmx.types import DetectorName\n", + "from ess.nmx.mcstas.load import raw_event_data_chunk_generator\n", + "from ess.nmx.streaming import calculate_number_of_chunks\n", + "from ipywidgets import IntProgress\n", + "\n", + "CHUNK_SIZE = 10 # Number of event rows to process at once\n", + "# Increase this number to speed up the processing\n", + "NUM_DETECTORS = 3\n", + "\n", + "# Loop over the detectors\n", + "file_path = scalefactor_wf.compute(FilePath)\n", + "scale_factors = {}\n", + "for detector_i in range(0, NUM_DETECTORS):\n", + " temp_wf = scalefactor_wf.copy()\n", + " temp_wf[DetectorIndex] = detector_i\n", + " detector_name = temp_wf.compute(DetectorName)\n", + " max_chunk_id = calculate_number_of_chunks(\n", + " temp_wf.compute(FilePath), detector_name=detector_name, chunk_size=CHUNK_SIZE\n", + " )\n", + " cur_detector_progress_bar = IntProgress(\n", + " min=0, max=max_chunk_id, description=f\"Detector {detector_i}\"\n", + " )\n", + " display(cur_detector_progress_bar)\n", + "\n", + " # Build the stream processor\n", + " processor = scalefactor_stream_processor(temp_wf)\n", + " for da in raw_event_data_chunk_generator(\n", + " file_path=file_path, detector_name=detector_name, chunk_size=CHUNK_SIZE\n", + " ):\n", + " if any(da.sizes.values()) == 0:\n", + " continue\n", + " else:\n", + " results = processor.add_chunk({RawEventProbability: da})\n", + " cur_detector_progress_bar.value += 1\n", + " scale_factors[detector_i] = results[McStasWeight2CountScaleFactor]\n", + "\n", + "# We take the minimum scale factor for the entire dataset\n", + "scale_factor = min(scale_factors.values())\n", + "scale_factor\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute Final Output\n", + "\n", + "Now with the `scale_factor: McStasWeight2CountScaleFactor`, we can compute the final output chunk by chunk.\n", + "\n", + "We will also compute static parameters in advance so that stream processor does not compute them every time another chunk is added." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ess.nmx.mcstas.xml import McStasInstrument\n", + "\n", + "final_wf = wf.copy()\n", + "# Add the scale factor to the workflow\n", + "final_wf[McStasWeight2CountScaleFactor] = scale_factor\n", + "\n", + "# Compute the static information in advance\n", + "# static_info = wf.compute([CrystalRotation, McStasInstrument])\n", + "static_info = wf.compute([McStasInstrument])\n", + "# final_wf[CrystalRotation] = static_info[CrystalRotation]\n", + "final_wf[CrystalRotation] = sc.vector([0, 0, 0.,], unit='deg')\n", + "final_wf[McStasInstrument] = static_info[McStasInstrument]\n", + "final_wf.visualize(NMXReducedDataGroup, compact=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from functools import partial\n", + "from ess.reduce.streaming import StreamProcessor, EternalAccumulator\n", + "\n", + "# Stream processor building helper\n", + "final_stream_processor = partial(\n", + " StreamProcessor,\n", + " dynamic_keys=(RawEventProbability,),\n", + " target_keys=(NMXReducedDataGroup,),\n", + " accumulators={NMXReducedCounts: EternalAccumulator},\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ess.nmx.types import DetectorName\n", + "from ess.nmx.mcstas.load import raw_event_data_chunk_generator\n", + "from ess.nmx.streaming import calculate_number_of_chunks\n", + "from ipywidgets import IntProgress\n", + "\n", + "CHUNK_SIZE = 10 # Number of event rows to process at once\n", + "# Increase this number to speed up the processing\n", + "NUM_DETECTORS = 3\n", + "final_wf[TimeBinSteps] = sc.linspace(\n", + " 't', 0.1, 0.15, 51, unit='s'\n", + ") # Time bin edges can be calculated from the data\n", + "# But streaming processor only sees the first chunk\n", + "# So we need to set it manually before processing the first chunk\n", + "# It is a bit cumbersome since we have to know the range of the time bins in advance\n", + "\n", + "# Loop over the detectors\n", + "file_path = final_wf.compute(FilePath)\n", + "for detector_i in range(0, NUM_DETECTORS):\n", + " temp_wf = final_wf.copy()\n", + " temp_wf[DetectorIndex] = detector_i\n", + " # First compute static information\n", + " detector_name = temp_wf.compute(DetectorName)\n", + " temp_wf[PixelIds] = temp_wf.compute(PixelIds)\n", + " max_chunk_id = calculate_number_of_chunks(\n", + " file_path, detector_name=detector_name, chunk_size=CHUNK_SIZE\n", + " )\n", + " cur_detector_progress_bar = IntProgress(\n", + " min=0, max=max_chunk_id, description=f\"Detector {detector_i}\"\n", + " )\n", + " display(cur_detector_progress_bar)\n", + "\n", + " # Build the stream processor\n", + " processor = final_stream_processor(temp_wf)\n", + " for da in raw_event_data_chunk_generator(\n", + " file_path=file_path, detector_name=detector_name, chunk_size=CHUNK_SIZE\n", + " ):\n", + " if any(da.sizes.values()) == 0:\n", + " continue\n", + " else:\n", + " results = processor.add_chunk({RawEventProbability: da})\n", + " cur_detector_progress_bar.value += 1\n", + "\n", + " result = results[NMXReducedDataGroup]\n", + " display(result)\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nmx-dev-310", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/packages/essnmx/src/ess/nmx/mcstas/load.py b/packages/essnmx/src/ess/nmx/mcstas/load.py index c1d67f58..66b7ef71 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/load.py +++ b/packages/essnmx/src/ess/nmx/mcstas/load.py @@ -121,6 +121,17 @@ def raw_event_data_chunk_generator( Note that it only works if the dataset is already chunked. """ + if 0 < chunk_size < 10_000_000: + import warnings + + warnings.warn( + "The chunk size may be too small < 10_000_000.\n" + "Consider increasing the chunk size for better performance.\n" + "Hint: NMX typically expect ~10^8 bins as reduced data.", + UserWarning, + stacklevel=2, + ) + # Find the data bank name associated with the detector bank_prefix = load_event_data_bank_name( detector_name=detector_name, file_path=file_path @@ -136,7 +147,19 @@ def raw_event_data_chunk_generator( if chunk_size == 0: for data_slice in dset.dataset.iter_chunks(): dim_0_slice, _ = data_slice # dim_0_slice, dim_1_slice - yield _wrap_raw_event_data(dset["dim_0", dim_0_slice]) + da = _wrap_raw_event_data(dset["dim_0", dim_0_slice]) + if da.sizes['event'] < 10_000_000: + import warnings + + warnings.warn( + "The chunk size may be too small < 10_000_000.\n" + "Consider increasing the chunk size for better performance.\n" + "Hint: NMX typically expect ~10^8 bins as reduced data.", + UserWarning, + stacklevel=2, + ) + yield da + else: num_events = dset.shape[0] for start in range(0, num_events, chunk_size): diff --git a/packages/essnmx/src/ess/nmx/streaming.py b/packages/essnmx/src/ess/nmx/streaming.py new file mode 100644 index 00000000..ea08b673 --- /dev/null +++ b/packages/essnmx/src/ess/nmx/streaming.py @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2025 Scipp contributors (https://github.com/scipp) +from typing import Any + +import scipp as sc +import scippnexus as snx + +from ess.reduce.streaming import Accumulator + +from .mcstas.load import load_event_data_bank_name +from .types import DetectorBankPrefix, DetectorName, FilePath + + +class MinAccumulator(Accumulator): + """Accumulator that keeps track of the maximum value seen so far.""" + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._cur_min: sc.Variable | None = None + + @property + def value(self) -> sc.Variable | None: + return self._cur_min + + def _do_push(self, value: sc.Variable) -> None: + new_min = value.min() + if self._cur_min is None: + self._cur_min = new_min + else: + self._cur_min = min(self._cur_min, new_min) + + +class MaxAccumulator(Accumulator): + """Accumulator that keeps track of the maximum value seen so far.""" + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._cur_max: sc.Variable | None = None + + @property + def value(self) -> sc.Variable | None: + return self._cur_max + + def _do_push(self, value: sc.Variable) -> None: + new_max = value.max() + if self._cur_max is None: + self._cur_max = new_max + else: + self._cur_max = max(self._cur_max, new_max) + + +def calculate_number_of_chunks( + file_path: FilePath, + *, + detector_name: DetectorName, + bank_prefix: DetectorBankPrefix | None = None, + chunk_size: int = 0, # Number of rows to read at a time +) -> int: + """Calculate number of chunks in the event data. + + Parameters + ---------- + file_path: + Path to the nexus file + detector_name: + Name of the detector to load + pixel_ids: + Pixel ids to generate the data array with the events + chunk_size: + Number of rows to read at a time. + If 0, chunk slice is determined automatically by the ``iter_chunks``. + Note that it only works if the dataset is already chunked. + + """ + # Find the data bank name associated with the detector + bank_prefix = load_event_data_bank_name( + detector_name=detector_name, file_path=file_path + ) + bank_name = f'{bank_prefix}_dat_list_p_x_y_n_id_t' + with snx.File(file_path, 'r') as f: + root = f["entry1/data"] + (bank_name,) = (name for name in root.keys() if bank_name in name) + with snx.File(file_path, 'r') as f: + root = f["entry1/data"] + dset: snx.Field = root[bank_name]["events"] + if chunk_size == 0: + return len(list(dset.dataset.iter_chunks())) + else: + return dset.shape[0] // chunk_size + int(dset.shape[0] % chunk_size != 0) From 14fa9301e9aeece1f5acc72604f976efeec00985 Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Wed, 26 Feb 2025 17:31:30 +0100 Subject: [PATCH 241/403] Cleaning up documentation [docs] (#112) --- packages/essnmx/docs/api-reference/index.md | 9 ++++++--- packages/essnmx/docs/index.md | 2 +- packages/essnmx/docs/{examples => user-guide}/index.md | 3 ++- .../docs/{examples => user-guide}/scaling_workflow.ipynb | 0 .../essnmx/docs/{examples => user-guide}/workflow.ipynb | 0 .../docs/{examples => user-guide}/workflow_chunk.ipynb | 2 +- 6 files changed, 10 insertions(+), 6 deletions(-) rename packages/essnmx/docs/{examples => user-guide}/index.md (69%) rename packages/essnmx/docs/{examples => user-guide}/scaling_workflow.ipynb (100%) rename packages/essnmx/docs/{examples => user-guide}/workflow.ipynb (100%) rename packages/essnmx/docs/{examples => user-guide}/workflow_chunk.ipynb (99%) diff --git a/packages/essnmx/docs/api-reference/index.md b/packages/essnmx/docs/api-reference/index.md index 5777419f..cb30925c 100644 --- a/packages/essnmx/docs/api-reference/index.md +++ b/packages/essnmx/docs/api-reference/index.md @@ -10,8 +10,8 @@ :template: class-template.rst :recursive: - NMXData - NMXReducedData + NMXRawEventCountsDataGroup + NMXReducedDataGroup ``` @@ -35,7 +35,10 @@ :recursive: data - mcstas_loader + mcstas + reduction + nexus + streaming types mtz_io scaling diff --git a/packages/essnmx/docs/index.md b/packages/essnmx/docs/index.md index f6214638..b374a8f0 100644 --- a/packages/essnmx/docs/index.md +++ b/packages/essnmx/docs/index.md @@ -10,7 +10,7 @@ hidden: --- -examples/index +user-guide/index api-reference/index developer/index about/index diff --git a/packages/essnmx/docs/examples/index.md b/packages/essnmx/docs/user-guide/index.md similarity index 69% rename from packages/essnmx/docs/examples/index.md rename to packages/essnmx/docs/user-guide/index.md index 9c3f900d..55c9089d 100644 --- a/packages/essnmx/docs/examples/index.md +++ b/packages/essnmx/docs/user-guide/index.md @@ -1,4 +1,4 @@ -# Examples +# User Guide ```{toctree} --- @@ -6,5 +6,6 @@ maxdepth: 2 --- workflow +workflow_chunk scaling_workflow ``` diff --git a/packages/essnmx/docs/examples/scaling_workflow.ipynb b/packages/essnmx/docs/user-guide/scaling_workflow.ipynb similarity index 100% rename from packages/essnmx/docs/examples/scaling_workflow.ipynb rename to packages/essnmx/docs/user-guide/scaling_workflow.ipynb diff --git a/packages/essnmx/docs/examples/workflow.ipynb b/packages/essnmx/docs/user-guide/workflow.ipynb similarity index 100% rename from packages/essnmx/docs/examples/workflow.ipynb rename to packages/essnmx/docs/user-guide/workflow.ipynb diff --git a/packages/essnmx/docs/examples/workflow_chunk.ipynb b/packages/essnmx/docs/user-guide/workflow_chunk.ipynb similarity index 99% rename from packages/essnmx/docs/examples/workflow_chunk.ipynb rename to packages/essnmx/docs/user-guide/workflow_chunk.ipynb index 1618b070..148fe175 100644 --- a/packages/essnmx/docs/examples/workflow_chunk.ipynb +++ b/packages/essnmx/docs/user-guide/workflow_chunk.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Chunk Workflow\n", + "# Workflow - Chunk by Chunk\n", "In this example, we will process McStas events chunk by chunk, panel by panel." ] }, From 5dbc15efb3161bc85a21f90c1c994b7170a867c1 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Wed, 26 Feb 2025 21:40:46 +0100 Subject: [PATCH 242/403] Fix broken links --- packages/essnmx/docs/about/.DS_Store | Bin 0 -> 6148 bytes .../essnmx/docs/about/data_workflow_overview.md | 2 +- packages/essnmx/docs/about/index.md | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 packages/essnmx/docs/about/.DS_Store diff --git a/packages/essnmx/docs/about/.DS_Store b/packages/essnmx/docs/about/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 Date: Thu, 27 Feb 2025 14:07:58 +0100 Subject: [PATCH 243/403] ci: copier update --- packages/essnmx/.copier-answers.yml | 2 +- packages/essnmx/.github/workflows/docs.yml | 4 +++- packages/essnmx/tox.ini | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/.copier-answers.yml b/packages/essnmx/.copier-answers.yml index 18cbbc8f..59b54739 100644 --- a/packages/essnmx/.copier-answers.yml +++ b/packages/essnmx/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY -_commit: 5c4fd02 +_commit: 18e7005 _src_path: gh:scipp/copier_template description: Data reduction for NMX at the European Spallation Source. max_python: '3.13' diff --git a/packages/essnmx/.github/workflows/docs.yml b/packages/essnmx/.github/workflows/docs.yml index a90bcf10..47294b92 100644 --- a/packages/essnmx/.github/workflows/docs.yml +++ b/packages/essnmx/.github/workflows/docs.yml @@ -65,11 +65,13 @@ jobs: - run: tox -e linkcheck if: ${{ inputs.linkcheck }} - uses: actions/upload-artifact@v4 + id: artifact-upload-step with: name: docs_html path: html/ + - run: echo "::notice::https://remote-unzip.deno.dev/${{ github.repository }}/artifacts/${{ steps.artifact-upload-step.outputs.artifact-id }}" - - uses: JamesIves/github-pages-deploy-action@v4.7.2 + - uses: JamesIves/github-pages-deploy-action@v4.7.3 if: ${{ inputs.publish }} with: branch: gh-pages diff --git a/packages/essnmx/tox.ini b/packages/essnmx/tox.ini index e1e1cbd0..b11d187c 100644 --- a/packages/essnmx/tox.ini +++ b/packages/essnmx/tox.ini @@ -64,6 +64,8 @@ description = Update dependencies by running pip-compile-multi deps = pip-compile-multi tomli + # Avoid https://github.com/jazzband/pip-tools/issues/2131 + pip==24.2 skip_install = true changedir = requirements commands = python ./make_base.py --nightly scipp,sciline,scippnexus,plopp From 4d28a2d9e326e8a932bfa4a93244c3ab8d019adb Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Tue, 4 Mar 2025 13:03:57 +0100 Subject: [PATCH 244/403] Retrieving raw data metadata separately (#114) * Separate metatadata from event data for easy export. * Add raw data metadata retrieval part. * Move functions to more proper module. --- .../docs/user-guide/workflow_chunk.ipynb | 64 ++++++++----- .../essnmx/src/ess/nmx/mcstas/__init__.py | 4 + packages/essnmx/src/ess/nmx/mcstas/load.py | 57 ++++++++--- packages/essnmx/src/ess/nmx/mcstas/xml.py | 96 +++++++++++++------ packages/essnmx/src/ess/nmx/reduction.py | 27 ++++-- packages/essnmx/src/ess/nmx/types.py | 18 +++- packages/essnmx/tests/loader_test.py | 6 +- 7 files changed, 194 insertions(+), 78 deletions(-) diff --git a/packages/essnmx/docs/user-guide/workflow_chunk.ipynb b/packages/essnmx/docs/user-guide/workflow_chunk.ipynb index 148fe175..a99e72bc 100644 --- a/packages/essnmx/docs/user-guide/workflow_chunk.ipynb +++ b/packages/essnmx/docs/user-guide/workflow_chunk.ipynb @@ -37,17 +37,17 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Compute Maximum Probabiliity\n", + "## Compute Raw Data Metadata\n", "\n", - "`McStasWeight2CountScaleFactor` should not be different from chunk to chunk.\n", + "`time-of-flight` coordinate and `McStasWeight2CountScaleFactor` should not be different from chunk to chunk.\n", "\n", - "Therefore we need to compute `McStasWeight2CoutScaleFactor` before we compute `NMXReducedDataGroup`.\n", + "Therefore we need to compute `TimeBinStep` and `McStasWeight2CoutScaleFactor` before we compute `NMXReducedData`.\n", "\n", "It can be done by `ess.reduce.streaming.StreamProcessor`.\n", "\n", - "In this example, `MaximumProbability` will be renewed every time a chunk is added to the streaming processor.\n", + "In this example, `MinimumTimeOfArrival`, `MaximumTimeOfArrival` and `MaximumProbability` will be renewed every time a chunk is added to the streaming processor.\n", "\n", - "`MaxAccumulator` remembers the previous maximum value and compute new maximum value with the new chunk.\n", + "`(Min/Max)Accumulator` remembers the previous minimum/maximum value and compute new minimum/maximum value with the new chunk.\n", "\n", "``raw_event_data_chunk_generator`` yields a chunk of raw event probability from mcstas h5 file.\n", "\n", @@ -62,16 +62,20 @@ "source": [ "from functools import partial\n", "from ess.reduce.streaming import StreamProcessor\n", - "from ess.nmx.streaming import MaxAccumulator\n", + "from ess.nmx.streaming import MaxAccumulator, MinAccumulator\n", "\n", "# Stream processor building helper\n", "scalefactor_stream_processor = partial(\n", " StreamProcessor,\n", " dynamic_keys=(RawEventProbability,),\n", - " target_keys=(McStasWeight2CountScaleFactor,),\n", - " accumulators={MaximumProbability: MaxAccumulator},\n", + " target_keys=(NMXRawDataMetadata,),\n", + " accumulators={\n", + " MaximumProbability: MaxAccumulator,\n", + " MaximumTimeOfArrival: MaxAccumulator,\n", + " MinimumTimeOfArrival: MinAccumulator,\n", + " },\n", ")\n", - "scalefactor_wf = wf.copy()" + "metadata_wf = wf.copy()" ] }, { @@ -80,7 +84,7 @@ "metadata": {}, "outputs": [], "source": [ - "scalefactor_wf.visualize(McStasWeight2CountScaleFactor, graph_attr={\"rankdir\": \"TD\"}, compact=True)" + "metadata_wf.visualize(NMXRawDataMetadata, graph_attr={\"rankdir\": \"TD\"}, compact=True)" ] }, { @@ -90,7 +94,10 @@ "outputs": [], "source": [ "from ess.nmx.types import DetectorName\n", - "from ess.nmx.mcstas.load import raw_event_data_chunk_generator\n", + "from ess.nmx.mcstas.load import (\n", + " raw_event_data_chunk_generator,\n", + " mcstas_weight_to_probability_scalefactor,\n", + ")\n", "from ess.nmx.streaming import calculate_number_of_chunks\n", "from ipywidgets import IntProgress\n", "\n", @@ -99,10 +106,11 @@ "NUM_DETECTORS = 3\n", "\n", "# Loop over the detectors\n", - "file_path = scalefactor_wf.compute(FilePath)\n", - "scale_factors = {}\n", + "file_path = metadata_wf.compute(FilePath)\n", + "raw_data_metadatas = {}\n", + "\n", "for detector_i in range(0, NUM_DETECTORS):\n", - " temp_wf = scalefactor_wf.copy()\n", + " temp_wf = metadata_wf.copy()\n", " temp_wf[DetectorIndex] = detector_i\n", " detector_name = temp_wf.compute(DetectorName)\n", " max_chunk_id = calculate_number_of_chunks(\n", @@ -123,11 +131,18 @@ " else:\n", " results = processor.add_chunk({RawEventProbability: da})\n", " cur_detector_progress_bar.value += 1\n", - " scale_factors[detector_i] = results[McStasWeight2CountScaleFactor]\n", + " display(results[NMXRawDataMetadata])\n", + " raw_data_metadatas[detector_i] = results[NMXRawDataMetadata]\n", "\n", - "# We take the minimum scale factor for the entire dataset\n", - "scale_factor = min(scale_factors.values())\n", - "scale_factor\n" + "# We take the min/maximum values of the scale factor\n", + "min_toa = min(dg['min_toa'] for dg in raw_data_metadatas.values())\n", + "max_toa = max(dg['max_toa'] for dg in raw_data_metadatas.values())\n", + "max_probability = max(dg['max_probability'] for dg in raw_data_metadatas.values())\n", + "\n", + "toa_bin_edges = sc.linspace(dim='t', start=min_toa, stop=max_toa, num=51)\n", + "scale_factor = mcstas_weight_to_probability_scalefactor(\n", + " max_counts=wf.compute(MaximumCounts), max_probability=max_probability\n", + ")" ] }, { @@ -150,15 +165,14 @@ "from ess.nmx.mcstas.xml import McStasInstrument\n", "\n", "final_wf = wf.copy()\n", - "# Add the scale factor to the workflow\n", + "# Set the scale factor and time bin edges\n", "final_wf[McStasWeight2CountScaleFactor] = scale_factor\n", + "final_wf[TimeBinSteps] = toa_bin_edges\n", "\n", - "# Compute the static information in advance\n", - "# static_info = wf.compute([CrystalRotation, McStasInstrument])\n", - "static_info = wf.compute([McStasInstrument])\n", - "# final_wf[CrystalRotation] = static_info[CrystalRotation]\n", - "final_wf[CrystalRotation] = sc.vector([0, 0, 0.,], unit='deg')\n", - "final_wf[McStasInstrument] = static_info[McStasInstrument]\n", + "# Set the crystal rotation manually for now ...\n", + "final_wf[CrystalRotation] = sc.vector([0, 0, 0.0], unit='deg')\n", + "# Set static info\n", + "final_wf[McStasInstrument] = wf.compute(McStasInstrument)\n", "final_wf.visualize(NMXReducedDataGroup, compact=True)" ] }, diff --git a/packages/essnmx/src/ess/nmx/mcstas/__init__.py b/packages/essnmx/src/ess/nmx/mcstas/__init__.py index c40c8057..9680faf6 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/__init__.py +++ b/packages/essnmx/src/ess/nmx/mcstas/__init__.py @@ -6,6 +6,8 @@ def McStasWorkflow(): import sciline as sl from ess.nmx.reduction import ( + calculate_maximum_toa, + calculate_minimum_toa, format_nmx_reduced_data, proton_charge_from_event_counts, raw_event_probability_to_counts, @@ -18,6 +20,8 @@ def McStasWorkflow(): return sl.Pipeline( ( *loader_providers, + calculate_maximum_toa, + calculate_minimum_toa, read_mcstas_geometry_xml, proton_charge_from_event_counts, reduce_raw_event_probability, diff --git a/packages/essnmx/src/ess/nmx/mcstas/load.py b/packages/essnmx/src/ess/nmx/mcstas/load.py index 66b7ef71..1e5f432f 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/load.py +++ b/packages/essnmx/src/ess/nmx/mcstas/load.py @@ -14,7 +14,12 @@ FilePath, MaximumCounts, MaximumProbability, + MaximumTimeOfArrival, McStasWeight2CountScaleFactor, + MinimumTimeOfArrival, + NMXDetectorMetadata, + NMXExperimentMetadata, + NMXRawDataMetadata, NMXRawEventCountsDataGroup, PixelIds, RawEventProbability, @@ -245,22 +250,34 @@ def bank_names_to_detector_names(description: str) -> dict[str, list[str]]: return bank_names_to_detector_names +def load_experiment_metadata( + instrument: McStasInstrument, crystal_rotation: CrystalRotation +) -> NMXExperimentMetadata: + """Load the experiment metadata from the McStas file.""" + return NMXExperimentMetadata( + sc.DataGroup( + crystal_rotation=crystal_rotation, **instrument.experiment_metadata() + ) + ) + + +def load_detector_metadata( + instrument: McStasInstrument, detector_name: DetectorName +) -> NMXDetectorMetadata: + """Load the detector metadata from the McStas file.""" + return NMXDetectorMetadata( + sc.DataGroup(**instrument.detector_metadata(detector_name)) + ) + + def load_mcstas( *, da: RawEventProbability, - crystal_rotation: CrystalRotation, - detector_name: DetectorName, - instrument: McStasInstrument, + experiment_metadata: NMXExperimentMetadata, + detector_metadata: NMXDetectorMetadata, ) -> NMXRawEventCountsDataGroup: - coords = instrument.to_coords(detector_name) return NMXRawEventCountsDataGroup( - sc.DataGroup( - weights=da, - crystal_rotation=crystal_rotation, - name=sc.scalar(detector_name), - pixel_id=instrument.pixel_ids(detector_name), - **coords, - ) + sc.DataGroup(weights=da, **experiment_metadata, **detector_metadata) ) @@ -271,7 +288,23 @@ def retrieve_pixel_ids( return PixelIds(instrument.pixel_ids(detector_name)) +def retrieve_raw_data_metadata( + min_toa: MinimumTimeOfArrival, + max_toa: MaximumTimeOfArrival, + max_probability: MaximumProbability, +) -> NMXRawDataMetadata: + """Retrieve the metadata of the raw data.""" + return NMXRawDataMetadata( + sc.DataGroup( + min_toa=min_toa, + max_toa=max_toa, + max_probability=max_probability, + ) + ) + + providers = ( + retrieve_raw_data_metadata, read_mcstas_geometry_xml, detector_name_from_index, load_event_data_bank_name, @@ -281,4 +314,6 @@ def retrieve_pixel_ids( retrieve_pixel_ids, load_crystal_rotation, load_mcstas, + load_experiment_metadata, + load_detector_metadata, ) diff --git a/packages/essnmx/src/ess/nmx/mcstas/xml.py b/packages/essnmx/src/ess/nmx/mcstas/xml.py index 3014220d..5478f6a0 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas/xml.py @@ -212,6 +212,11 @@ def num_fast_pixels_per_row(self) -> int: """Number of pixels in each row of the detector along the fast axis.""" return self.num_x if self.fast_axis_name == 'x' else self.num_y + @property + def detector_shape(self) -> tuple: + """Shape of the detector panel. (num_x, num_y)""" + return (self.num_x, self.num_y) + def _collect_detector_descriptions(tree: _XML) -> tuple[DetectorDesc, ...]: """Retrieve detector geometry descriptions from mcstas file.""" @@ -310,12 +315,18 @@ def from_xml( ) +def _construct_pixel_id(detector_desc: DetectorDesc) -> sc.Variable: + """Pixel IDs for single detector.""" + start, stop = ( + detector_desc.id_start, + detector_desc.id_start + detector_desc.total_pixels, + ) + return sc.arange('id', start, stop, unit=None) + + def _construct_pixel_ids(detector_descs: tuple[DetectorDesc, ...]) -> sc.Variable: """Pixel IDs for all detectors.""" - intervals = [ - (desc.id_start, desc.id_start + desc.total_pixels) for desc in detector_descs - ] - ids = [sc.arange('id', start, stop, unit=None) for start, stop in intervals] + ids = [_construct_pixel_id(det) for det in detector_descs] return sc.concat(ids, 'id') @@ -345,17 +356,6 @@ def _pixel_positions( ) + position_offset -def _detector_pixel_positions( - detector_descs: tuple[DetectorDesc, ...], sample: SampleDesc -) -> sc.Variable: - """Position of pixels of all detectors.""" - positions = [ - _pixel_positions(detector, sample.position_from_sample(detector.position)) - for detector in detector_descs - ] - return sc.concat(positions, 'panel') - - @dataclass class McStasInstrument: simulation_settings: SimulationSettings @@ -380,30 +380,72 @@ def from_xml(cls, tree: _XML) -> 'McStasInstrument': ) def pixel_ids(self, *det_names: str) -> sc.Variable: - detectors = tuple(det for det in self.detectors if det.name in det_names) - return _construct_pixel_ids(detectors) + """Pixel IDs for the detectors. - def to_coords(self, *det_names: str) -> dict[str, sc.Variable]: - """Extract coordinates from the McStas instrument description. + If multiple detectors are requested, all pixel IDs will be concatenated along + the 'id' dimension. Parameters ---------- det_names: - Names of the detectors to extract coordinates for. + Names of the detectors to extract pixel IDs for. """ detectors = tuple(det for det in self.detectors if det.name in det_names) - slow_axes = [det.slow_axis for det in detectors] - fast_axes = [det.fast_axis for det in detectors] - origins = [self.sample.position_from_sample(det.position) for det in detectors] + return _construct_pixel_ids(detectors) + + def experiment_metadata(self) -> dict[str, sc.Variable]: + """Extract experiment metadata from the McStas instrument description.""" return { - 'fast_axis': sc.concat(fast_axes, 'panel'), - 'slow_axis': sc.concat(slow_axes, 'panel'), - 'origin_position': sc.concat(origins, 'panel'), 'sample_position': self.sample.position_from_sample(self.sample.position), 'source_position': self.sample.position_from_sample(self.source.position), 'sample_name': sc.scalar(self.sample.name), - 'position': _detector_pixel_positions(detectors, self.sample), + } + + def _detector_metadata(self, det_name: str) -> dict[str, sc.Variable]: + try: + detector = next(det for det in self.detectors if det.name == det_name) + except StopIteration as e: + raise KeyError(f"Detector {det_name} not found.") from e + return { + 'fast_axis': detector.fast_axis, + 'slow_axis': detector.slow_axis, + 'origin_position': self.sample.position_from_sample(detector.position), + 'position': _pixel_positions( + detector, self.sample.position_from_sample(detector.position) + ), + 'detector_shape': sc.scalar(detector.detector_shape), + 'x_pixel_size': detector.step_x, + 'y_pixel_size': detector.step_y, + 'detector_name': sc.scalar(detector.name), + } + + def detector_metadata(self, *det_names: str) -> dict[str, sc.Variable]: + """Extract detector metadata from the McStas instrument description. + + If multiple detector is requested, all metadata will be concatenated along the + 'panel' dimension. + + Parameters + ---------- + det_names: + Names of the detectors to extract metadata for. + + """ + if len(det_names) == 1: + return self._detector_metadata(det_names[0]) + detector_metadatas = { + det_name: self._detector_metadata(det_name) for det_name in det_names + } + # Concat all metadata into panel dimension + metadata_keys: set[str] = set().union( + set(detector_metadatas[det_name].keys()) for det_name in det_names + ) + return { + key: sc.concat( + [metadata[key] for metadata in detector_metadatas.values()], 'panel' + ) + for key in metadata_keys } diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/reduction.py index 5d54323d..19e70535 100644 --- a/packages/essnmx/src/ess/nmx/reduction.py +++ b/packages/essnmx/src/ess/nmx/reduction.py @@ -2,11 +2,12 @@ # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) import scipp as sc -from .mcstas.xml import McStasInstrument from .types import ( - CrystalRotation, - DetectorName, + MaximumTimeOfArrival, McStasWeight2CountScaleFactor, + MinimumTimeOfArrival, + NMXDetectorMetadata, + NMXExperimentMetadata, NMXReducedCounts, NMXReducedDataGroup, NMXReducedProbability, @@ -17,6 +18,16 @@ ) +def calculate_minimum_toa(da: RawEventProbability) -> MinimumTimeOfArrival: + """Calculate the minimum time of arrival from the data.""" + return MinimumTimeOfArrival(da.coords['t'].min()) + + +def calculate_maximum_toa(da: RawEventProbability) -> MaximumTimeOfArrival: + """Calculate the maximum time of arrival from the data.""" + return MaximumTimeOfArrival(da.coords['t'].max()) + + def proton_charge_from_event_counts(da: NMXReducedCounts) -> ProtonCharge: """Make up the proton charge from the event counts. @@ -53,20 +64,18 @@ def raw_event_probability_to_counts( def format_nmx_reduced_data( da: NMXReducedCounts, - detector_name: DetectorName, - instrument: McStasInstrument, proton_charge: ProtonCharge, - crystal_rotation: CrystalRotation, + experiment_metadata: NMXExperimentMetadata, + detector_metadata: NMXDetectorMetadata, ) -> NMXReducedDataGroup: """Bin time of arrival data into ``time_bin_step`` bins.""" - new_coords = instrument.to_coords(detector_name) return NMXReducedDataGroup( sc.DataGroup( counts=da, proton_charge=proton_charge, - crystal_rotation=crystal_rotation, - **new_coords, + **experiment_metadata, + **detector_metadata, ) ) diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index a79921f6..100261b2 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -24,13 +24,18 @@ McStasWeight2CountScaleFactor = NewType("McStasWeight2CountScaleFactor", sc.Variable) """Scale factor to convert McStas weights to counts""" +NMXExperimentMetadata = NewType("NMXExperimentMetadata", sc.DataGroup) +"""Metadata of the experiment""" + +NMXDetectorMetadata = NewType("NMXDetectorMetadata", sc.DataGroup) +"""Metadata of the detector""" RawEventProbability = NewType("RawEventProbability", sc.DataArray) """DataArray containing the event probabilities read from the McStas file, has coordinates 'id' and 't' """ NMXRawEventCountsDataGroup = NewType("NMXRawEventCountsDataGroup", sc.DataGroup) -"""DataGroup containing the RawEventData and other metadata""" +"""DataGroup containing the RawEventData, experiment metadata and detector metadata""" ProtonCharge = NewType("ProtonCharge", sc.Variable) """The proton charge signal""" @@ -54,4 +59,13 @@ """Histogram of time-of-arrival and pixel-id.""" NMXReducedDataGroup = NewType("NMXReducedDataGroup", sc.DataGroup) -"""Histogram of time-of-arrival and pixel-id, with additional metadata.""" +"""Datagroup containing Histogram(id, t), experiment metadata and detector metadata""" + +MinimumTimeOfArrival = NewType("MinimumTimeOfArrival", sc.Variable) +"""Minimum time of arrival of the raw data""" + +MaximumTimeOfArrival = NewType("MaximumTimeOfArrival", sc.Variable) +"""Maximum time of arrival of the raw data""" + +NMXRawDataMetadata = NewType("NMXRawDataMetadata", sc.DataGroup) +"""Metadata of the raw data, i.e. maximum weight and min/max time of arrival""" diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index ccd306eb..622652d0 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -47,10 +47,8 @@ def check_nmxdata_properties( dg: NMXRawEventCountsDataGroup, fast_axis, slow_axis ) -> None: assert isinstance(dg, sc.DataGroup) - assert_allclose( - sc.squeeze(dg['fast_axis'], 'panel'), fast_axis, atol=sc.scalar(0.005) - ) - assert_identical(sc.squeeze(dg['slow_axis'], 'panel'), slow_axis) + assert_allclose(dg['fast_axis'], fast_axis, atol=sc.scalar(0.005)) + assert_identical(dg['slow_axis'], slow_axis) @pytest.mark.parametrize( From d8eac8e45f4fda3441e3efcdf840ee474a20f907 Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Tue, 11 Mar 2025 09:59:31 +0100 Subject: [PATCH 245/403] Bump pin of essreduce to use accumulators. (#120) * Lower pin of essreduce to use accumulators. * Use essreduce accumulator. --- packages/essnmx/docs/user-guide/workflow_chunk.ipynb | 5 ++--- packages/essnmx/pyproject.toml | 2 +- packages/essnmx/requirements/base.in | 2 +- packages/essnmx/requirements/base.txt | 6 +++--- packages/essnmx/requirements/basetest.txt | 2 +- packages/essnmx/requirements/docs.txt | 2 +- packages/essnmx/requirements/nightly.in | 2 +- packages/essnmx/requirements/nightly.txt | 8 ++++---- 8 files changed, 14 insertions(+), 15 deletions(-) diff --git a/packages/essnmx/docs/user-guide/workflow_chunk.ipynb b/packages/essnmx/docs/user-guide/workflow_chunk.ipynb index a99e72bc..911b6d83 100644 --- a/packages/essnmx/docs/user-guide/workflow_chunk.ipynb +++ b/packages/essnmx/docs/user-guide/workflow_chunk.ipynb @@ -61,8 +61,7 @@ "outputs": [], "source": [ "from functools import partial\n", - "from ess.reduce.streaming import StreamProcessor\n", - "from ess.nmx.streaming import MaxAccumulator, MinAccumulator\n", + "from ess.reduce.streaming import StreamProcessor, MaxAccumulator, MinAccumulator\n", "\n", "# Stream processor building helper\n", "scalefactor_stream_processor = partial(\n", @@ -263,7 +262,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 9559362b..15486a27 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -32,7 +32,7 @@ requires-python = ">=3.10" # Make sure to list one dependency per line. dependencies = [ "dask", - "essreduce>=24.02.3", + "essreduce>=24.03.0", "graphviz", "plopp", "sciline>=24.06.0", diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index 5b8a9a1a..f7ca9a08 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -3,7 +3,7 @@ # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask -essreduce>=24.02.3 +essreduce>=24.03.0 graphviz plopp sciline>=24.06.0 diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 4c060f5d..38f4a2f5 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:ddb5c021f8aab4013482100474cd29d0000f612d +# SHA1:ce4455b8a5d721535a6305df40709de1d6c8e4c1 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -29,7 +29,7 @@ dnspython==2.7.0 # via email-validator email-validator==2.2.0 # via scippneutron -essreduce==25.2.4 +essreduce==25.3.0 # via -r base.in fonttools==4.56.0 # via matplotlib @@ -57,7 +57,7 @@ lazy-loader==0.4 # scippneutron locket==1.0.0 # via partd -matplotlib==3.10.0 +matplotlib==3.10.1 # via # mpltoolbox # plopp diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt index e1f778ef..1d0e4fcf 100644 --- a/packages/essnmx/requirements/basetest.txt +++ b/packages/essnmx/requirements/basetest.txt @@ -13,7 +13,7 @@ packaging==24.2 # via pytest pluggy==1.5.0 # via pytest -pytest==8.3.4 +pytest==8.3.5 # via -r basetest.in tomli==2.2.1 # via pytest diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 11002120..48ccceb9 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -54,7 +54,7 @@ ipydatawidgets==4.3.5 # via pythreejs ipykernel==6.29.5 # via -r docs.in -ipython==8.32.0 +ipython==8.33.0 # via # -r docs.in # ipykernel diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index 003773ba..5d1d38fd 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -2,7 +2,7 @@ # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask -essreduce>=24.02.3 +essreduce>=24.03.0 graphviz pooch pandas diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 376e00e8..69623e98 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:a1aeab214632eed331a1c2075a27b248a26d4755 +# SHA1:707e21bd0906494a252089989bf9ab05df8a820e # # This file is autogenerated by pip-compile-multi # To update, run: @@ -32,7 +32,7 @@ dnspython==2.7.0 # via email-validator email-validator==2.2.0 # via scippneutron -essreduce==25.2.4 +essreduce==25.3.0 # via -r nightly.in exceptiongroup==1.2.2 # via pytest @@ -64,7 +64,7 @@ lazy-loader==0.4 # scippneutron locket==1.0.0 # via partd -matplotlib==3.10.0 +matplotlib==3.10.1 # via # mpltoolbox # plopp @@ -111,7 +111,7 @@ pydantic-core==2.29.0 # via pydantic pyparsing==3.2.1 # via matplotlib -pytest==8.3.4 +pytest==8.3.5 # via -r nightly.in python-dateutil==2.9.0.post0 # via From d19082dbce6b1ccd582cf7e490edfdb876834e86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Mar 2025 17:29:00 +0000 Subject: [PATCH 246/403] Bump scipp from 25.2.0 to 25.3.0 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 25.2.0 to 25.3.0. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/25.02.0...25.03.0) --- updated-dependencies: - dependency-name: scipp dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 38f4a2f5..b853d09b 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -117,7 +117,7 @@ sciline==24.10.0 # via # -r base.in # essreduce -scipp==25.2.0 +scipp==25.3.0 # via # -r base.in # essreduce From 71038e10148467fbda87f288a632638212eee8eb Mon Sep 17 00:00:00 2001 From: Mridul Seth Date: Wed, 26 Mar 2025 16:27:40 +0100 Subject: [PATCH 247/403] copier bump for pypi publish --- packages/essnmx/.copier-answers.yml | 2 +- packages/essnmx/.github/workflows/release.yml | 2 +- packages/essnmx/.gitignore | 1 + packages/essnmx/docs/index.md | 36 +++++++++++++++++-- packages/essnmx/docs/user-guide/index.md | 3 +- .../essnmx/docs/user-guide/installation.md | 16 +++++++++ 6 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 packages/essnmx/docs/user-guide/installation.md diff --git a/packages/essnmx/.copier-answers.yml b/packages/essnmx/.copier-answers.yml index 59b54739..cfbb0254 100644 --- a/packages/essnmx/.copier-answers.yml +++ b/packages/essnmx/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY -_commit: 18e7005 +_commit: cdbfd9c _src_path: gh:scipp/copier_template description: Data reduction for NMX at the European Spallation Source. max_python: '3.13' diff --git a/packages/essnmx/.github/workflows/release.yml b/packages/essnmx/.github/workflows/release.yml index fe8ef23d..f88b3447 100644 --- a/packages/essnmx/.github/workflows/release.yml +++ b/packages/essnmx/.github/workflows/release.yml @@ -68,7 +68,7 @@ jobs: if: github.event_name == 'release' && github.event.action == 'published' steps: - uses: actions/download-artifact@v4 - - uses: pypa/gh-action-pypi-publish@v1.8.14 + - uses: pypa/gh-action-pypi-publish@v1.12.4 upload_conda: name: Deploy Conda diff --git a/packages/essnmx/.gitignore b/packages/essnmx/.gitignore index 340e6499..720318ba 100644 --- a/packages/essnmx/.gitignore +++ b/packages/essnmx/.gitignore @@ -14,6 +14,7 @@ venv .venv # Caches +*.DS_Store .clangd/ *.ipynb_checkpoints __pycache__/ diff --git a/packages/essnmx/docs/index.md b/packages/essnmx/docs/index.md index b374a8f0..fe17dff6 100644 --- a/packages/essnmx/docs/index.md +++ b/packages/essnmx/docs/index.md @@ -1,10 +1,42 @@ -# ESSnmx +:::{image} _static/logo.svg +:class: only-light +:alt: ESSnmx +:width: 60% +:align: center +::: +:::{image} _static/logo-dark.svg +:class: only-dark +:alt: ESSnmx +:width: 60% +:align: center +::: - +```{raw} html + +``` + +```{role} transparent +``` + +# {transparent}`ESSnmx` + + Data reduction for NMX at the European Spallation Source.

+:::{include} user-guide/installation.md +:heading-offset: 1 +::: + +## Get in touch + +- If you have questions that are not answered by these documentation pages, ask on [discussions](https://github.com/scipp/essnmx/discussions). Please include a self-contained reproducible example if possible. +- Report bugs (including unclear, missing, or wrong documentation!), suggest features or view the source code [on GitHub](https://github.com/scipp/essnmx). + ```{toctree} --- hidden: diff --git a/packages/essnmx/docs/user-guide/index.md b/packages/essnmx/docs/user-guide/index.md index 55c9089d..8f7ddfd0 100644 --- a/packages/essnmx/docs/user-guide/index.md +++ b/packages/essnmx/docs/user-guide/index.md @@ -2,10 +2,11 @@ ```{toctree} --- -maxdepth: 2 +maxdepth: 1 --- workflow workflow_chunk scaling_workflow +installation ``` diff --git a/packages/essnmx/docs/user-guide/installation.md b/packages/essnmx/docs/user-guide/installation.md new file mode 100644 index 00000000..412db9dc --- /dev/null +++ b/packages/essnmx/docs/user-guide/installation.md @@ -0,0 +1,16 @@ +# Installation + +To install ESSnmx and all of its dependencies, use + +`````{tab-set} +````{tab-item} pip +```sh +pip install essnmx +``` +```` +````{tab-item} conda +```sh +conda install -c conda-forge -c scipp essnmx +``` +```` +````` From e6724add99593446de1fe46d069ae0a4853c5086 Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Thu, 27 Mar 2025 13:37:32 +0100 Subject: [PATCH 248/403] NXLauetof format and workflow/IO helpers. * Lauetof export interface. * Raw data metadata as dataclass * Allow arbitrary metadata and export time of flight from the coordinate. * Specify unit Co-authored-by: Simon Heybrock <12912489+SimonHeybrock@users.noreply.github.com> * Add docstring to export methods. * Add missing attributes. * Remove comments * Write unit only when applicable. * Update comment. * Validity check when appending histogram on top of metadata (#116) * Separate metatadata from event data for easy export. * Add raw data metadata retrieval part. * Lauetof export interface. * Raw data metadata as dataclass * Allow arbitrary metadata and export time of flight from the coordinate. * Separate metatadata from event data for easy export. * Add raw data metadata retrieval part. * Satety check in the export function. * Add warning filter. * Apply automatic formatting * Apply automatic formatting * Fix typo * Move functions to more proper module. * Lauetof export interface. * Raw data metadata as dataclass * Allow arbitrary metadata and export time of flight from the coordinate. * Specify unit Co-authored-by: Simon Heybrock <12912489+SimonHeybrock@users.noreply.github.com> * Add docstring to export methods. * Add missing attributes. * Remove comments * Fix typo. * Apply automatic formatting --------- Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Simon Heybrock <12912489+SimonHeybrock@users.noreply.github.com> * Wrap detector processing step. * Lower pin of essreduce to use accumulators. * Specify dtype of string. [skip ci] * Use essreduce accumulator. * Export NXsource. * Fix type hint * Apply automatic formatting * Data reduction wrapper interface (#122) * Executable module. * Save crystal rotation. * Fix crystal rotation. * Remove all zero lines, not just the first one (#123) * remove all zero lines, not just the first one * Apply automatic formatting * Update src/ess/nmx/mcstas/load.py * Apply automatic formatting --------- Co-authored-by: Aaron Finke Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Sunyoung Yoo * Apply suggestions from code review Co-authored-by: Mridul Seth * Apply automatic formatting * Add bitshuffle support, compress binned datasets using bitshuffle/LZ4 (#125) * add bitshuffle support, compress binned datasets using bitshuffle/LZ4 * Apply automatic formatting * Add docstring and option. --------- Co-authored-by: Aaron Finke Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: YooSunyoung * Update default space group in mtz io module (#126) * Executable module. * Save crystal rotation. * Fix crystal rotation. * Remove all zero lines, not just the first one (#123) * remove all zero lines, not just the first one * Apply automatic formatting * Update src/ess/nmx/mcstas/load.py * Apply automatic formatting --------- Co-authored-by: Aaron Finke Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Sunyoung Yoo * Apply suggestions from code review Co-authored-by: Mridul Seth * Apply automatic formatting * Add bitshuffle support, compress binned datasets using bitshuffle/LZ4 (#125) * add bitshuffle support, compress binned datasets using bitshuffle/LZ4 * Apply automatic formatting * Add docstring and option. --------- Co-authored-by: Aaron Finke Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: YooSunyoung * Update mtz_io.py make default space group P1 (lowest symmetry space group) * Update default space group in tests. --------- Co-authored-by: YooSunyoung Co-authored-by: Aaron Finke Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Mridul Seth --------- Co-authored-by: Aaron Finke <45569605+aaronfinke@users.noreply.github.com> Co-authored-by: Aaron Finke Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Mridul Seth --------- Co-authored-by: Simon Heybrock <12912489+SimonHeybrock@users.noreply.github.com> Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Aaron Finke <45569605+aaronfinke@users.noreply.github.com> Co-authored-by: Aaron Finke Co-authored-by: Mridul Seth --- .../docs/user-guide/workflow_chunk.ipynb | 98 ++-- packages/essnmx/pyproject.toml | 4 + .../essnmx/src/ess/nmx/mcstas/__init__.py | 6 +- .../essnmx/src/ess/nmx/mcstas/executables.py | 276 ++++++++++++ packages/essnmx/src/ess/nmx/mcstas/load.py | 15 +- packages/essnmx/src/ess/nmx/mtz_io.py | 2 +- packages/essnmx/src/ess/nmx/nexus.py | 424 +++++++++++++++++- packages/essnmx/src/ess/nmx/types.py | 11 +- packages/essnmx/tests/mtz_io_test.py | 6 +- 9 files changed, 765 insertions(+), 77 deletions(-) create mode 100644 packages/essnmx/src/ess/nmx/mcstas/executables.py diff --git a/packages/essnmx/docs/user-guide/workflow_chunk.ipynb b/packages/essnmx/docs/user-guide/workflow_chunk.ipynb index 911b6d83..0892b8bd 100644 --- a/packages/essnmx/docs/user-guide/workflow_chunk.ipynb +++ b/packages/essnmx/docs/user-guide/workflow_chunk.ipynb @@ -134,9 +134,9 @@ " raw_data_metadatas[detector_i] = results[NMXRawDataMetadata]\n", "\n", "# We take the min/maximum values of the scale factor\n", - "min_toa = min(dg['min_toa'] for dg in raw_data_metadatas.values())\n", - "max_toa = max(dg['max_toa'] for dg in raw_data_metadatas.values())\n", - "max_probability = max(dg['max_probability'] for dg in raw_data_metadatas.values())\n", + "min_toa = min(meta.min_toa for meta in raw_data_metadatas.values())\n", + "max_toa = max(meta.max_toa for meta in raw_data_metadatas.values())\n", + "max_probability = max(meta.max_probability for meta in raw_data_metadatas.values())\n", "\n", "toa_bin_edges = sc.linspace(dim='t', start=min_toa, stop=max_toa, num=51)\n", "scale_factor = mcstas_weight_to_probability_scalefactor(\n", @@ -144,15 +144,28 @@ ")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compute Metadata\n", + "\n", + "Other metadata does not require any chunk-based computation.\n", + "\n", + "Therefore we export the metadata first and append detector data later." + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Compute Final Output\n", "\n", - "Now with the `scale_factor: McStasWeight2CountScaleFactor`, we can compute the final output chunk by chunk.\n", + "Now with all the metadata, we can compute the final output chunk by chunk.\n", + "\n", + "We will also compute static parameters in advance so that stream processor does not compute them every time another chunk is added.\n", "\n", - "We will also compute static parameters in advance so that stream processor does not compute them every time another chunk is added." + "We will as well export the reduced data detector by detector." ] }, { @@ -181,47 +194,10 @@ "metadata": {}, "outputs": [], "source": [ - "from functools import partial\n", - "from ess.reduce.streaming import StreamProcessor, EternalAccumulator\n", + "from ess.nmx.nexus import NXLauetofWriter\n", "\n", - "# Stream processor building helper\n", - "final_stream_processor = partial(\n", - " StreamProcessor,\n", - " dynamic_keys=(RawEventProbability,),\n", - " target_keys=(NMXReducedDataGroup,),\n", - " accumulators={NMXReducedCounts: EternalAccumulator},\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from ess.nmx.types import DetectorName\n", - "from ess.nmx.mcstas.load import raw_event_data_chunk_generator\n", - "from ess.nmx.streaming import calculate_number_of_chunks\n", - "from ipywidgets import IntProgress\n", "\n", - "CHUNK_SIZE = 10 # Number of event rows to process at once\n", - "# Increase this number to speed up the processing\n", - "NUM_DETECTORS = 3\n", - "final_wf[TimeBinSteps] = sc.linspace(\n", - " 't', 0.1, 0.15, 51, unit='s'\n", - ") # Time bin edges can be calculated from the data\n", - "# But streaming processor only sees the first chunk\n", - "# So we need to set it manually before processing the first chunk\n", - "# It is a bit cumbersome since we have to know the range of the time bins in advance\n", - "\n", - "# Loop over the detectors\n", - "file_path = final_wf.compute(FilePath)\n", - "for detector_i in range(0, NUM_DETECTORS):\n", - " temp_wf = final_wf.copy()\n", - " temp_wf[DetectorIndex] = detector_i\n", - " # First compute static information\n", - " detector_name = temp_wf.compute(DetectorName)\n", - " temp_wf[PixelIds] = temp_wf.compute(PixelIds)\n", + "def temp_generator(file_path, detector_name):\n", " max_chunk_id = calculate_number_of_chunks(\n", " file_path, detector_name=detector_name, chunk_size=CHUNK_SIZE\n", " )\n", @@ -229,20 +205,36 @@ " min=0, max=max_chunk_id, description=f\"Detector {detector_i}\"\n", " )\n", " display(cur_detector_progress_bar)\n", - "\n", - " # Build the stream processor\n", - " processor = final_stream_processor(temp_wf)\n", " for da in raw_event_data_chunk_generator(\n", " file_path=file_path, detector_name=detector_name, chunk_size=CHUNK_SIZE\n", " ):\n", - " if any(da.sizes.values()) == 0:\n", - " continue\n", - " else:\n", - " results = processor.add_chunk({RawEventProbability: da})\n", + " yield da\n", " cur_detector_progress_bar.value += 1\n", "\n", - " result = results[NMXReducedDataGroup]\n", - " display(result)\n" + "\n", + "# When a panel is added to the writer,\n", + "# the writer will start processing the data from the generator\n", + "# and store the results in memory\n", + "# The writer will then write the data to the file\n", + "# when ``save`` is called\n", + "writer = NXLauetofWriter(\n", + " chunk_generator=temp_generator,\n", + " chunk_insert_key=RawEventProbability,\n", + " workflow=final_wf,\n", + " output_filename=\"test.h5\",\n", + " overwrite=True,\n", + " extra_meta={\"McStasWeight2CountScaleFactor\": scale_factor},\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for detector_i in range(3):\n", + " display(writer.add_panel(detector_id=detector_i))" ] } ], diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 15486a27..b1fda2b9 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -42,10 +42,14 @@ dependencies = [ "pandas", "gemmi", "defusedxml", + "bitshuffle", ] dynamic = ["version"] +[project.scripts] +essnmx_reduce_mcstas = "ess.nmx.mcstas.executables:main" + [project.optional-dependencies] test = [ "pytest", diff --git a/packages/essnmx/src/ess/nmx/mcstas/__init__.py b/packages/essnmx/src/ess/nmx/mcstas/__init__.py index 9680faf6..3026dc20 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/__init__.py +++ b/packages/essnmx/src/ess/nmx/mcstas/__init__.py @@ -1,5 +1,8 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +from ..types import MaximumCounts + +default_parameters = {MaximumCounts: 10000} def McStasWorkflow(): @@ -27,5 +30,6 @@ def McStasWorkflow(): reduce_raw_event_probability, raw_event_probability_to_counts, format_nmx_reduced_data, - ) + ), + params=default_parameters, ) diff --git a/packages/essnmx/src/ess/nmx/mcstas/executables.py b/packages/essnmx/src/ess/nmx/mcstas/executables.py new file mode 100644 index 00000000..4e68d44e --- /dev/null +++ b/packages/essnmx/src/ess/nmx/mcstas/executables.py @@ -0,0 +1,276 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2025 Scipp contributors (https://github.com/scipp) +import argparse +import logging +import pathlib +import sys +from collections.abc import Callable +from functools import partial + +import sciline as sl +import scipp as sc + +from ess.reduce.streaming import ( + EternalAccumulator, + MaxAccumulator, + MinAccumulator, + StreamProcessor, +) + +from ..nexus import ( + _export_detector_metadata_as_nxlauetof, + _export_reduced_data_as_nxlauetof, + _export_static_metadata_as_nxlauetof, +) +from ..streaming import calculate_number_of_chunks +from ..types import ( + DetectorIndex, + DetectorName, + FilePath, + MaximumCounts, + MaximumProbability, + MaximumTimeOfArrival, + McStasWeight2CountScaleFactor, + MinimumTimeOfArrival, + NMXDetectorMetadata, + NMXExperimentMetadata, + NMXRawDataMetadata, + NMXReducedCounts, + NMXReducedDataGroup, + PixelIds, + RawEventProbability, + TimeBinSteps, +) +from . import McStasWorkflow +from .load import ( + mcstas_weight_to_probability_scalefactor, + raw_event_data_chunk_generator, +) +from .xml import McStasInstrument + + +def _build_metadata_streaming_processor_helper() -> ( + Callable[[sl.Pipeline], StreamProcessor] +): + return partial( + StreamProcessor, + dynamic_keys=(RawEventProbability,), + target_keys=(NMXRawDataMetadata,), + accumulators={ + MaximumProbability: MaxAccumulator, + MaximumTimeOfArrival: MaxAccumulator, + MinimumTimeOfArrival: MinAccumulator, + }, + ) + + +def _build_final_streaming_processor_helper() -> ( + Callable[[sl.Pipeline], StreamProcessor] +): + return partial( + StreamProcessor, + dynamic_keys=(RawEventProbability,), + target_keys=(NMXReducedDataGroup,), + accumulators={NMXReducedCounts: EternalAccumulator}, + ) + + +def calculate_raw_data_metadata( + *detector_ids: DetectorIndex | DetectorName, + wf: sl.Pipeline, + chunk_size: int = 10_000_000, + logger: logging.Logger | None = None, +) -> NMXRawDataMetadata: + # Stream processor building helper + scalefactor_stream_processor = _build_metadata_streaming_processor_helper() + metadata_wf = wf.copy() + # Loop over the detectors + file_path = metadata_wf.compute(FilePath) + raw_data_metadatas = {} + + for detector_i in detector_ids: + temp_wf = metadata_wf.copy() + if isinstance(detector_i, str): + temp_wf[DetectorName] = detector_i + else: + temp_wf[DetectorIndex] = detector_i + + detector_name = temp_wf.compute(DetectorName) + max_chunk_id = calculate_number_of_chunks( + temp_wf.compute(FilePath), + detector_name=detector_name, + chunk_size=chunk_size, + ) + # Build the stream processor + processor = scalefactor_stream_processor(temp_wf) + for i_da, da in enumerate( + raw_event_data_chunk_generator( + file_path=file_path, detector_name=detector_name, chunk_size=chunk_size + ) + ): + if any(da.sizes.values()) == 0: + continue + else: + results = processor.add_chunk({RawEventProbability: da}) + if logger is not None: + logger.info( + "[{%s}/{%s}] Processed chunk for {%s}", + i_da + 1, + max_chunk_id, + detector_name, + ) + + raw_data_metadatas[detector_i] = results[NMXRawDataMetadata] + + # We take the min/maximum values of the scale factor + # We are doing it manually because it is not possible to update parameters + # in the workflow that stream processor uses. + min_toa = min(dg.min_toa for dg in raw_data_metadatas.values()) + max_toa = max(dg.max_toa for dg in raw_data_metadatas.values()) + max_probability = max(dg.max_probability for dg in raw_data_metadatas.values()) + + return NMXRawDataMetadata( + min_toa=min_toa, max_toa=max_toa, max_probability=max_probability + ) + + +def reduction( + *, + input_file: pathlib.Path, + output_file: pathlib.Path, + chunk_size: int = 10_000_000, + detector_ids: list[int | str], + wf: sl.Pipeline | None = None, + logger: logging.Logger | None = None, +) -> None: + wf = wf.copy() if wf is not None else McStasWorkflow() + wf[FilePath] = input_file + # Set static info + wf[McStasInstrument] = wf.compute(McStasInstrument) + + # Calculate parameters for data reduction + data_metadata = calculate_raw_data_metadata( + *detector_ids, wf=wf, logger=logger, chunk_size=chunk_size + ) + if logger is not None: + logger.info("Metadata retrieved: %s", data_metadata) + + toa_bin_edges = sc.linspace( + dim='t', start=data_metadata.min_toa, stop=data_metadata.max_toa, num=51 + ) + scale_factor = mcstas_weight_to_probability_scalefactor( + max_counts=wf.compute(MaximumCounts), + max_probability=data_metadata.max_probability, + ) + # Compute metadata and make the skeleton output file + experiment_metadata = wf.compute(NMXExperimentMetadata) + detector_metas = [] + for detector_i in range(3): + temp_wf = wf.copy() + temp_wf[DetectorIndex] = detector_i + detector_metas.append(temp_wf.compute(NMXDetectorMetadata)) + + if logger is not None: + logger.info("Exporting metadata into the output file %s", output_file) + + _export_static_metadata_as_nxlauetof( + experiment_metadata=experiment_metadata, + output_file=output_file, + # Arbitrary metadata falls into ``entry`` group as a variable. + mcstas_weight2count_scale_factor=scale_factor, + ) + _export_detector_metadata_as_nxlauetof(*detector_metas, output_file=output_file) + # Compute histogram + final_wf = wf.copy() + # Set the scale factor and time bin edges + final_wf[McStasWeight2CountScaleFactor] = scale_factor + final_wf[TimeBinSteps] = toa_bin_edges + + file_path = final_wf.compute(FilePath) + final_stream_processor = _build_final_streaming_processor_helper() + # Loop over the detectors + for detector_i in detector_ids: + temp_wf = final_wf.copy() + if isinstance(detector_i, str): + temp_wf[DetectorName] = detector_i + else: + temp_wf[DetectorIndex] = detector_i + # Set static information as parameters + detector_name = temp_wf.compute(DetectorName) + temp_wf[PixelIds] = temp_wf.compute(PixelIds) + max_chunk_id = calculate_number_of_chunks( + file_path, detector_name=detector_name, chunk_size=chunk_size + ) + + # Build the stream processor + processor = final_stream_processor(temp_wf) + for i_da, da in enumerate( + raw_event_data_chunk_generator( + file_path=file_path, detector_name=detector_name, chunk_size=chunk_size + ) + ): + if any(da.sizes.values()) == 0: + continue + else: + results = processor.add_chunk({RawEventProbability: da}) + if logger is not None: + logger.info( + "[{%s}/{%s}] Processed chunk for {%s}", + i_da + 1, + max_chunk_id, + detector_name, + ) + + result = results[NMXReducedDataGroup] + if logger is not None: + logger.info("Appending reduced data into the output file %s", output_file) + _export_reduced_data_as_nxlauetof(result, output_file=output_file) + + +def main() -> None: + parser = argparse.ArgumentParser(description="McStas Data Reduction.") + parser.add_argument( + "--input_file", type=str, help="Path to the input file", required=True + ) + parser.add_argument( + "--output_file", + type=str, + default="scipp_output.h5", + help="Path to the output file", + ) + parser.add_argument( + "--verbose", action="store_true", help="Increase output verbosity" + ) + parser.add_argument( + "--chunk_size", + type=int, + default=10_000_000, + help="Chunk size for processing", + ) + parser.add_argument( + "--detector_ids", + type=int, + nargs="+", + default=[0, 1, 2], + help="Detector indices to process", + ) + + args = parser.parse_args() + + input_file = pathlib.Path(args.input_file).resolve() + output_file = pathlib.Path(args.output_file).resolve() + + logger = logging.getLogger(__name__) + if args.verbose: + logger.setLevel(logging.INFO) + logger.addHandler(logging.StreamHandler(sys.stdout)) + + wf = McStasWorkflow() + reduction( + input_file=input_file, + output_file=output_file, + chunk_size=args.chunk_size, + detector_ids=args.detector_ids, + logger=logger, + wf=wf, + ) diff --git a/packages/essnmx/src/ess/nmx/mcstas/load.py b/packages/essnmx/src/ess/nmx/mcstas/load.py index 1e5f432f..186e671b 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/load.py +++ b/packages/essnmx/src/ess/nmx/mcstas/load.py @@ -49,13 +49,10 @@ def load_event_data_bank_name( def _exclude_zero_events(data: sc.Variable) -> sc.Variable: """Exclude events with zero counts from the data. - McStas can add an extra event line containing 0,0,0,0,0,0 - This line should not be included so we skip it. + McStas can add extra event lines containing 0,0,0,0,0,0 + These lines should not be included so we skip it. """ - if (data.values[0] == 0).all(): - data = data["event", 1:] - else: - data = data + data = data[(data != sc.scalar(0.0, unit=data.unit)).any(dim="dim_1")] return data @@ -295,11 +292,7 @@ def retrieve_raw_data_metadata( ) -> NMXRawDataMetadata: """Retrieve the metadata of the raw data.""" return NMXRawDataMetadata( - sc.DataGroup( - min_toa=min_toa, - max_toa=max_toa, - max_probability=max_probability, - ) + min_toa=min_toa, max_toa=max_toa, max_probability=max_probability ) diff --git a/packages/essnmx/src/ess/nmx/mtz_io.py b/packages/essnmx/src/ess/nmx/mtz_io.py index fbfb50e8..87ee79ec 100644 --- a/packages/essnmx/src/ess/nmx/mtz_io.py +++ b/packages/essnmx/src/ess/nmx/mtz_io.py @@ -17,7 +17,7 @@ """Path to the mtz file""" SpaceGroupDesc = NewType("SpaceGroupDesc", str) """The space group description. e.g. 'P 21 21 21'""" -DEFAULT_SPACE_GROUP_DESC = SpaceGroupDesc("P 21 21 21") +DEFAULT_SPACE_GROUP_DESC = SpaceGroupDesc("P 1") """The default space group description to use if not found in the mtz files.""" # Custom column names diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/nexus.py index bf6b509d..2a077f85 100644 --- a/packages/essnmx/src/ess/nmx/nexus.py +++ b/packages/essnmx/src/ess/nmx/nexus.py @@ -2,11 +2,30 @@ # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) import io import pathlib +import warnings +from collections.abc import Callable, Generator from functools import partial +from typing import Any, TypeVar +import bitshuffle.h5 import h5py +import numpy as np +import sciline as sl import scipp as sc +from .types import ( + DetectorIndex, + DetectorName, + FilePath, + NMXDetectorMetadata, + NMXExperimentMetadata, + NMXReducedDataGroup, +) + + +def _create_dataset_from_string(*, root_entry: h5py.Group, name: str, var: str) -> None: + root_entry.create_dataset(name, dtype=h5py.string_dtype(), data=var) + def _create_dataset_from_var( *, @@ -16,6 +35,8 @@ def _create_dataset_from_var( long_name: str | None = None, compression: str | None = None, compression_opts: int | None = None, + chunks: tuple[int, ...] | int | bool | None = None, + dtype: Any = None, ) -> h5py.Dataset: compression_options = {} if compression is not None: @@ -25,10 +46,12 @@ def _create_dataset_from_var( dataset = root_entry.create_dataset( name, - data=var.values, + data=var.values if dtype is None else var.values.astype(dtype, copy=False), + chunks=chunks, **compression_options, ) - dataset.attrs["units"] = str(var.unit) + if var.unit is not None: + dataset.attrs["units"] = str(var.unit) if long_name is not None: dataset.attrs["long_name"] = long_name return dataset @@ -36,9 +59,20 @@ def _create_dataset_from_var( _create_compressed_dataset = partial( _create_dataset_from_var, - compression="gzip", - compression_opts=4, + compression=bitshuffle.h5.H5FILTER, + compression_opts=(0, bitshuffle.h5.H5_COMPRESS_LZ4), ) +"""Create dataset with compression options. + +[``Bitshuffle/LZ4``](https://github.com/kiyo-masui/bitshuffle) is used for convenience. +Since ``Dectris`` uses it for their Nexus file compression, it is compatible with DIALS. +``Bitshuffle/LZ4`` tends to give similar results to +GZIP and other compression algorithms with better performance. +A naive implementation of bitshuffle/LZ4 compression, +shown in [issue #124](https://github.com/scipp/essnmx/issues/124), +led to 80% file reduction (365 MB vs 1.8 GB). + +""" def _create_root_data_entry(file_obj: h5py.File) -> h5py.Group: @@ -129,8 +163,6 @@ def export_as_nexus( Currently exporting step is not expected to be part of sciline pipelines. """ - import warnings - warnings.warn( DeprecationWarning( "Exporting to custom NeXus format will be deprecated in the near future." @@ -145,3 +177,383 @@ def export_as_nexus( _create_instrument_group(data, nx_entry) _create_detector_group(data, nx_entry) _create_source_group(data, nx_entry) + + +def _create_lauetof_data_entry(file_obj: h5py.File) -> h5py.Group: + nx_entry = file_obj.create_group("entry") + nx_entry.attrs["NX_class"] = "NXentry" + return nx_entry + + +def _add_lauetof_definition(nx_entry: h5py.Group) -> None: + _create_dataset_from_string(root_entry=nx_entry, name="definition", var="NXlauetof") + + +def _add_lauetof_instrument(nx_entry: h5py.Group) -> h5py.Group: + nx_instrument = nx_entry.create_group("instrument") + nx_instrument.attrs["NX_class"] = "NXinstrument" + _create_dataset_from_string(root_entry=nx_instrument, name="name", var="NMX") + return nx_instrument + + +def _add_lauetof_source_group( + dg: NMXExperimentMetadata, nx_instrument: h5py.Group +) -> None: + nx_source = nx_instrument.create_group("source") + nx_source.attrs["NX_class"] = "NXsource" + _create_dataset_from_string( + root_entry=nx_source, name="name", var="European Spallation Source" + ) + _create_dataset_from_string(root_entry=nx_source, name="short_name", var="ESS") + _create_dataset_from_string( + root_entry=nx_source, name="type", var="Spallation Neutron Source" + ) + _create_dataset_from_var( + root_entry=nx_source, name="distance", var=sc.norm(dg["source_position"]) + ) + # Legacy probe information. + _create_dataset_from_string(root_entry=nx_source, name="probe", var="neutron") + + +def _add_lauetof_detector_group(dg: sc.DataGroup, nx_instrument: h5py.Group) -> None: + nx_detector = nx_instrument.create_group(dg["detector_name"].value) # Detector name + nx_detector.attrs["NX_class"] = "NXdetector" + _create_dataset_from_var( + name="polar_angle", + root_entry=nx_detector, + var=sc.scalar(0, unit='deg'), # TODO: Add real data + ) + _create_dataset_from_var( + name="azimuthal_angle", + root_entry=nx_detector, + var=sc.scalar(0, unit='deg'), # TODO: Add real data + ) + _create_dataset_from_var( + name="x_pixel_size", root_entry=nx_detector, var=dg["x_pixel_size"] + ) + _create_dataset_from_var( + name="y_pixel_size", root_entry=nx_detector, var=dg["y_pixel_size"] + ) + _create_dataset_from_var( + name="distance", + root_entry=nx_detector, + var=sc.scalar(0, unit='m'), # TODO: Add real data + ) + # Legacy geometry information until we have a better way to store it + _create_dataset_from_var( + name="origin", root_entry=nx_detector, var=dg['origin_position'] + ) + # Fast axis, along where the pixel ID increases by 1 + _create_dataset_from_var( + root_entry=nx_detector, var=dg['fast_axis'], name="fast_axis" + ) + # Slow axis, along where the pixel ID increases + # by the number of pixels in the fast axis + _create_dataset_from_var( + root_entry=nx_detector, var=dg['slow_axis'], name="slow_axis" + ) + + +def _add_lauetof_sample_group(dg: NMXExperimentMetadata, nx_entry: h5py.Group) -> None: + nx_sample = nx_entry.create_group("sample") + nx_sample.attrs["NX_class"] = "NXsample" + _create_dataset_from_var( + root_entry=nx_sample, + var=dg['crystal_rotation'], + name='crystal_rotation', + long_name='crystal rotation in Phi (XYZ)', + ) + _create_dataset_from_string( + root_entry=nx_sample, + name='name', + var=dg['sample_name'].value, + ) + _create_dataset_from_var( + name='orientation_matrix', + root_entry=nx_sample, + var=sc.array( + dims=['i', 'j'], + values=[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], + unit="dimensionless", + ), # TODO: Add real data, the sample orientation matrix + ) + _create_dataset_from_var( + name='unit_cell', + root_entry=nx_sample, + var=sc.array( + dims=['i'], + values=[1.0, 1.0, 1.0, 90.0, 90.0, 90.0], + unit="dimensionless", # TODO: Add real data, + # a, b, c, alpha, beta, gamma + ), + ) + + +def _add_lauetof_monitor_group(data: sc.DataGroup, nx_entry: h5py.Group) -> None: + nx_monitor = nx_entry.create_group("control") + nx_monitor.attrs["NX_class"] = "NXmonitor" + _create_dataset_from_string(root_entry=nx_monitor, name='mode', var='monitor') + nx_monitor["preset"] = 0.0 # Check if this is the correct value + data_dset = _create_dataset_from_var( + name='data', + root_entry=nx_monitor, + var=sc.array( + dims=['tof'], values=[1, 1, 1], unit="counts" + ), # TODO: Add real data, bin values + ) + data_dset.attrs["signal"] = 1 + data_dset.attrs["primary"] = 1 + _create_dataset_from_var( + name='time_of_flight', + root_entry=nx_monitor, + var=sc.array( + dims=['tof'], values=[1, 1, 1], unit="s" + ), # TODO: Add real data, bin edges + ) + + +def _add_arbitrary_metadata( + nx_entry: h5py.Group, **arbitrary_metadata: sc.Variable +) -> None: + if not arbitrary_metadata: + return + + metadata_group = nx_entry.create_group("metadata") + for key, value in arbitrary_metadata.items(): + if not isinstance(value, sc.Variable): + import warnings + + msg = f"Skipping metadata key '{key}' as it is not a scipp.Variable." + warnings.warn(UserWarning(msg), stacklevel=2) + continue + else: + _create_dataset_from_var( + name=key, + root_entry=metadata_group, + var=value, + ) + + +def _export_static_metadata_as_nxlauetof( + experiment_metadata: NMXExperimentMetadata, + output_file: str | pathlib.Path | io.BytesIO, + **arbitrary_metadata: sc.Variable, +) -> None: + """Export the metadata to a NeXus file with the LAUE_TOF application definition. + + ``Metadata`` in this context refers to the information + that is not part of the reduced detector counts itself, + but is necessary for the interpretation of the reduced data. + Since NMX can have arbitrary number of detectors, + this function can take multiple detector metadata objects. + + Parameters + ---------- + experiment_metadata: + Experiment metadata object. + output_file: + Output file path. + arbitrary_metadata: + Arbitrary metadata that does not fit into the existing metadata objects. + + """ + with h5py.File(output_file, "w") as f: + f.attrs["NX_class"] = "NXlauetof" + nx_entry = _create_lauetof_data_entry(f) + _add_lauetof_definition(nx_entry) + _add_lauetof_sample_group(experiment_metadata, nx_entry) + nx_instrument = _add_lauetof_instrument(nx_entry) + _add_lauetof_source_group(experiment_metadata, nx_instrument) + # Placeholder for ``monitor`` group + _add_lauetof_monitor_group(experiment_metadata, nx_entry) + # Skipping ``NXdata``(name) field with data link + # Add arbitrary metadata + _add_arbitrary_metadata(nx_entry, **arbitrary_metadata) + + +def _export_detector_metadata_as_nxlauetof( + *detector_metadatas: NMXDetectorMetadata, + output_file: str | pathlib.Path | io.BytesIO, + append_mode: bool = True, +) -> None: + """Export the detector specific metadata to a NeXus file. + + Since NMX can have arbitrary number of detectors, + this function can take multiple detector metadata objects. + + Parameters + ---------- + detector_metadatas: + Detector metadata objects. + output_file: + Output file path. + + """ + + if not append_mode: + raise NotImplementedError("Only append mode is supported for now.") + + with h5py.File(output_file, "r+") as f: + nx_entry = f["entry"] + if "instrument" not in nx_entry: + nx_instrument = _add_lauetof_instrument(f["entry"]) + else: + nx_instrument = nx_entry["instrument"] + # Add detector group metadata + for detector_metadata in detector_metadatas: + _add_lauetof_detector_group(detector_metadata, nx_instrument) + + +def _export_reduced_data_as_nxlauetof( + dg: NMXReducedDataGroup, + output_file: str | pathlib.Path | io.BytesIO, + *, + append_mode: bool = True, + compress_counts: bool = True, +) -> None: + """Export the reduced data to a NeXus file with the LAUE_TOF application definition. + + Even though this function only exports + reduced data(detector counts and its coordinates), + the input should contain all the necessary metadata + for minimum sanity check. + + Parameters + ---------- + dg: + Reduced data and metadata. + output_file: + Output file path. + append_mode: + If ``True``, the file is opened in append mode. + If ``False``, the file is opened in None-append mode. + > None-append mode is not supported for now. + > Only append mode is supported for now. + compress_counts: + If ``True``, the detector counts are compressed using bitshuffle. + It is because only the detector counts are expected to be large. + + """ + if not append_mode: + raise NotImplementedError("Only append mode is supported for now.") + + with h5py.File(output_file, "r+") as f: + nx_detector: h5py.Group = f[f"entry/instrument/{dg['detector_name'].value}"] + # Data - shape: [n_x_pixels, n_y_pixels, n_tof_bins] + # The actual application definition defines it as integer, + # but we keep the original data type for now + num_x, num_y = dg["detector_shape"].value # Probably better way to do this + if compress_counts: + data_dset = _create_compressed_dataset( + name="data", + root_entry=nx_detector, + var=sc.fold( + dg['counts'].data, dim='id', sizes={'x': num_x, 'y': num_y} + ), + chunks=(num_x, num_y, 1), + dtype=np.uint, + ) + else: + data_dset = _create_dataset_from_var( + name="data", + root_entry=nx_detector, + var=sc.fold( + dg['counts'].data, dim='id', sizes={'x': num_x, 'y': num_y} + ), + dtype=np.uint, + ) + data_dset.attrs["signal"] = 1 + _create_dataset_from_var( + name='time_of_flight', + root_entry=nx_detector, + var=sc.midpoints(dg['counts'].coords['t'], dim='t'), + ) + + +def _check_file( + filename: str | pathlib.Path | io.BytesIO, overwrite: bool +) -> pathlib.Path | io.BytesIO: + if isinstance(filename, str | pathlib.Path): + filename = pathlib.Path(filename) + if filename.exists() and not overwrite: + raise FileExistsError( + f"File '{filename}' already exists. Use `overwrite=True` to overwrite." + ) + return filename + + +T = TypeVar("T", bound=sc.DataArray) + + +class NXLauetofWriter: + def __init__( + self, + *, + output_filename: str | pathlib.Path | io.BytesIO, + workflow: sl.Pipeline, + chunk_generator: Callable[[FilePath, DetectorName], Generator[T, None, None]], + chunk_insert_key: type[T], + extra_meta: dict[str, sc.Variable] | None = None, + compress_counts: bool = True, + overwrite: bool = False, + ) -> None: + from ess.reduce.streaming import EternalAccumulator, StreamProcessor + + from .types import FilePath, NMXReducedCounts + + self.compress_counts = compress_counts + self._chunk_generator = chunk_generator + self._chunk_insert_key = chunk_insert_key + self._workflow = workflow + self._output_filename = _check_file(output_filename, overwrite) + self._input_filename = workflow.compute(FilePath) + self._final_stream_processor = partial( + StreamProcessor, + dynamic_keys=(chunk_insert_key,), + target_keys=(NMXReducedDataGroup,), + accumulators={NMXReducedCounts: EternalAccumulator}, + ) + self._detector_metas: dict[DetectorName, NMXDetectorMetadata] = {} + self._detector_reduced: dict[DetectorName, NMXReducedDataGroup] = {} + _export_static_metadata_as_nxlauetof( + experiment_metadata=self._workflow.compute(NMXExperimentMetadata), + output_file=self._output_filename, + **(extra_meta or {}), + ) + + def add_panel( + self, *, detector_id: DetectorIndex | DetectorName + ) -> NMXReducedDataGroup: + from .types import PixelIds + + temp_wf = self._workflow.copy() + if isinstance(detector_id, int): + temp_wf[DetectorIndex] = detector_id + elif isinstance(detector_id, str): + temp_wf[DetectorName] = detector_id + else: + raise TypeError( + f"Expected detector_id to be an int or str, got {type(detector_id)}" + ) + + _export_detector_metadata_as_nxlauetof( + temp_wf.compute(NMXDetectorMetadata), + output_file=self._output_filename, + ) + # First compute static information + detector_name = temp_wf.compute(DetectorName) + temp_wf[PixelIds] = temp_wf.compute(PixelIds) + processor = self._final_stream_processor(temp_wf) + # Then iterate over the chunks + for da in self._chunk_generator(self._input_filename, detector_name): + if any(da.sizes.values()) == 0: + continue + else: + results = processor.add_chunk({self._chunk_insert_key: da}) + + _export_reduced_data_as_nxlauetof( + results[NMXReducedDataGroup], + self._output_filename, + compress_counts=self.compress_counts, + ) + return results[NMXReducedDataGroup] diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 100261b2..0d629021 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass from typing import Any, NewType import scipp as sc @@ -67,5 +68,11 @@ MaximumTimeOfArrival = NewType("MaximumTimeOfArrival", sc.Variable) """Maximum time of arrival of the raw data""" -NMXRawDataMetadata = NewType("NMXRawDataMetadata", sc.DataGroup) -"""Metadata of the raw data, i.e. maximum weight and min/max time of arrival""" + +@dataclass +class NMXRawDataMetadata: + """Metadata of the raw data, i.e. maximum weight and min/max time of arrival""" + + max_probability: MaximumProbability + min_toa: MinimumTimeOfArrival + max_toa: MaximumTimeOfArrival diff --git a/packages/essnmx/tests/mtz_io_test.py b/packages/essnmx/tests/mtz_io_test.py index 33981359..e585cbb6 100644 --- a/packages/essnmx/tests/mtz_io_test.py +++ b/packages/essnmx/tests/mtz_io_test.py @@ -9,7 +9,7 @@ from ess.nmx import mtz_io from ess.nmx.data import get_small_mtz_samples from ess.nmx.mtz_io import ( - DEFAULT_SPACE_GROUP_DESC, # P 21 21 21 + DEFAULT_SPACE_GROUP_DESC, # P 1 MtzDataFrame, MTZFileIndex, MTZFilePath, @@ -75,7 +75,7 @@ def mtz_list() -> list[gemmi.Mtz]: def test_get_space_group_with_spacegroup_desc() -> None: assert ( mtz_io.get_space_group_from_description(DEFAULT_SPACE_GROUP_DESC).short_name() - == "P212121" + == "P1" ) @@ -96,7 +96,7 @@ def conflicting_mtz_series( def test_get_unique_space_group_raises_on_conflict( conflicting_mtz_series: list[gemmi.Mtz], ) -> None: - reg = r"Multiple space groups found:.+P 21 21 21.+C 1 2 1" + reg = r"Multiple space groups found:.+P 1.+C 1 2 1" space_groups = [ mtz_io.get_space_group_from_mtz(mtz) for mtz in conflicting_mtz_series ] From 51ba45f0445cb5f688cb00bb861ee234aa0649c1 Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Thu, 27 Mar 2025 16:20:53 +0100 Subject: [PATCH 249/403] Work around bitshuffle if not available. (#130) * Work around bitshuffle if not available. * Update pytest to filter platform. --- packages/essnmx/src/ess/nmx/nexus.py | 58 ++++++++++++++++++-------- packages/essnmx/tests/exporter_test.py | 18 +++++++- 2 files changed, 58 insertions(+), 18 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/nexus.py index 2a077f85..42a11156 100644 --- a/packages/essnmx/src/ess/nmx/nexus.py +++ b/packages/essnmx/src/ess/nmx/nexus.py @@ -4,10 +4,9 @@ import pathlib import warnings from collections.abc import Callable, Generator -from functools import partial +from functools import partial, wraps from typing import Any, TypeVar -import bitshuffle.h5 import h5py import numpy as np import sciline as sl @@ -34,7 +33,7 @@ def _create_dataset_from_var( name: str, long_name: str | None = None, compression: str | None = None, - compression_opts: int | None = None, + compression_opts: int | tuple[int, int] | None = None, chunks: tuple[int, ...] | int | bool | None = None, dtype: Any = None, ) -> h5py.Dataset: @@ -57,22 +56,47 @@ def _create_dataset_from_var( return dataset -_create_compressed_dataset = partial( - _create_dataset_from_var, - compression=bitshuffle.h5.H5FILTER, - compression_opts=(0, bitshuffle.h5.H5_COMPRESS_LZ4), -) -"""Create dataset with compression options. +@wraps(_create_dataset_from_var) +def _create_compressed_dataset(*args, **kwargs): + """Create dataset with compression options. + + It will try to use ``bitshuffle`` for compression if available. + Otherwise, it will fall back to ``gzip`` compression. -[``Bitshuffle/LZ4``](https://github.com/kiyo-masui/bitshuffle) is used for convenience. -Since ``Dectris`` uses it for their Nexus file compression, it is compatible with DIALS. -``Bitshuffle/LZ4`` tends to give similar results to -GZIP and other compression algorithms with better performance. -A naive implementation of bitshuffle/LZ4 compression, -shown in [issue #124](https://github.com/scipp/essnmx/issues/124), -led to 80% file reduction (365 MB vs 1.8 GB). + [``Bitshuffle/LZ4``](https://github.com/kiyo-masui/bitshuffle) + is used for convenience. + Since ``Dectris`` uses it for their Nexus file compression, + it is compatible with DIALS. + ``Bitshuffle/LZ4`` tends to give similar results to + GZIP and other compression algorithms with better performance. + A naive implementation of bitshuffle/LZ4 compression, + shown in [issue #124](https://github.com/scipp/essnmx/issues/124), + led to 80% file reduction (365 MB vs 1.8 GB). -""" + """ + try: + import bitshuffle.h5 + + compression_filter = bitshuffle.h5.H5FILTER + default_compression_opts = (0, bitshuffle.h5.H5_COMPRESS_LZ4) + except ImportError: + warnings.warn( + UserWarning( + "Could not find the bitshuffle.h5 module from bitshuffle package. " + "The bitshuffle package is not installed or only partially installed. " + "Exporting to NeXus files with bitshuffle compression is not possible." + ), + stacklevel=2, + ) + compression_filter = "gzip" + default_compression_opts = 4 + + return _create_dataset_from_var( + *args, + **kwargs, + compression=compression_filter, + compression_opts=default_compression_opts, + ) def _create_root_data_entry(file_obj: h5py.File) -> h5py.Group: diff --git a/packages/essnmx/tests/exporter_test.py b/packages/essnmx/tests/exporter_test.py index 5b519c38..379dbe62 100644 --- a/packages/essnmx/tests/exporter_test.py +++ b/packages/essnmx/tests/exporter_test.py @@ -54,6 +54,14 @@ def reduced_data() -> NMXReducedDataGroup: ) +def _is_bitshuffle_available() -> bool: + import platform + + return not ( + platform.machine().startswith("arm") or platform.platform().startswith('win') + ) + + def test_mcstas_reduction_export_to_bytestream( reduced_data: NMXReducedDataGroup, ) -> None: @@ -75,7 +83,15 @@ def test_mcstas_reduction_export_to_bytestream( with pytest.warns( DeprecationWarning, match='Please use ``export_as_nxlauetof`` instead.' ): - export_as_nexus(reduced_data, bio) + if not _is_bitshuffle_available(): + # bitshuffle does not build correctly on Windows and ARM machines + # We are keeping this test here to catch when it builds correctly + # in the future. + with pytest.warns(UserWarning, match='bitshuffle.h5'): + export_as_nexus(reduced_data, bio) + else: + export_as_nexus(reduced_data, bio) + with h5py.File(bio, 'r') as f: assert 'NMX_data' in f nmx_data: h5py.Group = f.require_group('NMX_data') From 49dc7711aa6d2f47674f08829fc0deada44a05ce Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Tue, 1 Apr 2025 11:39:40 +0200 Subject: [PATCH 250/403] Allow -1 chunk size as a whole dataset size. (#132) * Work around bitshuffle if not available. * Update pytest to filter platform. * Whole chunk size * Check chunksize earlier. * Reuse userwarning lines. * Apply suggestions from code review Co-authored-by: Mridul Seth --------- Co-authored-by: Mridul Seth --- .../essnmx/src/ess/nmx/mcstas/executables.py | 2 +- packages/essnmx/src/ess/nmx/mcstas/load.py | 76 +++++++++++++------ packages/essnmx/src/ess/nmx/streaming.py | 17 ++++- packages/essnmx/tests/mcstas_io_test.py | 45 +++++++++++ 4 files changed, 115 insertions(+), 25 deletions(-) create mode 100644 packages/essnmx/tests/mcstas_io_test.py diff --git a/packages/essnmx/src/ess/nmx/mcstas/executables.py b/packages/essnmx/src/ess/nmx/mcstas/executables.py index 4e68d44e..205cbfb2 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/executables.py +++ b/packages/essnmx/src/ess/nmx/mcstas/executables.py @@ -245,7 +245,7 @@ def main() -> None: "--chunk_size", type=int, default=10_000_000, - help="Chunk size for processing", + help="Chunk size for processing. Pass -1 to process the whole file at once", ) parser.add_argument( "--detector_ids", diff --git a/packages/essnmx/src/ess/nmx/mcstas/load.py b/packages/essnmx/src/ess/nmx/mcstas/load.py index 186e671b..713f1d8a 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/load.py +++ b/packages/essnmx/src/ess/nmx/mcstas/load.py @@ -100,6 +100,35 @@ def load_raw_event_data( return _wrap_raw_event_data(data) +def _check_chunk_size(chunk_size: int) -> None: + if 0 < chunk_size < 10_000_000: + import warnings + + warnings.warn( + "The chunk size may be too small < 10_000_000.\n" + "Consider increasing the chunk size for better performance.\n" + "Hint: NMX typically expect ~10^8 bins as reduced data.", + UserWarning, + stacklevel=2, + ) + + +def _check_maximum_chunk_size(d_slices: tuple[slice, ...]) -> None: + """Check the maximum size of the slices.""" + max_chunk_size = max( + (d_slice.stop - d_slice.start) / d_slice.step for d_slice in d_slices + ) + _check_chunk_size(max_chunk_size) + + +def _validate_chunk_size(chunk_size: int) -> None: + """Validate the chunk size.""" + if not isinstance(chunk_size, int): + raise TypeError("Chunk size must be an integer.") + if chunk_size < -1: + raise ValueError("Invalid chunk size. It should be -1(for all) or > 0.") + + def raw_event_data_chunk_generator( file_path: FilePath, *, @@ -122,17 +151,23 @@ def raw_event_data_chunk_generator( If 0, chunk slice is determined automatically by the ``iter_chunks``. Note that it only works if the dataset is already chunked. - """ - if 0 < chunk_size < 10_000_000: - import warnings + Yields + ------ + RawEventProbability: + Data array containing the events of the detector. - warnings.warn( - "The chunk size may be too small < 10_000_000.\n" - "Consider increasing the chunk size for better performance.\n" - "Hint: NMX typically expect ~10^8 bins as reduced data.", - UserWarning, - stacklevel=2, - ) + Raises + ------ + ValueError: + If the chunk size is not valid. (>= -1) + TypeError: + If the chunk size is not an integer. + Warning + If the chunk size is too small (< 10_000_000). + + """ + _check_chunk_size(chunk_size) + _validate_chunk_size(chunk_size) # Find the data bank name associated with the detector bank_prefix = load_event_data_bank_name( @@ -147,21 +182,16 @@ def raw_event_data_chunk_generator( root = f["entry1/data"] dset = root[bank_name]["events"] if chunk_size == 0: - for data_slice in dset.dataset.iter_chunks(): - dim_0_slice, _ = data_slice # dim_0_slice, dim_1_slice + # dset.dataset.iter_chunks() yields (dim_0_slice, dim_1_slice) + dim_0_slices = tuple(dim0_sl for dim0_sl, _ in dset.dataset.iter_chunks()) + # Only checking maximum chunk size + # since the last chunk may be smaller than the rest of the chunks + _check_maximum_chunk_size(dim_0_slices) + for dim_0_slice in dim_0_slices: da = _wrap_raw_event_data(dset["dim_0", dim_0_slice]) - if da.sizes['event'] < 10_000_000: - import warnings - - warnings.warn( - "The chunk size may be too small < 10_000_000.\n" - "Consider increasing the chunk size for better performance.\n" - "Hint: NMX typically expect ~10^8 bins as reduced data.", - UserWarning, - stacklevel=2, - ) yield da - + elif chunk_size == -1: + yield _wrap_raw_event_data(dset[()]) else: num_events = dset.shape[0] for start in range(0, num_events, chunk_size): diff --git a/packages/essnmx/src/ess/nmx/streaming.py b/packages/essnmx/src/ess/nmx/streaming.py index ea08b673..153a01ff 100644 --- a/packages/essnmx/src/ess/nmx/streaming.py +++ b/packages/essnmx/src/ess/nmx/streaming.py @@ -7,7 +7,7 @@ from ess.reduce.streaming import Accumulator -from .mcstas.load import load_event_data_bank_name +from .mcstas.load import _validate_chunk_size, load_event_data_bank_name from .types import DetectorBankPrefix, DetectorName, FilePath @@ -71,7 +71,20 @@ def calculate_number_of_chunks( If 0, chunk slice is determined automatically by the ``iter_chunks``. Note that it only works if the dataset is already chunked. + Returns + ------- + : + Number of chunks in the event data. + + Raises + ------ + ValueError: + If the chunk size is not valid. (>= -1) + TypeError: + If the chunk size is not an integer. + """ + _validate_chunk_size(chunk_size) # Find the data bank name associated with the detector bank_prefix = load_event_data_bank_name( detector_name=detector_name, file_path=file_path @@ -85,5 +98,7 @@ def calculate_number_of_chunks( dset: snx.Field = root[bank_name]["events"] if chunk_size == 0: return len(list(dset.dataset.iter_chunks())) + elif chunk_size == -1: + return 1 # Read all at once else: return dset.shape[0] // chunk_size + int(dset.shape[0] % chunk_size != 0) diff --git a/packages/essnmx/tests/mcstas_io_test.py b/packages/essnmx/tests/mcstas_io_test.py new file mode 100644 index 00000000..bf193c5e --- /dev/null +++ b/packages/essnmx/tests/mcstas_io_test.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2025 Scipp contributors (https://github.com/scipp) +import pytest +import scipp as sc + +from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample +from ess.nmx.mcstas.load import load_raw_event_data, raw_event_data_chunk_generator + + +@pytest.fixture(params=[small_mcstas_2_sample, small_mcstas_3_sample]) +def mcstas_file_path( + request: pytest.FixtureRequest, mcstas_2_deprecation_warning_context +) -> str: + if request.param == small_mcstas_2_sample: + with mcstas_2_deprecation_warning_context(): + return request.param() + + return request.param() + + +def test_generator_loading_at_once(mcstas_file_path) -> None: + from ess.nmx.mcstas.load import detector_name_from_index + + detector_name = detector_name_from_index(0) + whole_chunk = next( + raw_event_data_chunk_generator( + mcstas_file_path, detector_name=detector_name, chunk_size=-1 + ) + ) + loaded_data = load_raw_event_data( + mcstas_file_path, detector_name=detector_name, bank_prefix=None + ) + assert sc.identical(whole_chunk, loaded_data) + + +def test_generator_loading_warns_if_too_small(mcstas_file_path) -> None: + from ess.nmx.mcstas.load import detector_name_from_index + + detector_name = detector_name_from_index(0) + with pytest.warns(UserWarning, match="The chunk size may be too small"): + next( + raw_event_data_chunk_generator( + mcstas_file_path, detector_name=detector_name, chunk_size=1 + ) + ) From 244fd0e29230b643d507b544e186373156d2b822 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Apr 2025 17:33:54 +0000 Subject: [PATCH 251/403] Bump scipp from 25.3.0 to 25.4.0 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 25.3.0 to 25.4.0. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/25.03.0...25.04.0) --- updated-dependencies: - dependency-name: scipp dependency-version: 25.4.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index b853d09b..188c9c53 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -117,7 +117,7 @@ sciline==24.10.0 # via # -r base.in # essreduce -scipp==25.3.0 +scipp==25.4.0 # via # -r base.in # essreduce From e449327cec86b144c02c72692c5b3422c1c44303 Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Mon, 7 Apr 2025 11:43:06 +0200 Subject: [PATCH 252/403] Update dependencies. --- packages/essnmx/pyproject.toml | 1 + packages/essnmx/requirements/base.in | 2 + packages/essnmx/requirements/base.txt | 48 +++++++++++++++-------- packages/essnmx/requirements/basetest.txt | 2 +- packages/essnmx/requirements/ci.txt | 10 ++--- packages/essnmx/requirements/dev.txt | 16 ++++---- packages/essnmx/requirements/docs.txt | 19 +++++---- packages/essnmx/requirements/mypy.txt | 3 ++ packages/essnmx/requirements/nightly.in | 2 + packages/essnmx/requirements/nightly.txt | 44 ++++++++++++++------- packages/essnmx/requirements/static.txt | 10 ++--- packages/essnmx/requirements/test.txt | 3 ++ 12 files changed, 101 insertions(+), 59 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index b1fda2b9..f7684abf 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -43,6 +43,7 @@ dependencies = [ "gemmi", "defusedxml", "bitshuffle", + "msgpack", ] dynamic = ["version"] diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index f7ca9a08..2dba136e 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -13,3 +13,5 @@ pooch pandas gemmi defusedxml +bitshuffle +msgpack diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 188c9c53..d50257bc 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:ce4455b8a5d721535a6305df40709de1d6c8e4c1 +# SHA1:e30515c3fb52510b8e8abf8e785cc372219f7527 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -7,6 +7,8 @@ # annotated-types==0.7.0 # via pydantic +bitshuffle==0.5.2 + # via -r base.in certifi==2025.1.31 # via requests charset-normalizer==3.4.1 @@ -21,7 +23,9 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -dask==2025.2.0 +cython==3.0.12 + # via bitshuffle +dask==2025.3.0 # via -r base.in defusedxml==0.7.1 # via -r base.in @@ -29,18 +33,19 @@ dnspython==2.7.0 # via email-validator email-validator==2.2.0 # via scippneutron -essreduce==25.3.0 +essreduce==25.3.1 # via -r base.in -fonttools==4.56.0 +fonttools==4.57.0 # via matplotlib -fsspec==2025.2.0 +fsspec==2025.3.2 # via dask -gemmi==0.7.0 +gemmi==0.7.1 # via -r base.in graphviz==0.20.3 # via -r base.in h5py==3.13.0 # via + # bitshuffle # scippneutron # scippnexus idna==3.10 @@ -63,10 +68,13 @@ matplotlib==3.10.1 # plopp mpltoolbox==24.5.1 # via scippneutron +msgpack==1.1.0 + # via -r base.in networkx==3.4.2 # via cyclebane -numpy==2.2.3 +numpy==2.2.4 # via + # bitshuffle # contourpy # h5py # matplotlib @@ -87,19 +95,19 @@ partd==1.4.2 # via dask pillow==11.1.0 # via matplotlib -platformdirs==4.3.6 +platformdirs==4.3.7 # via pooch -plopp==25.2.0 +plopp==25.3.0 # via # -r base.in # scippneutron pooch==1.8.2 # via -r base.in -pydantic==2.10.6 +pydantic==2.11.2 # via scippneutron -pydantic-core==2.27.2 +pydantic-core==2.33.1 # via pydantic -pyparsing==3.2.1 +pyparsing==3.2.3 # via matplotlib python-dateutil==2.9.0.post0 # via @@ -107,13 +115,13 @@ python-dateutil==2.9.0.post0 # pandas # scippneutron # scippnexus -pytz==2025.1 +pytz==2025.2 # via pandas pyyaml==6.0.2 # via dask requests==2.32.3 # via pooch -sciline==24.10.0 +sciline==25.4.0 # via # -r base.in # essreduce @@ -125,7 +133,7 @@ scipp==25.4.0 # scippnexus scippneutron==25.2.1 # via essreduce -scippnexus==24.11.1 +scippnexus==25.4.0 # via # -r base.in # essreduce @@ -140,13 +148,19 @@ toolz==1.0.0 # via # dask # partd -typing-extensions==4.12.2 +typing-extensions==4.13.1 # via # pydantic # pydantic-core -tzdata==2025.1 + # typing-inspection +typing-inspection==0.4.0 + # via pydantic +tzdata==2025.2 # via pandas urllib3==2.3.0 # via requests zipp==3.21.0 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt index 1d0e4fcf..76aebec7 100644 --- a/packages/essnmx/requirements/basetest.txt +++ b/packages/essnmx/requirements/basetest.txt @@ -7,7 +7,7 @@ # exceptiongroup==1.2.2 # via pytest -iniconfig==2.0.0 +iniconfig==2.1.0 # via pytest packaging==24.2 # via pytest diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index 1faaceaf..176b5124 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -17,7 +17,7 @@ colorama==0.4.6 # via tox distlib==0.3.9 # via virtualenv -filelock==3.17.0 +filelock==3.18.0 # via # tox # virtualenv @@ -32,7 +32,7 @@ packaging==24.2 # -r ci.in # pyproject-api # tox -platformdirs==4.3.6 +platformdirs==4.3.7 # via # tox # virtualenv @@ -48,11 +48,11 @@ tomli==2.2.1 # via # pyproject-api # tox -tox==4.24.1 +tox==4.25.0 # via -r ci.in -typing-extensions==4.12.2 +typing-extensions==4.13.1 # via tox urllib3==2.3.0 # via requests -virtualenv==20.29.2 +virtualenv==20.30.0 # via tox diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index b9014c97..091efdaf 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -12,7 +12,7 @@ -r static.txt -r test.txt -r wheels.txt -anyio==4.8.0 +anyio==4.9.0 # via # httpx # jupyter-server @@ -22,13 +22,13 @@ argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.3.0 # via isoduration -async-lru==2.0.4 +async-lru==2.0.5 # via jupyterlab cffi==1.17.1 # via argon2-cffi-bindings -copier==9.5.0 +copier==9.6.0 # via -r dev.in -dunamai==1.23.0 +dunamai==1.23.1 # via copier fqdn==1.5.1 # via jsonschema @@ -44,7 +44,7 @@ isoduration==20.11.0 # via jsonschema jinja2-ansible-filters==1.3.2 # via copier -json5==0.10.0 +json5==0.12.0 # via jupyterlab-server jsonpointer==3.0.0 # via jsonschema @@ -65,7 +65,7 @@ jupyter-server==2.15.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.3.5 +jupyterlab==4.4.0 # via -r dev.in jupyterlab-server==2.27.3 # via jupyterlab @@ -75,7 +75,7 @@ overrides==7.7.0 # via jupyter-server pathspec==0.12.1 # via copier -pip-compile-multi==2.7.1 +pip-compile-multi==2.8.0 # via -r dev.in pip-tools==7.4.1 # via pip-compile-multi @@ -85,7 +85,7 @@ prometheus-client==0.21.1 # via jupyter-server pycparser==2.22 # via cffi -python-json-logger==3.2.1 +python-json-logger==3.3.0 # via jupyter-events questionary==2.1.0 # via copier diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 48ccceb9..c1e66318 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -14,7 +14,7 @@ appnope==0.1.4 # via ipykernel asttokens==3.0.0 # via stack-data -attrs==25.1.0 +attrs==25.3.0 # via # jsonschema # referencing @@ -32,7 +32,7 @@ comm==0.2.2 # via # ipykernel # ipywidgets -debugpy==1.8.12 +debugpy==1.8.13 # via ipykernel decorator==5.2.1 # via ipython @@ -54,7 +54,7 @@ ipydatawidgets==4.3.5 # via pythreejs ipykernel==6.29.5 # via -r docs.in -ipython==8.33.0 +ipython==8.34.0 # via # -r docs.in # ipykernel @@ -65,7 +65,7 @@ ipywidgets==8.1.5 # pythreejs jedi==0.19.2 # via ipython -jinja2==3.1.5 +jinja2==3.1.6 # via # myst-parser # nbconvert @@ -106,7 +106,7 @@ mdit-py-plugins==0.4.2 # via myst-parser mdurl==0.1.2 # via markdown-it-py -mistune==3.1.2 +mistune==3.1.3 # via nbconvert myst-parser==4.0.1 # via -r docs.in @@ -119,7 +119,7 @@ nbformat==5.10.4 # nbclient # nbconvert # nbsphinx -nbsphinx==0.9.6 +nbsphinx==0.9.7 # via -r docs.in nest-asyncio==1.6.0 # via ipykernel @@ -148,7 +148,7 @@ pygments==2.19.1 # sphinx pythreejs==2.4.2 # via -r docs.in -pyzmq==26.2.1 +pyzmq==26.4.0 # via # ipykernel # jupyter-client @@ -156,7 +156,7 @@ referencing==0.36.2 # via # jsonschema # jsonschema-specifications -rpds-py==0.23.1 +rpds-py==0.24.0 # via # jsonschema # referencing @@ -226,3 +226,6 @@ webencodings==0.5.1 # tinycss2 widgetsnbextension==4.0.13 # via ipywidgets + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index 61d88db1..40e3f07e 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -10,3 +10,6 @@ mypy==1.15.0 # via -r mypy.in mypy-extensions==1.0.0 # via mypy + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index 5d1d38fd..c0bad079 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -8,6 +8,8 @@ pooch pandas gemmi defusedxml +bitshuffle +msgpack pytest scipp --index-url=https://pypi.anaconda.org/scipp-nightly-wheels/simple/ diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 69623e98..1b275814 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:707e21bd0906494a252089989bf9ab05df8a820e +# SHA1:894ecb8cad4ccdb545bbd81b2215094550fa9ccc # # This file is autogenerated by pip-compile-multi # To update, run: @@ -10,6 +10,8 @@ annotated-types==0.7.0 # via pydantic +bitshuffle==0.5.2 + # via -r nightly.in certifi==2025.1.31 # via requests charset-normalizer==3.4.1 @@ -24,7 +26,9 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -dask==2025.2.0 +cython==3.1.0b1 + # via bitshuffle +dask==2025.3.0 # via -r nightly.in defusedxml==0.8.0rc2 # via -r nightly.in @@ -32,20 +36,21 @@ dnspython==2.7.0 # via email-validator email-validator==2.2.0 # via scippneutron -essreduce==25.3.0 +essreduce==25.3.1 # via -r nightly.in exceptiongroup==1.2.2 # via pytest -fonttools==4.56.0 +fonttools==4.57.0 # via matplotlib -fsspec==2025.2.0 +fsspec==2025.3.2 # via dask -gemmi==0.7.0 +gemmi==0.7.1 # via -r nightly.in graphviz==0.20.3 # via -r nightly.in h5py==3.13.0 # via + # bitshuffle # scippneutron # scippnexus idna==3.10 @@ -54,7 +59,7 @@ idna==3.10 # requests importlib-metadata==8.6.1 # via dask -iniconfig==2.0.0 +iniconfig==2.1.0 # via pytest kiwisolver==1.4.8 # via matplotlib @@ -70,10 +75,13 @@ matplotlib==3.10.1 # plopp mpltoolbox==24.5.1 # via scippneutron +msgpack==1.1.0 + # via -r nightly.in networkx==3.4.2 # via cyclebane -numpy==2.2.3 +numpy==2.2.4 # via + # bitshuffle # contourpy # h5py # matplotlib @@ -95,7 +103,7 @@ partd==1.4.2 # via dask pillow==11.1.0 # via matplotlib -platformdirs==4.3.6 +platformdirs==4.3.7 # via pooch plopp @ git+https://github.com/scipp/plopp@main # via @@ -105,11 +113,11 @@ pluggy==1.5.0 # via pytest pooch==1.8.2 # via -r nightly.in -pydantic==2.11.0a2 +pydantic==2.11.2 # via scippneutron -pydantic-core==2.29.0 +pydantic-core==2.33.1 # via pydantic -pyparsing==3.2.1 +pyparsing==3.2.3 # via matplotlib pytest==8.3.5 # via -r nightly.in @@ -119,7 +127,7 @@ python-dateutil==2.9.0.post0 # pandas # scippneutron # scippnexus -pytz==2025.1 +pytz==2025.2 # via pandas pyyaml==6.0.2 # via dask @@ -154,13 +162,19 @@ toolz==1.0.0 # via # dask # partd -typing-extensions==4.12.2 +typing-extensions==4.13.1 # via # pydantic # pydantic-core -tzdata==2025.1 + # typing-inspection +typing-inspection==0.4.0 + # via pydantic +tzdata==2025.2 # via pandas urllib3==2.3.0 # via requests zipp==3.21.0 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index 098c4f5a..c0e73660 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -9,17 +9,17 @@ cfgv==3.4.0 # via pre-commit distlib==0.3.9 # via virtualenv -filelock==3.17.0 +filelock==3.18.0 # via virtualenv -identify==2.6.8 +identify==2.6.9 # via pre-commit nodeenv==1.9.1 # via pre-commit -platformdirs==4.3.6 +platformdirs==4.3.7 # via virtualenv -pre-commit==4.1.0 +pre-commit==4.2.0 # via -r static.in pyyaml==6.0.2 # via pre-commit -virtualenv==20.29.2 +virtualenv==20.30.0 # via pre-commit diff --git a/packages/essnmx/requirements/test.txt b/packages/essnmx/requirements/test.txt index 3c7454d8..7e9f65fe 100644 --- a/packages/essnmx/requirements/test.txt +++ b/packages/essnmx/requirements/test.txt @@ -7,3 +7,6 @@ # -r base.txt -r basetest.txt + +# The following packages are considered to be unsafe in a requirements file: +# setuptools From b1b49999c22d6f84dc40f3e0e5c525a8a837b004 Mon Sep 17 00:00:00 2001 From: Aaron Finke <45569605+aaronfinke@users.noreply.github.com> Date: Mon, 7 Apr 2025 13:15:49 +0200 Subject: [PATCH 253/403] Scaling routine using dials file and update executable (#134) * dials reflections IO for scaling * dials reflections IO for scaling * temporary changes to deal with trimmed mcstas files and support for input toa_min_max_prob * load_reflection_file in dials_io * Apply automatic formatting * Fix ruff. (#137) * Temporarily remove a documentation page until we add test datasets. --------- Co-authored-by: YooSunyoung Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Aaron Finke --- .../essnmx/docs/user-guide/workflow.ipynb | 4 +- packages/essnmx/src/ess/nmx/dials_io.py | 319 ++++++++++++++++++ .../essnmx/src/ess/nmx/dials_reflection_io.py | 255 ++++++++++++++ .../essnmx/src/ess/nmx/mcstas/executables.py | 59 +++- packages/essnmx/src/ess/nmx/mcstas/load.py | 41 ++- packages/essnmx/src/ess/nmx/scaling.py | 2 +- 6 files changed, 651 insertions(+), 29 deletions(-) create mode 100644 packages/essnmx/src/ess/nmx/dials_io.py create mode 100644 packages/essnmx/src/ess/nmx/dials_reflection_io.py diff --git a/packages/essnmx/docs/user-guide/workflow.ipynb b/packages/essnmx/docs/user-guide/workflow.ipynb index 6fb39732..833c1fdb 100644 --- a/packages/essnmx/docs/user-guide/workflow.ipynb +++ b/packages/essnmx/docs/user-guide/workflow.ipynb @@ -170,7 +170,7 @@ ], "metadata": { "kernelspec": { - "display_name": "nmx-dev-310", + "display_name": "scipp", "language": "python", "name": "python3" }, @@ -184,7 +184,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.13.0" } }, "nbformat": 4, diff --git a/packages/essnmx/src/ess/nmx/dials_io.py b/packages/essnmx/src/ess/nmx/dials_io.py new file mode 100644 index 00000000..570516e0 --- /dev/null +++ b/packages/essnmx/src/ess/nmx/dials_io.py @@ -0,0 +1,319 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2024 Scipp contributors (https://github.com/scipp) +import json +import pathlib +from typing import NewType + +import gemmi +import numpy as np +import pandas as pd +import scipp as sc + +from .dials_reflection_io import load_reflection_file +from .mtz_io import NMXMtzDataArray + +# User defined or configurable types +DialsReflectionFilePath = NewType("DialsReflectionFilePath", pathlib.Path) +"""Path to the dials reflection file""" +DialsReflections = NewType("DialsReflections", dict) +"""The raw DIALS reflection file, read in as a dict""" +DialsExperimentFilePath = NewType("DialsExperimentFilePath", pathlib.Path) +"""Path to the dials experiment file""" +DialsExperiment = NewType("DialsExperiment", dict) +"""Experiment details from DIALS .expt file (JSON format)""" +SpaceGroupDesc = NewType("SpaceGroupDesc", str) +"""The space group description. e.g. 'P 21 21 21'""" +DEFAULT_SPACE_GROUP_DESC = SpaceGroupDesc("P 1") +"""The default space group description to use if not found in the input files.""" +UnitCell = NewType("UnitCell", tuple[float]) +"""The unit cell a, b, c in Angstrom, alpha, beta, gamma in degrees""" + +# Custom column names +WavelengthColumnName = NewType("WavelengthColumnName", str) +"""The name of the wavelength column in the DIALS reflection file.""" +DEFAULT_WAVELENGTH_COLUMN_NAME = WavelengthColumnName("LAMBDA") + +IntensityColumnName = NewType("IntensityColumnName", str) +"""The name of the intensity column in the DIALS reflection file.""" +DEFAULT_INTENSITY_COLUMN_NAME = IntensityColumnName("I") + +VarianceColumnName = NewType("VarianceColumnName", str) +"""The name of the variance (stdev(I)**2) of intensity column in the DIALS reflection +file.""" +DEFAULT_VARIANCE_COLUMN_NAME = VarianceColumnName("VARI") + +StdDevColumnName = NewType("StdDevColumnName", str) +"""The name of the standard deviation of intensity column in the DIALS reflection +file.""" +DEFAULT_STDEV_COLUMN_NAME = VarianceColumnName("SIGI") + + +# Computed types +DialsDataFrame = NewType("DialsDataFrame", pd.DataFrame) +"""The raw mtz dataframe.""" +NMXDialsDataFrame = NewType("NMXDialsDataFrame", pd.DataFrame) +"""The processed mtz dataframe with derived columns.""" +NMXDialsDataArray = NewType("NMXDialsDataArray", sc.DataArray) + + +def read_dials_reflection_file(file_path: DialsReflectionFilePath) -> DialsReflections: + """read dials reflection file""" + + return DialsReflections(load_reflection_file(file_path.as_posix(), copy=True)) + + +def read_dials_experiment_file(file_path: DialsExperimentFilePath) -> DialsExperiment: + """Read Dials Experiment .expt file""" + with open(file_path) as fp: + json_data = json.load(fp) + return DialsExperiment(json_data) + + +def get_unit_cell(dials_expt: DialsExperiment) -> UnitCell: + """ + Get the unit cell from the expt file. + It is saved as real-space vectors so the unit cell has to be + calculated from them. + """ + crystal = dials_expt['crystal'][0] + ra, rb, rc = tuple([crystal[f'real_space_{x}'] for x in 'abc']) + a = np.linalg.norm(ra) + b = np.linalg.norm(rb) + c = np.linalg.norm(rc) + al = np.rad2deg(np.arccos(np.dot(rb, rc) / (b * c))) + be = np.rad2deg(np.arccos(np.dot(ra, rc) / (a * c))) + ga = np.rad2deg(np.arccos(np.dot(ra, rb) / (a * b))) + + return UnitCell(a, b, c, al, be, ga) + + +def get_space_group(dials_expt: DialsExperiment) -> gemmi.SpaceGroup: + """ + Get space group from Dials expt file. + For cctbx/disambiguation reasons it is saved as the Hall symbol, but + the H-M notation can be back-determined with gemmi. + """ + crystal = dials_expt['crystal'][0] + sg_hall = crystal['space_group_hall_symbol'] + + return gemmi.find_spacegroup_by_ops(gemmi.symops_from_hall(sg_hall)) + + +def get_reciprocal_asu(spacegroup: gemmi.SpaceGroup) -> gemmi.ReciprocalAsu: + """Returns the reciprocal asymmetric unit from the space group.""" + + return gemmi.ReciprocalAsu(spacegroup) + + +def dials_refl_to_pandas(refls: DialsReflections) -> pd.DataFrame: + """Converts the loaded DIALS reflection file to a pandas dataframe. + + It is equivalent to the following code: + + .. code-block:: python + + import numpy as np + import pandas as pd + + data = np.array(mtz, copy=False) + columns = mtz.column_labels() + return pd.DataFrame(data, columns=columns) + + It is recommended in the gemmi documentation. + + """ + if refls.get('experiment_identifier'): # this has no relevant information + del refls['experiment_identifier'] # and it complicates loading as a df + df = pd.DataFrame( + { + key: list(val) if isinstance(val, np.ndarray) and val.ndim > 1 else val + for key, val in refls.items() + } + ) + for col in df.select_dtypes('uint64'): + df[col] = df[col].astype('int64') + return df + + +def process_dials_refl_list_to_dataframe( + refls: DialsReflections, +) -> DialsDataFrame: + """Select and derive columns from the original ``MtzDataFrame``. + + Parameters + ---------- + mtz: + The raw mtz dataset. + + wavelength_column_name: + The name of the wavelength column in the mtz file. + + intensity_column_name: + The name of the intensity column in the mtz file. + + intensity_sig_col_name: + The name of the standard uncertainty of intensity column in the mtz file. + + Returns + ------- + : + The new mtz dataframe with derived and renamed columns. + + The derived columns are: + + - ``SIGI``: The uncertainty of the intensity value, defined as the square root + of the measured variance. + + For consistent names of columns/coordinates, the following columns are renamed: + + - ``wavelength_column_name`` -> ``'wavelength'`` + - ``intensity_column_name`` -> ``'I'`` + - ``intensity_sig_col_name`` -> ``'SIGI'`` + + Other columns are kept as they are. + + """ + from .dials_io import dials_refl_to_pandas + + orig_df = dials_refl_to_pandas(refls) + new_df = pd.DataFrame() + + new_df['H'] = orig_df['miller_index'].map(lambda x: x[0]).astype(int) + new_df['K'] = orig_df['miller_index'].map(lambda x: x[1]).astype(int) + new_df['L'] = orig_df['miller_index'].map(lambda x: x[2]).astype(int) + + new_df['hkl'] = orig_df['miller_index'] + new_df["d"] = orig_df['d'] + new_df['wavelength'] = orig_df['wavelength_cal'] + + new_df[DEFAULT_INTENSITY_COLUMN_NAME] = orig_df['intensity.sum.value'] + new_df[DEFAULT_VARIANCE_COLUMN_NAME] = orig_df['intensity.sum.variance'] + new_df[DEFAULT_STDEV_COLUMN_NAME] = np.sqrt(orig_df['intensity.sum.variance']) + + return DialsDataFrame(new_df) + + +def process_dials_dataframe( + *, + dials_df: DialsDataFrame, + reciprocal_asu: gemmi.ReciprocalAsu, + sg: gemmi.SpaceGroup, +) -> NMXDialsDataFrame: + """Modify/Add columns of the shallow copy of a dials dataframe. + + This method must be called after merging multiple mtz dataframe. + """ + + df = dials_df.copy(deep=False) + + def _reciprocal_asu(row: pd.Series) -> list[int]: + """Converts miller indices(HKL) to ASU indices.""" + + return reciprocal_asu.to_asu(row["hkl"], sg.operations())[0] + + df["hkl_asu"] = df.apply(_reciprocal_asu, axis=1) + # Unpack the indices for later. + df[["H_ASU", "K_ASU", "L_ASU"]] = pd.DataFrame( + df["hkl_asu"].to_list(), index=df.index + ) + + return NMXDialsDataFrame(df) + + +def nmx_dials_dataframe_to_scipp_dataarray( + nmx_mtz_df: NMXDialsDataFrame, +) -> NMXMtzDataArray: + """Converts the processed mtz dataframe to a scipp dataarray. + + The intensity, with column name :attr:`~DEFAULT_INTENSITY_COLUMN_NAME` + becomes the data and the standard uncertainty of intensity, + with column name :attr:`~DEFAULT_SIGMA_INTENSITY_COLUMN_NAME` + becomes the variances of the data. + + Parameters + ---------- + nmx_mtz_df: + The merged and processed mtz dataframe. + + Returns + ------- + : + The scipp dataarray with the intensity and variances. + The ``I`` column becomes the data and the + squared ``SIGI`` column becomes the variances. + Therefore they are not in the coordinates. + + Following coordinates are modified: + + - ``hkl``: The miller indices as a string. + It is modified to have a string dtype + since is no dtype that can represent this in scipp. + + - ``hkl_asu``: The asymmetric unit of miller indices as a string. + This coordinate will be used to derive estimated scale factors. + It is modified to have a string dtype + as the same reason as why ``hkl`` coordinate is modified. + + Zero or negative intensities are removed from the dataarray. + It can happen due to the post-processing of the data, + e.g. background subtraction. + + """ + from scipp.compat.pandas_compat import from_pandas_dataframe, parse_bracket_header + + to_scipp = nmx_mtz_df.copy(deep=False) + # Convert to scipp Dataset + nmx_mtz_ds = from_pandas_dataframe( + to_scipp, + data_columns=[ + DEFAULT_INTENSITY_COLUMN_NAME, + DEFAULT_STDEV_COLUMN_NAME, + DEFAULT_VARIANCE_COLUMN_NAME, + ], + header_parser=parse_bracket_header, + ) + # Pop the indices columns. + # TODO: We can put them back once we support tuple[int] dtype. + # See https://github.com/scipp/scipp/issues/3046 for more details. + # Temporarily, we will manually convert them to a string. + # It is done on the scipp variable instead of the dataframe + # since columns with string dtype are converted to PyObject dtype + # instead of string by `from_pandas_dataframe`. + for indices_name in ("hkl", "hkl_asu"): + nmx_mtz_ds.coords[indices_name] = sc.array( + dims=nmx_mtz_ds.coords[indices_name].dims, + values=nmx_mtz_df[indices_name].astype(str).tolist(), + # `astype`` is not enough to convert the dtype to string. + # The result of `astype` will have `PyObject` as a dtype. + ) + # Add units + nmx_mtz_ds.coords["wavelength"].unit = sc.units.angstrom + for key in nmx_mtz_ds.keys(): + nmx_mtz_ds[key].unit = sc.units.dimensionless + + # Add variances + nmx_mtz_da = nmx_mtz_ds[DEFAULT_INTENSITY_COLUMN_NAME].copy(deep=False) + nmx_mtz_da.variances = nmx_mtz_ds[DEFAULT_VARIANCE_COLUMN_NAME].values + + # Return DataArray without negative intensities + return NMXMtzDataArray(nmx_mtz_da[nmx_mtz_da.data > 0]) + + +providers = ( + read_dials_reflection_file, + read_dials_experiment_file, + process_dials_refl_list_to_dataframe, + process_dials_dataframe, + get_space_group, + get_reciprocal_asu, + process_dials_dataframe, + nmx_dials_dataframe_to_scipp_dataarray, +) +"""The providers related to the Dials IO.""" + +default_parameters = { + WavelengthColumnName: DEFAULT_WAVELENGTH_COLUMN_NAME, + IntensityColumnName: DEFAULT_INTENSITY_COLUMN_NAME, + StdDevColumnName: DEFAULT_STDEV_COLUMN_NAME, +} +"""The parameters related to the Dials IO.""" diff --git a/packages/essnmx/src/ess/nmx/dials_reflection_io.py b/packages/essnmx/src/ess/nmx/dials_reflection_io.py new file mode 100644 index 00000000..971f34f8 --- /dev/null +++ b/packages/essnmx/src/ess/nmx/dials_reflection_io.py @@ -0,0 +1,255 @@ +""" +DIALS .refl file loader + +This loads msgpack-type DIALS reflection files, without having DIALS or +cctbx in the python environment. + +Note: All modern .refl files are at time of writing msgpack-based. Some +much older files might be in pickle format, which this doesn't read. + +Adapted from Nick Cavendish of the DIALS team. +""" + +import functools +import logging +import operator +import os +import struct +from collections.abc import Iterable +from dataclasses import dataclass +from io import BytesIO +from pathlib import Path +from typing import IO, cast + +import msgpack +import numpy as np + + +@dataclass +class Shoebox: + panel: int + bbox: tuple[int] + data: np.array = None + mask: np.array = None + background: np.array = None + + +def _decode_raw_numpy(dtype, shape: int | Iterable = 1): + """ + Decoding a column that maps straight to a numpy array. + + Args: + dtype: The numpy dtype for the array + shape: + The shape of a single item. Either an int, or a collection + of ints, in C-array order (row major) + """ + # Convert to a shape tuple + if isinstance(shape, int): + shape = (shape,) + else: + shape = tuple(shape) + + def _decode_specific(data, copy): + num_items, raw = data + array = np.frombuffer(raw, dtype=dtype) + + if shape != (1,): + item_width = functools.reduce(operator.mul, shape) + if (len(raw) % item_width) != 0: + raise AssertionError( + "Data length %s is not divisible by item width %s", + len(raw), + item_width, + ) + elif (num_items * item_width) != len(array): + raise AssertionError( + "Data length %s is not equal to " + "number of items %s times item width %s", + len(array), + num_items, + item_width, + ) + array = array.reshape(num_items, *shape) + if copy: + return np.copy(array) + return array + + return _decode_specific + + +def _decode_shoeboxes(data: list, copy) -> list[Shoebox | None]: + # Shoebox is float + num_items, raw = data + shoeboxes: list[Shoebox | None] = [] + pos = 0 + while pos < len(raw): + sbox_header_fmt = "": _decode_raw_numpy(np.double, shape=3), + "cctbx::miller::index<>": _decode_raw_numpy(np.int32, shape=3), + "Shoebox<>": _decode_shoeboxes, + "vec2": _decode_raw_numpy(np.double, shape=2), + "mat3": _decode_raw_numpy(np.double, shape=(3, 3)), + # "std::string": _decode_wip, # - string writing broken; dials/dials#1858 +} + + +def decode_column(column_entry, copy): + """Decode a single column value""" + datatype, data = column_entry + + converter = _reftable_decoders.get(datatype) + if not converter: + logging.warning( + "Data type '%s' does not have a converter; cannot read", datatype + ) + return None + return converter(data, copy=copy) + + +def _get_unpacked(stream_or_path: str | IO | bytes | os.PathLike): + """Works out the logic to pass a stream/pathlike to msgpack""" + try: + logging.INFO(type(stream_or_path)) + path = os.fspath(cast(str, stream_or_path)) + is_fspathlike = True + except (TypeError, ValueError): + path = stream_or_path + is_fspathlike = isinstance(stream_or_path, str) + + if is_fspathlike: + with open(path, "rb") as f: + un = msgpack.Unpacker(f, strict_map_key=False) + return un.unpack() + else: + un = msgpack.Unpacker(stream_or_path, strict_map_key=False) + return un.unpack() + + +def loads(data: bytes, copy=False): + """ + Load a DIALS msgpack-encoded .refl file. + + Args: + data: bytes data, already read from the file. + copy: Should the data be copied into writable numpy arrays. + + Returns: See .load(stream_or_path) + """ + return load_reflection_file(BytesIO(data), copy) + + +def load_reflection_file(stream_or_path: IO | Path | os.PathLike, copy=False) -> dict: + """ + Load a DIALS msgpack-encoded .refl file + + Args: + stream_or_path: The filename or data to load + copy: + Should the data be copied. This will cause more memory usage + whilst loading the raw data. + + Returns: + + A dictionary with each column in the reflection table. If there + is an identifier mapping as part of the reflection table, then + this is returned as an extra 'experiment_identifier' column. + All columns except Shoeboxes are returned as numpy arrays, + except Shoebox columns, which are returned as Dataclass objects + which contain the portions of data from the file. + + With copy=False, all numpy arrays are pointing against the raw + memory returned by msgpack, which means they are read-only. + With copy=True, an immediate copy is done. This causes memory + usage to double while loading, but the created numpy arrays own + their own memory. + """ + root_data = _get_unpacked(stream_or_path) + + if not root_data[0] == "dials::af::reflection_table": + raise ValueError("Does not appear to be a dials reflection table file") + if not root_data[1] == 1: + raise ValueError( + f"reflection_table data is version {root_data[1]}. " + "Only Version 1 is understood" + ) + refdata = root_data[2] + + rows = refdata["nrows"] + identifiers = refdata["identifiers"] + data = refdata["data"] + + decoded_data = { + name: decode_column(value, copy=copy) for name, value in data.items() + } + + # Filter out empty (unknown) columns + decoded_data = {k: v for k, v in decoded_data.items() if v is not None} + + # Cross-check the columns are the expected lengths + for name, column in decoded_data.items(): + if len(column) != rows: + logging.warning( + "Warning: Mismatch of column lengths: %s is %s instead of expected %s", + name, + len(column), + rows, + ) + + # Make an "identifiers" column + if "id" in decoded_data and identifiers: + decoded_data["experiment_identifier"] = [ + identifiers[x] for x in decoded_data["id"] if x > 0 + ] + + return decoded_data diff --git a/packages/essnmx/src/ess/nmx/mcstas/executables.py b/packages/essnmx/src/ess/nmx/mcstas/executables.py index 205cbfb2..08b50c7d 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/executables.py +++ b/packages/essnmx/src/ess/nmx/mcstas/executables.py @@ -140,28 +140,43 @@ def reduction( output_file: pathlib.Path, chunk_size: int = 10_000_000, detector_ids: list[int | str], + compression: bool = True, wf: sl.Pipeline | None = None, logger: logging.Logger | None = None, + toa_min_max_prob: tuple[float] | None = None, ) -> None: wf = wf.copy() if wf is not None else McStasWorkflow() wf[FilePath] = input_file # Set static info wf[McStasInstrument] = wf.compute(McStasInstrument) - # Calculate parameters for data reduction - data_metadata = calculate_raw_data_metadata( - *detector_ids, wf=wf, logger=logger, chunk_size=chunk_size - ) - if logger is not None: - logger.info("Metadata retrieved: %s", data_metadata) + if not toa_min_max_prob: + # Calculate parameters for data reduction + data_metadata = calculate_raw_data_metadata( + *detector_ids, wf=wf, logger=logger, chunk_size=chunk_size + ) + if logger is not None: + logger.info("Metadata retrieved: %s", data_metadata) + + toa_bin_edges = sc.linspace( + dim='t', start=data_metadata.min_toa, stop=data_metadata.max_toa, num=51 + ) + scale_factor = mcstas_weight_to_probability_scalefactor( + max_counts=wf.compute(MaximumCounts), + max_probability=data_metadata.max_probability, + ) + else: + if logger is not None: + logger.info("Metadata given: %s", toa_min_max_prob) + toa_min = sc.scalar(toa_min_max_prob[0], unit='s') + toa_max = sc.scalar(toa_min_max_prob[1], unit='s') + prob_max = sc.scalar(toa_min_max_prob[2]) + toa_bin_edges = sc.linspace(dim='t', start=toa_min, stop=toa_max, num=51) + scale_factor = mcstas_weight_to_probability_scalefactor( + max_counts=wf.compute(MaximumCounts), + max_probability=prob_max, + ) - toa_bin_edges = sc.linspace( - dim='t', start=data_metadata.min_toa, stop=data_metadata.max_toa, num=51 - ) - scale_factor = mcstas_weight_to_probability_scalefactor( - max_counts=wf.compute(MaximumCounts), - max_probability=data_metadata.max_probability, - ) # Compute metadata and make the skeleton output file experiment_metadata = wf.compute(NMXExperimentMetadata) detector_metas = [] @@ -189,6 +204,7 @@ def reduction( file_path = final_wf.compute(FilePath) final_stream_processor = _build_final_streaming_processor_helper() # Loop over the detectors + result_list = [] for detector_i in detector_ids: temp_wf = final_wf.copy() if isinstance(detector_i, str): @@ -222,9 +238,15 @@ def reduction( ) result = results[NMXReducedDataGroup] + result_list.append(result) if logger is not None: logger.info("Appending reduced data into the output file %s", output_file) - _export_reduced_data_as_nxlauetof(result, output_file=output_file) + _export_reduced_data_as_nxlauetof( + result, output_file=output_file, compress_counts=compression + ) + from ess.nmx.reduction import merge_panels + + return merge_panels(*result_list) def main() -> None: @@ -245,7 +267,7 @@ def main() -> None: "--chunk_size", type=int, default=10_000_000, - help="Chunk size for processing. Pass -1 to process the whole file at once", + help="Chunk size for processing", ) parser.add_argument( "--detector_ids", @@ -254,6 +276,12 @@ def main() -> None: default=[0, 1, 2], help="Detector indices to process", ) + parser.add_argument( + "--compression", + type=bool, + default=True, + help="Compress reduced output with bitshuffle/lz4", + ) args = parser.parse_args() @@ -271,6 +299,7 @@ def main() -> None: output_file=output_file, chunk_size=args.chunk_size, detector_ids=args.detector_ids, + compression=args.compression, logger=logger, wf=wf, ) diff --git a/packages/essnmx/src/ess/nmx/mcstas/load.py b/packages/essnmx/src/ess/nmx/mcstas/load.py index 713f1d8a..477a2adc 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/load.py +++ b/packages/essnmx/src/ess/nmx/mcstas/load.py @@ -59,18 +59,37 @@ def _exclude_zero_events(data: sc.Variable) -> sc.Variable: def _wrap_raw_event_data(data: sc.Variable) -> RawEventProbability: data = data.rename_dims({'dim_0': 'event'}) data = _exclude_zero_events(data) - event_da = sc.DataArray( - coords={ - 'id': sc.array( - dims=['event'], - values=data['dim_1', 4].values, - dtype='int64', - unit=None, + try: + event_da = sc.DataArray( + coords={ + 'id': sc.array( + dims=['event'], + values=data['dim_1', 4].values, + dtype='int64', + unit=None, + ), + 't': sc.array(dims=['event'], values=data['dim_1', 5].values, unit='s'), + }, + data=sc.array( + dims=['event'], values=data['dim_1', 0].values, unit='counts' ), - 't': sc.array(dims=['event'], values=data['dim_1', 5].values, unit='s'), - }, - data=sc.array(dims=['event'], values=data['dim_1', 0].values, unit='counts'), - ) + ) + except IndexError: + event_da = sc.DataArray( + coords={ + 'id': sc.array( + dims=['event'], + values=data['dim_1', 1].values, + dtype='int64', + unit=None, + ), + 't': sc.array(dims=['event'], values=data['dim_1', 2].values, unit='s'), + }, + data=sc.array( + dims=['event'], values=data['dim_1', 0].values, unit='counts' + ), + ) + return RawEventProbability(event_da) diff --git a/packages/essnmx/src/ess/nmx/scaling.py b/packages/essnmx/src/ess/nmx/scaling.py index c76ae0d8..97ae4174 100644 --- a/packages/essnmx/src/ess/nmx/scaling.py +++ b/packages/essnmx/src/ess/nmx/scaling.py @@ -455,7 +455,7 @@ def calculate_wavelength_scale_factor( """Providers for scaling data.""" default_parameters = { - WavelengthBins: sc.linspace("wavelength", 2.6, 3.6, 250, unit="angstrom"), + WavelengthBins: sc.linspace("wavelength", 1.8, 3.5, 250, unit="angstrom"), ScaledIntensityLeftTailThreshold: DEFAULT_LEFT_TAIL_THRESHOLD, ScaledIntensityRightTailThreshold: DEFAULT_RIGHT_TAIL_THRESHOLD, WavelengthFittingPolynomialDegree: WavelengthFittingPolynomialDegree(7), From c4045e473173cf2d4a4ec9efba70e78f367d3bb8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 May 2025 17:55:01 +0000 Subject: [PATCH 254/403] Bump scipp from 25.4.0 to 25.5.0 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 25.4.0 to 25.5.0. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/25.04.0...25.05.0) --- updated-dependencies: - dependency-name: scipp dependency-version: 25.5.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index d50257bc..d20cb627 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -125,7 +125,7 @@ sciline==25.4.0 # via # -r base.in # essreduce -scipp==25.4.0 +scipp==25.5.0 # via # -r base.in # essreduce From c46a7fad04f027271c996d6964e3f4155848e0a0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 28 May 2025 17:43:28 +0000 Subject: [PATCH 255/403] Bump scipp from 25.5.0 to 25.5.1 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 25.5.0 to 25.5.1. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/25.05.0...25.05.1) --- updated-dependencies: - dependency-name: scipp dependency-version: 25.5.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index d20cb627..84635591 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -125,7 +125,7 @@ sciline==25.4.0 # via # -r base.in # essreduce -scipp==25.5.0 +scipp==25.5.1 # via # -r base.in # essreduce From b10e3cb46fd1bde33d527814fea58e0f704e1d1d Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Fri, 4 Apr 2025 14:56:51 +0200 Subject: [PATCH 256/403] Checkout dial related modules from scaling_dials branch. --- packages/essnmx/src/ess/nmx/dials_io.py | 56 ++++++------------- .../essnmx/src/ess/nmx/dials_reflection_io.py | 49 +++++++++++++++- 2 files changed, 63 insertions(+), 42 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/dials_io.py b/packages/essnmx/src/ess/nmx/dials_io.py index 570516e0..d615f5b4 100644 --- a/packages/essnmx/src/ess/nmx/dials_io.py +++ b/packages/essnmx/src/ess/nmx/dials_io.py @@ -9,13 +9,12 @@ import pandas as pd import scipp as sc -from .dials_reflection_io import load_reflection_file -from .mtz_io import NMXMtzDataArray +from .dials_reflection_io import load # User defined or configurable types DialsReflectionFilePath = NewType("DialsReflectionFilePath", pathlib.Path) """Path to the dials reflection file""" -DialsReflections = NewType("DialsReflections", dict) +DialsReflectionFile = NewType("DialsReflectionFile", dict) """The raw DIALS reflection file, read in as a dict""" DialsExperimentFilePath = NewType("DialsExperimentFilePath", pathlib.Path) """Path to the dials experiment file""" @@ -56,17 +55,18 @@ NMXDialsDataArray = NewType("NMXDialsDataArray", sc.DataArray) -def read_dials_reflection_file(file_path: DialsReflectionFilePath) -> DialsReflections: +def read_dials_reflection_file( + file_path: DialsReflectionFilePath, +) -> DialsReflectionFile: """read dials reflection file""" - return DialsReflections(load_reflection_file(file_path.as_posix(), copy=True)) + return DialsReflectionFile(load(file_path.as_posix(), copy=True)) def read_dials_experiment_file(file_path: DialsExperimentFilePath) -> DialsExperiment: """Read Dials Experiment .expt file""" - with open(file_path) as fp: - json_data = json.load(fp) - return DialsExperiment(json_data) + + return DialsExperiment(json.load(open(file_path))) def get_unit_cell(dials_expt: DialsExperiment) -> UnitCell: @@ -87,7 +87,7 @@ def get_unit_cell(dials_expt: DialsExperiment) -> UnitCell: return UnitCell(a, b, c, al, be, ga) -def get_space_group(dials_expt: DialsExperiment) -> gemmi.SpaceGroup: +def get_unique_space_group(dials_expt: DialsExperiment) -> gemmi.SpaceGroup: """ Get space group from Dials expt file. For cctbx/disambiguation reasons it is saved as the Hall symbol, but @@ -105,7 +105,7 @@ def get_reciprocal_asu(spacegroup: gemmi.SpaceGroup) -> gemmi.ReciprocalAsu: return gemmi.ReciprocalAsu(spacegroup) -def dials_refl_to_pandas(refls: DialsReflections) -> pd.DataFrame: +def dials_refl_to_pandas(refls: dict) -> pd.DataFrame: """Converts the loaded DIALS reflection file to a pandas dataframe. It is equivalent to the following code: @@ -124,19 +124,16 @@ def dials_refl_to_pandas(refls: DialsReflections) -> pd.DataFrame: """ if refls.get('experiment_identifier'): # this has no relevant information del refls['experiment_identifier'] # and it complicates loading as a df - df = pd.DataFrame( + return pd.DataFrame( { key: list(val) if isinstance(val, np.ndarray) and val.ndim > 1 else val for key, val in refls.items() } ) - for col in df.select_dtypes('uint64'): - df[col] = df[col].astype('int64') - return df def process_dials_refl_list_to_dataframe( - refls: DialsReflections, + refls: dict, ) -> DialsDataFrame: """Select and derive columns from the original ``MtzDataFrame``. @@ -173,8 +170,6 @@ def process_dials_refl_list_to_dataframe( Other columns are kept as they are. """ - from .dials_io import dials_refl_to_pandas - orig_df = dials_refl_to_pandas(refls) new_df = pd.DataFrame() @@ -190,6 +185,9 @@ def process_dials_refl_list_to_dataframe( new_df[DEFAULT_VARIANCE_COLUMN_NAME] = orig_df['intensity.sum.variance'] new_df[DEFAULT_STDEV_COLUMN_NAME] = np.sqrt(orig_df['intensity.sum.variance']) + for column in [col for col in orig_df.columns if col not in new_df]: + new_df[column] = orig_df[column] + return DialsDataFrame(new_df) @@ -222,7 +220,7 @@ def _reciprocal_asu(row: pd.Series) -> list[int]: def nmx_dials_dataframe_to_scipp_dataarray( nmx_mtz_df: NMXDialsDataFrame, -) -> NMXMtzDataArray: +) -> NMXDialsDataArray: """Converts the processed mtz dataframe to a scipp dataarray. The intensity, with column name :attr:`~DEFAULT_INTENSITY_COLUMN_NAME` @@ -296,24 +294,4 @@ def nmx_dials_dataframe_to_scipp_dataarray( nmx_mtz_da.variances = nmx_mtz_ds[DEFAULT_VARIANCE_COLUMN_NAME].values # Return DataArray without negative intensities - return NMXMtzDataArray(nmx_mtz_da[nmx_mtz_da.data > 0]) - - -providers = ( - read_dials_reflection_file, - read_dials_experiment_file, - process_dials_refl_list_to_dataframe, - process_dials_dataframe, - get_space_group, - get_reciprocal_asu, - process_dials_dataframe, - nmx_dials_dataframe_to_scipp_dataarray, -) -"""The providers related to the Dials IO.""" - -default_parameters = { - WavelengthColumnName: DEFAULT_WAVELENGTH_COLUMN_NAME, - IntensityColumnName: DEFAULT_INTENSITY_COLUMN_NAME, - StdDevColumnName: DEFAULT_STDEV_COLUMN_NAME, -} -"""The parameters related to the Dials IO.""" + return NMXDialsDataArray(nmx_mtz_da[nmx_mtz_da.data > 0]) diff --git a/packages/essnmx/src/ess/nmx/dials_reflection_io.py b/packages/essnmx/src/ess/nmx/dials_reflection_io.py index 971f34f8..c01fe55c 100644 --- a/packages/essnmx/src/ess/nmx/dials_reflection_io.py +++ b/packages/essnmx/src/ess/nmx/dials_reflection_io.py @@ -29,9 +29,9 @@ class Shoebox: panel: int bbox: tuple[int] - data: np.array = None - mask: np.array = None - background: np.array = None + data: np.ndarray | None = None + mask: np.ndarray | None = None + background: np.ndarray | None = None def _decode_raw_numpy(dtype, shape: int | Iterable = 1): @@ -56,6 +56,7 @@ def _decode_specific(data, copy): if shape != (1,): item_width = functools.reduce(operator.mul, shape) +<<<<<<< HEAD if (len(raw) % item_width) != 0: raise AssertionError( "Data length %s is not divisible by item width %s", @@ -69,6 +70,20 @@ def _decode_specific(data, copy): len(array), num_items, item_width, +======= + if len(raw) % item_width != 0: + raise AssertionError( + "Raw data length %s not divisible by item width %s", + len(raw), + item_width, + ) + if num_items * item_width != len(array): + raise AssertionError( + "(Num items) %s * (item width) %s != (raw data length) %s", + num_items, + item_width, + len(raw), +>>>>>>> 2ea4824 (Checkout dial related modules from scaling_dials branch.) ) array = array.reshape(num_items, *shape) if copy: @@ -78,7 +93,11 @@ def _decode_specific(data, copy): return _decode_specific +<<<<<<< HEAD def _decode_shoeboxes(data: list, copy) -> list[Shoebox | None]: +======= +def _decode_shoeboxes(data: list, copy) -> Iterable[Shoebox]: +>>>>>>> 2ea4824 (Checkout dial related modules from scaling_dials branch.) # Shoebox is float num_items, raw = data shoeboxes: list[Shoebox | None] = [] @@ -120,12 +139,17 @@ def _decode_shoeboxes(data: list, copy) -> list[Shoebox | None]: shoeboxes.append(Shoebox(**shoebox)) if len(shoeboxes) != num_items: raise AssertionError( +<<<<<<< HEAD "Warning: Mismatch of shoebox length: %s " "is not same as the number of items: %s", len(shoeboxes), num_items, ) +======= + f"Mismatch of shoebox lengths: {len(shoeboxes)} != {num_items}" + ) +>>>>>>> 2ea4824 (Checkout dial related modules from scaling_dials branch.) return np.array(shoeboxes, dtype=np.object_) @@ -164,7 +188,10 @@ def _get_unpacked(stream_or_path: str | IO | bytes | os.PathLike): path = os.fspath(cast(str, stream_or_path)) is_fspathlike = True except (TypeError, ValueError): +<<<<<<< HEAD path = stream_or_path +======= +>>>>>>> 2ea4824 (Checkout dial related modules from scaling_dials branch.) is_fspathlike = isinstance(stream_or_path, str) if is_fspathlike: @@ -186,10 +213,17 @@ def loads(data: bytes, copy=False): Returns: See .load(stream_or_path) """ +<<<<<<< HEAD return load_reflection_file(BytesIO(data), copy) def load_reflection_file(stream_or_path: IO | Path | os.PathLike, copy=False) -> dict: +======= + return load(BytesIO(data), copy) + + +def load(stream_or_path: IO | Path | os.PathLike, copy=False) -> dict: +>>>>>>> 2ea4824 (Checkout dial related modules from scaling_dials branch.) """ Load a DIALS msgpack-encoded .refl file @@ -220,7 +254,11 @@ def load_reflection_file(stream_or_path: IO | Path | os.PathLike, copy=False) -> raise ValueError("Does not appear to be a dials reflection table file") if not root_data[1] == 1: raise ValueError( +<<<<<<< HEAD f"reflection_table data is version {root_data[1]}. " +======= + f"reflection_table data is version {root_data[1]}." +>>>>>>> 2ea4824 (Checkout dial related modules from scaling_dials branch.) "Only Version 1 is understood" ) refdata = root_data[2] @@ -240,7 +278,12 @@ def load_reflection_file(stream_or_path: IO | Path | os.PathLike, copy=False) -> for name, column in decoded_data.items(): if len(column) != rows: logging.warning( +<<<<<<< HEAD "Warning: Mismatch of column lengths: %s is %s instead of expected %s", +======= + "Warning: Mismatch of column lengths: " + "[%s] is [%d] instead of expected [%s]", +>>>>>>> 2ea4824 (Checkout dial related modules from scaling_dials branch.) name, len(column), rows, From 9178611f46d214a39b616a23fd6723185008979e Mon Sep 17 00:00:00 2001 From: Aaron Finke Date: Tue, 27 May 2025 09:35:49 +0200 Subject: [PATCH 257/403] adding nbins to executable --- .../essnmx/src/ess/nmx/mcstas/executables.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas/executables.py b/packages/essnmx/src/ess/nmx/mcstas/executables.py index 08b50c7d..8db8ca0b 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/executables.py +++ b/packages/essnmx/src/ess/nmx/mcstas/executables.py @@ -139,6 +139,7 @@ def reduction( input_file: pathlib.Path, output_file: pathlib.Path, chunk_size: int = 10_000_000, + nbins: int = 51, detector_ids: list[int | str], compression: bool = True, wf: sl.Pipeline | None = None, @@ -159,7 +160,7 @@ def reduction( logger.info("Metadata retrieved: %s", data_metadata) toa_bin_edges = sc.linspace( - dim='t', start=data_metadata.min_toa, stop=data_metadata.max_toa, num=51 + dim='t', start=data_metadata.min_toa, stop=data_metadata.max_toa, num=nbins ) scale_factor = mcstas_weight_to_probability_scalefactor( max_counts=wf.compute(MaximumCounts), @@ -171,12 +172,16 @@ def reduction( toa_min = sc.scalar(toa_min_max_prob[0], unit='s') toa_max = sc.scalar(toa_min_max_prob[1], unit='s') prob_max = sc.scalar(toa_min_max_prob[2]) - toa_bin_edges = sc.linspace(dim='t', start=toa_min, stop=toa_max, num=51) + toa_bin_edges = sc.linspace(dim='t', start=toa_min, stop=toa_max, num=nbins) scale_factor = mcstas_weight_to_probability_scalefactor( max_counts=wf.compute(MaximumCounts), max_probability=prob_max, ) + scale_factor = mcstas_weight_to_probability_scalefactor( + max_counts=wf.compute(MaximumCounts), + max_probability=data_metadata.max_probability, + ) # Compute metadata and make the skeleton output file experiment_metadata = wf.compute(NMXExperimentMetadata) detector_metas = [] @@ -269,6 +274,12 @@ def main() -> None: default=10_000_000, help="Chunk size for processing", ) + parser.add_argument( + "--nbins", + type=int, + default=51, + help="Number of TOF bins", + ) parser.add_argument( "--detector_ids", type=int, @@ -298,6 +309,7 @@ def main() -> None: input_file=input_file, output_file=output_file, chunk_size=args.chunk_size, + nbins=args.nbins, detector_ids=args.detector_ids, compression=args.compression, logger=logger, From 2507d81b3a6b4e3a6c77da348edae151541ca1d8 Mon Sep 17 00:00:00 2001 From: Aaron Finke Date: Wed, 4 Jun 2025 11:22:04 +0200 Subject: [PATCH 258/403] added option for monitor to be union_abs_logger_nD or Montior_nD --- packages/essnmx/src/ess/nmx/mcstas/load.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas/load.py b/packages/essnmx/src/ess/nmx/mcstas/load.py index 477a2adc..a3158407 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/load.py +++ b/packages/essnmx/src/ess/nmx/mcstas/load.py @@ -276,7 +276,8 @@ def bank_names_to_detector_names(description: str) -> dict[str, list[str]]: detector_component_regex = ( # Start of the detector component definition, contains the detector name. - r'^COMPONENT (?P.*) = Monitor_nD\(\n' + # r'^COMPONENT (?P.*) = Monitor_nD\(\n' + r'^COMPONENT (?P.*) = (Monitor_nD|Union_abs_logger_nD)\(\n' # Some uninteresting lines, we're looking for 'filename'. # Make sure no new component begins. r'(?:(?!COMPONENT)(?!filename)(?:.|\s))*' From e36ca939c4c125eadeb415bcc9fd42a395901d5e Mon Sep 17 00:00:00 2001 From: Aaron Finke Date: Mon, 9 Jun 2025 14:26:26 +0200 Subject: [PATCH 259/403] adding max_counts to executable options --- .../essnmx/src/ess/nmx/mcstas/executables.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/mcstas/executables.py b/packages/essnmx/src/ess/nmx/mcstas/executables.py index 8db8ca0b..47abe419 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/executables.py +++ b/packages/essnmx/src/ess/nmx/mcstas/executables.py @@ -140,6 +140,7 @@ def reduction( output_file: pathlib.Path, chunk_size: int = 10_000_000, nbins: int = 51, + max_counts: int | None = None, detector_ids: list[int | str], compression: bool = True, wf: sl.Pipeline | None = None, @@ -178,10 +179,16 @@ def reduction( max_probability=prob_max, ) - scale_factor = mcstas_weight_to_probability_scalefactor( - max_counts=wf.compute(MaximumCounts), - max_probability=data_metadata.max_probability, - ) + if max_counts: + scale_factor = mcstas_weight_to_probability_scalefactor( + max_counts=MaximumCounts(max_counts), + max_probability=data_metadata.max_probability, + ) + else: + scale_factor = mcstas_weight_to_probability_scalefactor( + max_counts=wf.compute(MaximumCounts), + max_probability=data_metadata.max_probability, + ) # Compute metadata and make the skeleton output file experiment_metadata = wf.compute(NMXExperimentMetadata) detector_metas = [] @@ -280,6 +287,12 @@ def main() -> None: default=51, help="Number of TOF bins", ) + parser.add_argument( + "--max_counts", + type=int, + default=None, + help="Maximum Counts", + ) parser.add_argument( "--detector_ids", type=int, @@ -310,6 +323,7 @@ def main() -> None: output_file=output_file, chunk_size=args.chunk_size, nbins=args.nbins, + max_counts=args.max_counts, detector_ids=args.detector_ids, compression=args.compression, logger=logger, From 187a8483a86d897c6d47312df88281786961c9da Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Mon, 16 Jun 2025 13:54:56 +0200 Subject: [PATCH 260/403] Resolve merge conflicts. --- .../essnmx/src/ess/nmx/dials_reflection_io.py | 47 ------------------- 1 file changed, 47 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/dials_reflection_io.py b/packages/essnmx/src/ess/nmx/dials_reflection_io.py index c01fe55c..dd72f624 100644 --- a/packages/essnmx/src/ess/nmx/dials_reflection_io.py +++ b/packages/essnmx/src/ess/nmx/dials_reflection_io.py @@ -56,21 +56,6 @@ def _decode_specific(data, copy): if shape != (1,): item_width = functools.reduce(operator.mul, shape) -<<<<<<< HEAD - if (len(raw) % item_width) != 0: - raise AssertionError( - "Data length %s is not divisible by item width %s", - len(raw), - item_width, - ) - elif (num_items * item_width) != len(array): - raise AssertionError( - "Data length %s is not equal to " - "number of items %s times item width %s", - len(array), - num_items, - item_width, -======= if len(raw) % item_width != 0: raise AssertionError( "Raw data length %s not divisible by item width %s", @@ -83,7 +68,6 @@ def _decode_specific(data, copy): num_items, item_width, len(raw), ->>>>>>> 2ea4824 (Checkout dial related modules from scaling_dials branch.) ) array = array.reshape(num_items, *shape) if copy: @@ -93,11 +77,7 @@ def _decode_specific(data, copy): return _decode_specific -<<<<<<< HEAD -def _decode_shoeboxes(data: list, copy) -> list[Shoebox | None]: -======= def _decode_shoeboxes(data: list, copy) -> Iterable[Shoebox]: ->>>>>>> 2ea4824 (Checkout dial related modules from scaling_dials branch.) # Shoebox is float num_items, raw = data shoeboxes: list[Shoebox | None] = [] @@ -139,17 +119,8 @@ def _decode_shoeboxes(data: list, copy) -> Iterable[Shoebox]: shoeboxes.append(Shoebox(**shoebox)) if len(shoeboxes) != num_items: raise AssertionError( -<<<<<<< HEAD - "Warning: Mismatch of shoebox length: %s " - "is not same as the number of items: %s", - len(shoeboxes), - num_items, - ) - -======= f"Mismatch of shoebox lengths: {len(shoeboxes)} != {num_items}" ) ->>>>>>> 2ea4824 (Checkout dial related modules from scaling_dials branch.) return np.array(shoeboxes, dtype=np.object_) @@ -188,10 +159,7 @@ def _get_unpacked(stream_or_path: str | IO | bytes | os.PathLike): path = os.fspath(cast(str, stream_or_path)) is_fspathlike = True except (TypeError, ValueError): -<<<<<<< HEAD path = stream_or_path -======= ->>>>>>> 2ea4824 (Checkout dial related modules from scaling_dials branch.) is_fspathlike = isinstance(stream_or_path, str) if is_fspathlike: @@ -213,17 +181,10 @@ def loads(data: bytes, copy=False): Returns: See .load(stream_or_path) """ -<<<<<<< HEAD - return load_reflection_file(BytesIO(data), copy) - - -def load_reflection_file(stream_or_path: IO | Path | os.PathLike, copy=False) -> dict: -======= return load(BytesIO(data), copy) def load(stream_or_path: IO | Path | os.PathLike, copy=False) -> dict: ->>>>>>> 2ea4824 (Checkout dial related modules from scaling_dials branch.) """ Load a DIALS msgpack-encoded .refl file @@ -254,11 +215,7 @@ def load(stream_or_path: IO | Path | os.PathLike, copy=False) -> dict: raise ValueError("Does not appear to be a dials reflection table file") if not root_data[1] == 1: raise ValueError( -<<<<<<< HEAD - f"reflection_table data is version {root_data[1]}. " -======= f"reflection_table data is version {root_data[1]}." ->>>>>>> 2ea4824 (Checkout dial related modules from scaling_dials branch.) "Only Version 1 is understood" ) refdata = root_data[2] @@ -278,12 +235,8 @@ def load(stream_or_path: IO | Path | os.PathLike, copy=False) -> dict: for name, column in decoded_data.items(): if len(column) != rows: logging.warning( -<<<<<<< HEAD - "Warning: Mismatch of column lengths: %s is %s instead of expected %s", -======= "Warning: Mismatch of column lengths: " "[%s] is [%d] instead of expected [%s]", ->>>>>>> 2ea4824 (Checkout dial related modules from scaling_dials branch.) name, len(column), rows, From c538b6317604e988ffcf9336a1c5b548a177ffc1 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Tue, 17 Jun 2025 12:50:47 +0200 Subject: [PATCH 261/403] copier update --- packages/essnmx/.copier-answers.ess.yml | 2 +- .../.github/ISSUE_TEMPLATE/high-level-requirement.yml | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/essnmx/.copier-answers.ess.yml b/packages/essnmx/.copier-answers.ess.yml index 1ca4397b..6425ea7c 100644 --- a/packages/essnmx/.copier-answers.ess.yml +++ b/packages/essnmx/.copier-answers.ess.yml @@ -1,3 +1,3 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY -_commit: 815268a +_commit: 34ca4ba _src_path: https://github.com/scipp/ess_template diff --git a/packages/essnmx/.github/ISSUE_TEMPLATE/high-level-requirement.yml b/packages/essnmx/.github/ISSUE_TEMPLATE/high-level-requirement.yml index cedef1d9..4d87603b 100644 --- a/packages/essnmx/.github/ISSUE_TEMPLATE/high-level-requirement.yml +++ b/packages/essnmx/.github/ISSUE_TEMPLATE/high-level-requirement.yml @@ -79,6 +79,14 @@ body: description: How can we test this requirement? Links to tests data and reference data, or other suggestions. validations: required: true + - type: textarea + id: existingimplementations + attributes: + label: Existing implementations + description: Are there any existing implementations or proof-of-concept implementations that we can imitate? This field is specifically for linking to source code. + placeholder: "Example: See this repository ... This script implements the procedure: https://file-storage.server.eu/script.code." + validations: + required: false - type: textarea id: comments attributes: From 7045236ef3dc9f91159f7a2f664677530ef0fd90 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Thu, 17 Jul 2025 10:22:16 +0200 Subject: [PATCH 262/403] copier update --- packages/essnmx/.copier-answers.yml | 2 +- .../workflows/nightly_at_main_lower_bound.yml | 37 +++++++++++++++++++ packages/essnmx/.pre-commit-config.yaml | 2 + packages/essnmx/docs/about/index.md | 4 +- packages/essnmx/docs/conf.py | 32 +++++++++------- packages/essnmx/docs/index.md | 4 +- packages/essnmx/pyproject.toml | 6 +-- packages/essnmx/requirements/docs.in | 1 + 8 files changed, 66 insertions(+), 22 deletions(-) create mode 100644 packages/essnmx/.github/workflows/nightly_at_main_lower_bound.yml diff --git a/packages/essnmx/.copier-answers.yml b/packages/essnmx/.copier-answers.yml index cfbb0254..2036def7 100644 --- a/packages/essnmx/.copier-answers.yml +++ b/packages/essnmx/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY -_commit: cdbfd9c +_commit: 3561fcd _src_path: gh:scipp/copier_template description: Data reduction for NMX at the European Spallation Source. max_python: '3.13' diff --git a/packages/essnmx/.github/workflows/nightly_at_main_lower_bound.yml b/packages/essnmx/.github/workflows/nightly_at_main_lower_bound.yml new file mode 100644 index 00000000..c13c3f78 --- /dev/null +++ b/packages/essnmx/.github/workflows/nightly_at_main_lower_bound.yml @@ -0,0 +1,37 @@ +name: Nightly test using lower bound dependencies + +on: + workflow_dispatch: + schedule: + - cron: '30 1 * * 1-5' + +jobs: + setup: + name: Setup variables + runs-on: 'ubuntu-24.04' + outputs: + min_python: ${{ steps.vars.outputs.min_python }} + steps: + - uses: actions/checkout@v4 + - name: Get Python version for other CI jobs + id: vars + run: echo "min_python=$(< .github/workflows/python-version-ci)" >> "$GITHUB_OUTPUT" + + tests: + name: Tests at lower bound + needs: setup + strategy: + matrix: + os: ['ubuntu-24.04'] + python: + - version: '${{needs.setup.outputs.min_python}}' + runs-on: ${{ matrix.os }} + env: + ESS_PROTECTED_FILESTORE_USERNAME: ${{ secrets.ESS_PROTECTED_FILESTORE_USERNAME }} + ESS_PROTECTED_FILESTORE_PASSWORD: ${{ secrets.ESS_PROTECTED_FILESTORE_PASSWORD }} + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + with: + python-version: ${{ matrix.python.version }} + - run: uv run --extra=test --resolution=lowest-direct pytest diff --git a/packages/essnmx/.pre-commit-config.yaml b/packages/essnmx/.pre-commit-config.yaml index 0f3f9a95..4f913e71 100644 --- a/packages/essnmx/.pre-commit-config.yaml +++ b/packages/essnmx/.pre-commit-config.yaml @@ -36,6 +36,8 @@ repos: - id: codespell additional_dependencies: - tomli + exclude_types: + - svg - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 hooks: diff --git a/packages/essnmx/docs/about/index.md b/packages/essnmx/docs/about/index.md index fcdc603f..61b745df 100644 --- a/packages/essnmx/docs/about/index.md +++ b/packages/essnmx/docs/about/index.md @@ -10,11 +10,11 @@ data_workflow_overview ## Development -ESSnmx is an open source project by the [European Spallation Source ERIC](https://europeanspallationsource.se/) (ESS). +ESSnmx is an open source project by the [European Spallation Source ERIC](https://ess.eu/) (ESS). ## License -ESSnmx is available as open source under the [BSD-3 license](https://opensource.org/licenses/BSD-3-Clause). +ESSnmx is available as open source under the [BSD-3 license](https://opensource.org/license/BSD-3-Clause). ## Citing ESSnmx diff --git a/packages/essnmx/docs/conf.py b/packages/essnmx/docs/conf.py index 1601faf3..6144897c 100644 --- a/packages/essnmx/docs/conf.py +++ b/packages/essnmx/docs/conf.py @@ -21,19 +21,20 @@ html_show_sourcelink = True extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.autosummary", - "sphinx.ext.doctest", - "sphinx.ext.githubpages", - "sphinx.ext.intersphinx", - "sphinx.ext.mathjax", - "sphinx.ext.napoleon", - "sphinx.ext.viewcode", - "sphinx_autodoc_typehints", - "sphinx_copybutton", - "sphinx_design", - "nbsphinx", - "myst_parser", + 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', + 'sphinx.ext.doctest', + 'sphinx.ext.githubpages', + 'sphinx.ext.intersphinx', + 'sphinx.ext.mathjax', + 'sphinx.ext.napoleon', + 'sphinx.ext.viewcode', + 'sphinx_autodoc_typehints', + 'sphinx_copybutton', + 'sphinx_design', + 'sphinxcontrib.autodoc_pydantic', + 'nbsphinx', + 'myst_parser', ] try: @@ -263,5 +264,8 @@ def do_not_plot(*args, **kwargs): linkcheck_ignore = [ # Specific lines in Github blobs cannot be found by linkcheck. - r"https?://github\.com/.*?/blob/[a-f0-9]+/.+?#", + r'https?://github\.com/.*?/blob/[a-f0-9]+/.+?#', + # Linkcheck seems to be denied access by some DOI resolvers. + # Since DOIs are supposed to be permanent, we don't need to check them.' + r'https://doi\.org/', ] diff --git a/packages/essnmx/docs/index.md b/packages/essnmx/docs/index.md index fe17dff6..2445e0fd 100644 --- a/packages/essnmx/docs/index.md +++ b/packages/essnmx/docs/index.md @@ -23,10 +23,10 @@ # {transparent}`ESSnmx` - +
Data reduction for NMX at the European Spallation Source.

- +
:::{include} user-guide/installation.md :heading-offset: 1 diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index f7684abf..2fb17ce1 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -1,6 +1,6 @@ [build-system] requires = [ - "setuptools>=68", + "setuptools>=77", "setuptools_scm[toml]>=8.0", ] build-backend = "setuptools.build_meta" @@ -9,11 +9,11 @@ build-backend = "setuptools.build_meta" name = "essnmx" description = "Data reduction for NMX at the European Spallation Source." authors = [{ name = "Scipp contributors" }] -license = { file = "LICENSE" } +license = "BSD-3-Clause" +license-files = ["LICENSE"] readme = "README.md" classifiers = [ "Intended Audience :: Science/Research", - "License :: OSI Approved :: BSD License", "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", diff --git a/packages/essnmx/requirements/docs.in b/packages/essnmx/requirements/docs.in index cc0c975f..4693d337 100644 --- a/packages/essnmx/requirements/docs.in +++ b/packages/essnmx/requirements/docs.in @@ -1,4 +1,5 @@ -r base.in +autodoc-pydantic ipykernel ipython!=8.7.0 # Breaks syntax highlighting in Jupyter code cells. myst-parser From e3724fc6396c3c57e4bfec35697d13cf019e7f20 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Thu, 17 Jul 2025 10:26:59 +0200 Subject: [PATCH 263/403] deps: lower pins --- packages/essnmx/pyproject.toml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 2fb17ce1..6620b91b 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -31,19 +31,19 @@ requires-python = ">=3.10" # Run 'tox -e deps' after making changes here. This will update requirement files. # Make sure to list one dependency per line. dependencies = [ - "dask", + "dask>=2022.1.0", "essreduce>=24.03.0", "graphviz", - "plopp", + "plopp>=24.7.0", "sciline>=24.06.0", "scipp>=23.8.0", "scippnexus>=23.12.0", - "pooch", - "pandas", - "gemmi", - "defusedxml", - "bitshuffle", - "msgpack", + "pooch>=1.5", + "pandas>=2.1.2", + "gemmi>=0.6.6", + "defusedxml>=0.7.1", + "bitshuffle>=0.5.2", + "msgpack>=1.0.8", ] dynamic = ["version"] @@ -53,7 +53,7 @@ essnmx_reduce_mcstas = "ess.nmx.mcstas.executables:main" [project.optional-dependencies] test = [ - "pytest", + "pytest>=7.0", ] [project.urls] From b1d8aba5becbc86fb346f984e43619c90b7ade51 Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Thu, 17 Jul 2025 10:28:20 +0200 Subject: [PATCH 264/403] tox -e deps --- packages/essnmx/requirements/base.in | 16 ++--- packages/essnmx/requirements/base.txt | 74 +++++++++++----------- packages/essnmx/requirements/basetest.in | 2 +- packages/essnmx/requirements/basetest.txt | 18 +++--- packages/essnmx/requirements/ci.txt | 28 ++++----- packages/essnmx/requirements/dev.txt | 28 ++++----- packages/essnmx/requirements/docs.txt | 49 ++++++++------- packages/essnmx/requirements/mypy.txt | 10 +-- packages/essnmx/requirements/nightly.in | 16 ++--- packages/essnmx/requirements/nightly.txt | 77 ++++++++++++----------- packages/essnmx/requirements/static.txt | 10 +-- packages/essnmx/requirements/test.txt | 4 +- packages/essnmx/requirements/wheels.txt | 6 +- 13 files changed, 175 insertions(+), 163 deletions(-) diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index 2dba136e..8826f94f 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -2,16 +2,16 @@ # will not be touched by ``make_base.py`` # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! -dask +dask>=2022.1.0 essreduce>=24.03.0 graphviz -plopp +plopp>=24.7.0 sciline>=24.06.0 scipp>=23.8.0 scippnexus>=23.12.0 -pooch -pandas -gemmi -defusedxml -bitshuffle -msgpack +pooch>=1.5 +pandas>=2.1.2 +gemmi>=0.6.6 +defusedxml>=0.7.1 +bitshuffle>=0.5.2 +msgpack>=1.0.8 diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 84635591..a06d06c8 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,31 +1,31 @@ -# SHA1:e30515c3fb52510b8e8abf8e785cc372219f7527 +# SHA1:57dea2fd04558cbe6e1a72f838773564fbe6cb3c # -# This file is autogenerated by pip-compile-multi +# This file was generated by pip-compile-multi. # To update, run: # -# pip-compile-multi +# requirements upgrade # annotated-types==0.7.0 # via pydantic bitshuffle==0.5.2 # via -r base.in -certifi==2025.1.31 +certifi==2025.7.14 # via requests -charset-normalizer==3.4.1 +charset-normalizer==3.4.2 # via requests -click==8.1.8 +click==8.2.1 # via dask cloudpickle==3.1.1 # via dask -contourpy==1.3.1 +contourpy==1.3.2 # via matplotlib cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -cython==3.0.12 +cython==3.1.2 # via bitshuffle -dask==2025.3.0 +dask==2025.7.0 # via -r base.in defusedxml==0.7.1 # via -r base.in @@ -33,17 +33,17 @@ dnspython==2.7.0 # via email-validator email-validator==2.2.0 # via scippneutron -essreduce==25.3.1 +essreduce==25.7.1 # via -r base.in -fonttools==4.57.0 +fonttools==4.59.0 # via matplotlib -fsspec==2025.3.2 +fsspec==2025.7.0 # via dask -gemmi==0.7.1 +gemmi==0.7.3 # via -r base.in -graphviz==0.20.3 +graphviz==0.21 # via -r base.in -h5py==3.13.0 +h5py==3.14.0 # via # bitshuffle # scippneutron @@ -52,7 +52,7 @@ idna==3.10 # via # email-validator # requests -importlib-metadata==8.6.1 +importlib-metadata==8.7.0 # via dask kiwisolver==1.4.8 # via matplotlib @@ -62,50 +62,49 @@ lazy-loader==0.4 # scippneutron locket==1.0.0 # via partd -matplotlib==3.10.1 +matplotlib==3.10.3 # via # mpltoolbox # plopp -mpltoolbox==24.5.1 +mpltoolbox==25.5.0 # via scippneutron -msgpack==1.1.0 +msgpack==1.1.1 # via -r base.in networkx==3.4.2 # via cyclebane -numpy==2.2.4 +numpy==2.2.6 # via # bitshuffle # contourpy # h5py # matplotlib - # mpltoolbox # pandas # scipp # scippneutron # scipy -packaging==24.2 +packaging==25.0 # via # dask # lazy-loader # matplotlib # pooch -pandas==2.2.3 +pandas==2.3.1 # via -r base.in partd==1.4.2 # via dask -pillow==11.1.0 +pillow==11.3.0 # via matplotlib -platformdirs==4.3.7 +platformdirs==4.3.8 # via pooch -plopp==25.3.0 +plopp==25.7.0 # via # -r base.in # scippneutron pooch==1.8.2 # via -r base.in -pydantic==2.11.2 +pydantic==2.11.7 # via scippneutron -pydantic-core==2.33.1 +pydantic-core==2.33.2 # via pydantic pyparsing==3.2.3 # via matplotlib @@ -119,9 +118,9 @@ pytz==2025.2 # via pandas pyyaml==6.0.2 # via dask -requests==2.32.3 +requests==2.32.4 # via pooch -sciline==25.4.0 +sciline==25.5.2 # via # -r base.in # essreduce @@ -131,14 +130,14 @@ scipp==25.5.1 # essreduce # scippneutron # scippnexus -scippneutron==25.2.1 +scippneutron==25.7.0 # via essreduce -scippnexus==25.4.0 +scippnexus==25.6.0 # via # -r base.in # essreduce # scippneutron -scipy==1.15.2 +scipy==1.15.3 # via # scippneutron # scippnexus @@ -148,18 +147,19 @@ toolz==1.0.0 # via # dask # partd -typing-extensions==4.13.1 +typing-extensions==4.14.1 # via # pydantic # pydantic-core + # sciline # typing-inspection -typing-inspection==0.4.0 +typing-inspection==0.4.1 # via pydantic tzdata==2025.2 # via pandas -urllib3==2.3.0 +urllib3==2.5.0 # via requests -zipp==3.21.0 +zipp==3.23.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/packages/essnmx/requirements/basetest.in b/packages/essnmx/requirements/basetest.in index 5b3942ea..231016ec 100644 --- a/packages/essnmx/requirements/basetest.in +++ b/packages/essnmx/requirements/basetest.in @@ -7,4 +7,4 @@ # will not be touched by ``make_base.py`` # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! -pytest +pytest>=7.0 diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt index 76aebec7..464e50dc 100644 --- a/packages/essnmx/requirements/basetest.txt +++ b/packages/essnmx/requirements/basetest.txt @@ -1,19 +1,23 @@ -# SHA1:0eaa389e1fdb3a1917c0f987514bd561be5718ee +# SHA1:8287decb8676bd4ad5934cc138073b38af537418 # -# This file is autogenerated by pip-compile-multi +# This file was generated by pip-compile-multi. # To update, run: # -# pip-compile-multi +# requirements upgrade # -exceptiongroup==1.2.2 +exceptiongroup==1.3.0 # via pytest iniconfig==2.1.0 # via pytest -packaging==24.2 +packaging==25.0 # via pytest -pluggy==1.5.0 +pluggy==1.6.0 # via pytest -pytest==8.3.5 +pygments==2.19.2 + # via pytest +pytest==8.4.1 # via -r basetest.in tomli==2.2.1 # via pytest +typing-extensions==4.14.1 + # via exceptiongroup diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index 176b5124..629701a6 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -1,17 +1,17 @@ # SHA1:6344d52635ea11dca331a3bc6eb1833c4c64d585 # -# This file is autogenerated by pip-compile-multi +# This file was generated by pip-compile-multi. # To update, run: # -# pip-compile-multi +# requirements upgrade # -cachetools==5.5.2 +cachetools==6.1.0 # via tox -certifi==2025.1.31 +certifi==2025.7.14 # via requests chardet==5.2.0 # via tox -charset-normalizer==3.4.1 +charset-normalizer==3.4.2 # via requests colorama==0.4.6 # via tox @@ -27,20 +27,20 @@ gitpython==3.1.44 # via -r ci.in idna==3.10 # via requests -packaging==24.2 +packaging==25.0 # via # -r ci.in # pyproject-api # tox -platformdirs==4.3.7 +platformdirs==4.3.8 # via # tox # virtualenv -pluggy==1.5.0 +pluggy==1.6.0 # via tox -pyproject-api==1.9.0 +pyproject-api==1.9.1 # via tox -requests==2.32.3 +requests==2.32.4 # via -r ci.in smmap==5.0.2 # via gitdb @@ -48,11 +48,11 @@ tomli==2.2.1 # via # pyproject-api # tox -tox==4.25.0 +tox==4.27.0 # via -r ci.in -typing-extensions==4.13.1 +typing-extensions==4.14.1 # via tox -urllib3==2.3.0 +urllib3==2.5.0 # via requests -virtualenv==20.30.0 +virtualenv==20.31.2 # via tox diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index 091efdaf..290f4dcc 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -1,9 +1,9 @@ # SHA1:efd19a3a98c69fc3d6d6233ed855de7e4a208f74 # -# This file is autogenerated by pip-compile-multi +# This file was generated by pip-compile-multi. # To update, run: # -# pip-compile-multi +# requirements upgrade # -r base.txt -r ci.txt @@ -16,7 +16,7 @@ anyio==4.9.0 # via # httpx # jupyter-server -argon2-cffi==23.1.0 +argon2-cffi==25.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 # via argon2-cffi @@ -26,17 +26,17 @@ async-lru==2.0.5 # via jupyterlab cffi==1.17.1 # via argon2-cffi-bindings -copier==9.6.0 +copier==9.8.0 # via -r dev.in -dunamai==1.23.1 +dunamai==1.25.0 # via copier fqdn==1.5.1 # via jsonschema funcy==2.0 # via copier -h11==0.14.0 +h11==0.16.0 # via httpcore -httpcore==1.0.7 +httpcore==1.0.9 # via httpx httpx==0.28.1 # via jupyterlab @@ -48,7 +48,7 @@ json5==0.12.0 # via jupyterlab-server jsonpointer==3.0.0 # via jsonschema -jsonschema[format-nongpl]==4.23.0 +jsonschema[format-nongpl]==4.24.0 # via # jupyter-events # jupyterlab-server @@ -57,7 +57,7 @@ jupyter-events==0.12.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab -jupyter-server==2.15.0 +jupyter-server==2.16.0 # via # jupyter-lsp # jupyterlab @@ -65,7 +65,7 @@ jupyter-server==2.15.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.4.0 +jupyterlab==4.4.4 # via -r dev.in jupyterlab-server==2.27.3 # via jupyterlab @@ -73,15 +73,13 @@ notebook-shim==0.2.4 # via jupyterlab overrides==7.7.0 # via jupyter-server -pathspec==0.12.1 - # via copier -pip-compile-multi==2.8.0 +pip-compile-multi==3.2.1 # via -r dev.in pip-tools==7.4.1 # via pip-compile-multi plumbum==1.9.0 # via copier -prometheus-client==0.21.1 +prometheus-client==0.22.1 # via jupyter-server pycparser==2.22 # via cffi @@ -107,7 +105,7 @@ terminado==0.18.1 # jupyter-server-terminals toposort==1.10 # via pip-compile-multi -types-python-dateutil==2.9.0.20241206 +types-python-dateutil==2.9.0.20250708 # via arrow uri-template==1.3.0 # via jsonschema diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index c1e66318..9025cbb5 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -1,28 +1,28 @@ -# SHA1:a4b21b1181ffe0d85627ecf73a8b997699993f2a +# SHA1:ac8033c9d3c36ca4ae5c2f2b12733e14aa0155ff # -# This file is autogenerated by pip-compile-multi +# This file was generated by pip-compile-multi. # To update, run: # -# pip-compile-multi +# requirements upgrade # -r base.txt accessible-pygments==0.0.5 # via pydata-sphinx-theme alabaster==1.0.0 # via sphinx -appnope==0.1.4 - # via ipykernel asttokens==3.0.0 # via stack-data attrs==25.3.0 # via # jsonschema # referencing +autodoc-pydantic==2.2.0 + # via -r docs.in babel==2.17.0 # via # pydata-sphinx-theme # sphinx -beautifulsoup4==4.13.3 +beautifulsoup4==4.13.4 # via # nbconvert # pydata-sphinx-theme @@ -32,7 +32,7 @@ comm==0.2.2 # via # ipykernel # ipywidgets -debugpy==1.8.13 +debugpy==1.8.15 # via ipykernel decorator==5.2.1 # via ipython @@ -42,7 +42,7 @@ docutils==0.21.2 # nbsphinx # pydata-sphinx-theme # sphinx -exceptiongroup==1.2.2 +exceptiongroup==1.3.0 # via ipython executing==2.2.0 # via stack-data @@ -54,12 +54,12 @@ ipydatawidgets==4.3.5 # via pythreejs ipykernel==6.29.5 # via -r docs.in -ipython==8.34.0 +ipython==8.37.0 # via # -r docs.in # ipykernel # ipywidgets -ipywidgets==8.1.5 +ipywidgets==8.1.7 # via # ipydatawidgets # pythreejs @@ -71,15 +71,15 @@ jinja2==3.1.6 # nbconvert # nbsphinx # sphinx -jsonschema==4.23.0 +jsonschema==4.24.0 # via nbformat -jsonschema-specifications==2024.10.1 +jsonschema-specifications==2025.4.1 # via jsonschema jupyter-client==8.6.3 # via # ipykernel # nbclient -jupyter-core==5.7.2 +jupyter-core==5.8.1 # via # ipykernel # jupyter-client @@ -88,7 +88,7 @@ jupyter-core==5.7.2 # nbformat jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-widgets==3.0.13 +jupyterlab-widgets==3.0.15 # via ipywidgets markdown-it-py==3.0.0 # via @@ -129,7 +129,7 @@ parso==0.8.4 # via jedi pexpect==4.9.0 # via ipython -prompt-toolkit==3.0.50 +prompt-toolkit==3.0.51 # via ipython psutil==7.0.0 # via ipykernel @@ -137,18 +137,22 @@ ptyprocess==0.7.0 # via pexpect pure-eval==0.2.3 # via stack-data +pydantic-settings==2.10.1 + # via autodoc-pydantic pydata-sphinx-theme==0.16.1 # via -r docs.in -pygments==2.19.1 +pygments==2.19.2 # via # accessible-pygments # ipython # nbconvert # pydata-sphinx-theme # sphinx +python-dotenv==1.1.1 + # via pydantic-settings pythreejs==2.4.2 # via -r docs.in -pyzmq==26.4.0 +pyzmq==27.0.0 # via # ipykernel # jupyter-client @@ -156,17 +160,18 @@ referencing==0.36.2 # via # jsonschema # jsonschema-specifications -rpds-py==0.24.0 +rpds-py==0.26.0 # via # jsonschema # referencing -snowballstemmer==2.2.0 +snowballstemmer==3.0.1 # via sphinx -soupsieve==2.6 +soupsieve==2.7 # via beautifulsoup4 sphinx==8.1.3 # via # -r docs.in + # autodoc-pydantic # myst-parser # nbsphinx # pydata-sphinx-theme @@ -197,7 +202,7 @@ tinycss2==1.4.0 # via bleach tomli==2.2.1 # via sphinx -tornado==6.4.2 +tornado==6.5.1 # via # ipykernel # jupyter-client @@ -224,7 +229,7 @@ webencodings==0.5.1 # via # bleach # tinycss2 -widgetsnbextension==4.0.13 +widgetsnbextension==4.0.14 # via ipywidgets # The following packages are considered to be unsafe in a requirements file: diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index 40e3f07e..99a887e2 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -1,14 +1,16 @@ # SHA1:859ef9c15e5e57c6c91510133c01f5751feee941 # -# This file is autogenerated by pip-compile-multi +# This file was generated by pip-compile-multi. # To update, run: # -# pip-compile-multi +# requirements upgrade # -r test.txt -mypy==1.15.0 +mypy==1.17.0 # via -r mypy.in -mypy-extensions==1.0.0 +mypy-extensions==1.1.0 + # via mypy +pathspec==0.12.1 # via mypy # The following packages are considered to be unsafe in a requirements file: diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index c0bad079..a2f5be19 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -1,16 +1,16 @@ # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! -dask +dask>=2022.1.0 essreduce>=24.03.0 graphviz -pooch -pandas -gemmi -defusedxml -bitshuffle -msgpack -pytest +pooch>=1.5 +pandas>=2.1.2 +gemmi>=0.6.6 +defusedxml>=0.7.1 +bitshuffle>=0.5.2 +msgpack>=1.0.8 +pytest>=7.0 scipp --index-url=https://pypi.anaconda.org/scipp-nightly-wheels/simple/ --extra-index-url=https://pypi.org/simple diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 1b275814..72138814 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,9 +1,9 @@ -# SHA1:894ecb8cad4ccdb545bbd81b2215094550fa9ccc +# SHA1:3aa4a12a5ca5339b013c64f1d999b73adb5b3ca1 # -# This file is autogenerated by pip-compile-multi +# This file was generated by pip-compile-multi. # To update, run: # -# pip-compile-multi +# requirements upgrade # --index-url https://pypi.anaconda.org/scipp-nightly-wheels/simple/ --extra-index-url https://pypi.org/simple @@ -12,23 +12,23 @@ annotated-types==0.7.0 # via pydantic bitshuffle==0.5.2 # via -r nightly.in -certifi==2025.1.31 +certifi==2025.7.14 # via requests -charset-normalizer==3.4.1 +charset-normalizer==3.4.2 # via requests -click==8.1.8 +click==8.2.1 # via dask cloudpickle==3.1.1 # via dask -contourpy==1.3.1 +contourpy==1.3.2 # via matplotlib cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -cython==3.1.0b1 +cython==3.1.2 # via bitshuffle -dask==2025.3.0 +dask==2025.7.0 # via -r nightly.in defusedxml==0.8.0rc2 # via -r nightly.in @@ -36,19 +36,19 @@ dnspython==2.7.0 # via email-validator email-validator==2.2.0 # via scippneutron -essreduce==25.3.1 +essreduce==25.7.1 # via -r nightly.in -exceptiongroup==1.2.2 +exceptiongroup==1.3.0 # via pytest -fonttools==4.57.0 +fonttools==4.59.0 # via matplotlib -fsspec==2025.3.2 +fsspec==2025.7.0 # via dask -gemmi==0.7.1 +gemmi==0.7.3 # via -r nightly.in -graphviz==0.20.3 +graphviz==0.21 # via -r nightly.in -h5py==3.13.0 +h5py==3.14.0 # via # bitshuffle # scippneutron @@ -57,7 +57,7 @@ idna==3.10 # via # email-validator # requests -importlib-metadata==8.6.1 +importlib-metadata==8.7.0 # via dask iniconfig==2.1.0 # via pytest @@ -69,57 +69,58 @@ lazy-loader==0.4 # scippneutron locket==1.0.0 # via partd -matplotlib==3.10.1 +matplotlib==3.10.3 # via # mpltoolbox # plopp -mpltoolbox==24.5.1 +mpltoolbox==25.5.0 # via scippneutron -msgpack==1.1.0 +msgpack==1.1.1 # via -r nightly.in networkx==3.4.2 # via cyclebane -numpy==2.2.4 +numpy==2.2.6 # via # bitshuffle # contourpy # h5py # matplotlib - # mpltoolbox # pandas # scipp # scippneutron # scipy -packaging==24.2 +packaging==25.0 # via # dask # lazy-loader # matplotlib # pooch # pytest -pandas==2.2.3 +pandas==2.3.1 # via -r nightly.in partd==1.4.2 # via dask -pillow==11.1.0 +pillow==11.3.0 # via matplotlib -platformdirs==4.3.7 +platformdirs==4.3.8 # via pooch plopp @ git+https://github.com/scipp/plopp@main # via # -r nightly.in # scippneutron -pluggy==1.5.0 +pluggy==1.6.0 # via pytest pooch==1.8.2 # via -r nightly.in -pydantic==2.11.2 +pydantic==2.11.7 # via scippneutron -pydantic-core==2.33.1 +pydantic-core==2.33.2 # via pydantic +pygments==2.19.2 + # via pytest pyparsing==3.2.3 # via matplotlib -pytest==8.3.5 +pytest==8.4.1 # via -r nightly.in python-dateutil==2.9.0.post0 # via @@ -131,7 +132,7 @@ pytz==2025.2 # via pandas pyyaml==6.0.2 # via dask -requests==2.32.3 +requests==2.32.4 # via pooch sciline @ git+https://github.com/scipp/sciline@main # via @@ -143,14 +144,14 @@ scipp==100.0.0.dev0 # essreduce # scippneutron # scippnexus -scippneutron==25.2.1 +scippneutron==25.7.0 # via essreduce scippnexus @ git+https://github.com/scipp/scippnexus@main # via # -r nightly.in # essreduce # scippneutron -scipy==1.15.2 +scipy==1.15.3 # via # scippneutron # scippnexus @@ -162,18 +163,20 @@ toolz==1.0.0 # via # dask # partd -typing-extensions==4.13.1 +typing-extensions==4.14.1 # via + # exceptiongroup # pydantic # pydantic-core + # sciline # typing-inspection -typing-inspection==0.4.0 +typing-inspection==0.4.1 # via pydantic tzdata==2025.2 # via pandas -urllib3==2.3.0 +urllib3==2.5.0 # via requests -zipp==3.21.0 +zipp==3.23.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index c0e73660..7dbd6032 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -1,9 +1,9 @@ # SHA1:5a0b1bb22ae805d8aebba0f3bf05ab91aceae0d8 # -# This file is autogenerated by pip-compile-multi +# This file was generated by pip-compile-multi. # To update, run: # -# pip-compile-multi +# requirements upgrade # cfgv==3.4.0 # via pre-commit @@ -11,15 +11,15 @@ distlib==0.3.9 # via virtualenv filelock==3.18.0 # via virtualenv -identify==2.6.9 +identify==2.6.12 # via pre-commit nodeenv==1.9.1 # via pre-commit -platformdirs==4.3.7 +platformdirs==4.3.8 # via virtualenv pre-commit==4.2.0 # via -r static.in pyyaml==6.0.2 # via pre-commit -virtualenv==20.30.0 +virtualenv==20.31.2 # via pre-commit diff --git a/packages/essnmx/requirements/test.txt b/packages/essnmx/requirements/test.txt index 7e9f65fe..c900374a 100644 --- a/packages/essnmx/requirements/test.txt +++ b/packages/essnmx/requirements/test.txt @@ -1,9 +1,9 @@ # SHA1:ef2ee9576d8a9e65b44e2865a26887eed3fc49d1 # -# This file is autogenerated by pip-compile-multi +# This file was generated by pip-compile-multi. # To update, run: # -# pip-compile-multi +# requirements upgrade # -r base.txt -r basetest.txt diff --git a/packages/essnmx/requirements/wheels.txt b/packages/essnmx/requirements/wheels.txt index bfae20bf..651191e5 100644 --- a/packages/essnmx/requirements/wheels.txt +++ b/packages/essnmx/requirements/wheels.txt @@ -1,13 +1,13 @@ # SHA1:80754af91bfb6d1073585b046fe0a474ce868509 # -# This file is autogenerated by pip-compile-multi +# This file was generated by pip-compile-multi. # To update, run: # -# pip-compile-multi +# requirements upgrade # build==1.2.2.post1 # via -r wheels.in -packaging==24.2 +packaging==25.0 # via build pyproject-hooks==1.2.0 # via build From 60157393877214e1164e5b6aead172d941d8eef8 Mon Sep 17 00:00:00 2001 From: Mridul Seth Date: Wed, 6 Aug 2025 14:15:46 +0200 Subject: [PATCH 265/403] Drop python3.10, bump copier and tox (#148) * copier update * tox -e deps * remove conflict --- packages/essnmx/.copier-answers.yml | 4 +- .../.github/workflows/python-version-ci | 2 +- packages/essnmx/.github/workflows/release.yml | 44 +----------- packages/essnmx/.pre-commit-config.yaml | 1 - packages/essnmx/.python-version | 2 +- packages/essnmx/README.md | 2 +- packages/essnmx/conda/meta.yaml | 67 ------------------- packages/essnmx/docs/conf.py | 5 +- .../essnmx/docs/developer/getting-started.md | 2 +- .../essnmx/docs/user-guide/installation.md | 2 +- packages/essnmx/pyproject.toml | 3 +- packages/essnmx/requirements/base.txt | 14 ++-- packages/essnmx/requirements/basetest.txt | 6 -- packages/essnmx/requirements/ci.txt | 16 ++--- packages/essnmx/requirements/dev.txt | 18 +++-- packages/essnmx/requirements/docs.txt | 20 +++--- packages/essnmx/requirements/mypy.txt | 2 +- packages/essnmx/requirements/nightly.txt | 22 +++--- packages/essnmx/requirements/static.txt | 4 +- packages/essnmx/requirements/wheels.txt | 4 +- packages/essnmx/tox.ini | 2 +- 21 files changed, 59 insertions(+), 183 deletions(-) delete mode 100644 packages/essnmx/conda/meta.yaml diff --git a/packages/essnmx/.copier-answers.yml b/packages/essnmx/.copier-answers.yml index 2036def7..dc6b394f 100644 --- a/packages/essnmx/.copier-answers.yml +++ b/packages/essnmx/.copier-answers.yml @@ -1,9 +1,9 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY -_commit: 3561fcd +_commit: 024a41b _src_path: gh:scipp/copier_template description: Data reduction for NMX at the European Spallation Source. max_python: '3.13' -min_python: '3.10' +min_python: '3.11' namespace_package: ess nightly_deps: scipp,sciline,scippnexus,plopp orgname: scipp diff --git a/packages/essnmx/.github/workflows/python-version-ci b/packages/essnmx/.github/workflows/python-version-ci index c8cfe395..2c073331 100644 --- a/packages/essnmx/.github/workflows/python-version-ci +++ b/packages/essnmx/.github/workflows/python-version-ci @@ -1 +1 @@ -3.10 +3.11 diff --git a/packages/essnmx/.github/workflows/release.yml b/packages/essnmx/.github/workflows/release.yml index f88b3447..66b8a942 100644 --- a/packages/essnmx/.github/workflows/release.yml +++ b/packages/essnmx/.github/workflows/release.yml @@ -10,29 +10,6 @@ defaults: shell: bash -l {0} # required for conda env jobs: - build_conda: - name: Conda build - runs-on: 'ubuntu-24.04' - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 0 # history required so setuptools_scm can determine version - - - uses: mamba-org/setup-micromamba@v1 - with: - environment-name: build-env - create-args: >- - conda-build - boa - - run: conda mambabuild --channel conda-forge --channel scipp --no-anaconda-upload --override-channels --output-folder conda/package conda - - - uses: actions/upload-artifact@v4 - with: - name: conda-package-noarch - path: conda/package/noarch/*.tar.bz2 - build_wheels: name: Wheels runs-on: 'ubuntu-24.04' @@ -60,7 +37,7 @@ jobs: upload_pypi: name: Deploy PyPI - needs: [build_wheels, build_conda] + needs: [build_wheels] runs-on: 'ubuntu-24.04' environment: release permissions: @@ -70,25 +47,8 @@ jobs: - uses: actions/download-artifact@v4 - uses: pypa/gh-action-pypi-publish@v1.12.4 - upload_conda: - name: Deploy Conda - needs: [build_wheels, build_conda] - runs-on: 'ubuntu-24.04' - if: github.event_name == 'release' && github.event.action == 'published' - - steps: - - uses: actions/download-artifact@v4 - - uses: mamba-org/setup-micromamba@v1 - with: - environment-name: upload-env - # frozen python due to breaking removal of 'imp' in 3.12 - create-args: >- - anaconda-client - python=3.11 - - run: anaconda --token ${{ secrets.ANACONDATOKEN }} upload --user scipp --label main $(ls conda-package-noarch/*.tar.bz2) - docs: - needs: [upload_conda, upload_pypi] + needs: [upload_pypi] uses: ./.github/workflows/docs.yml with: publish: ${{ github.event_name == 'release' && github.event.action == 'published' }} diff --git a/packages/essnmx/.pre-commit-config.yaml b/packages/essnmx/.pre-commit-config.yaml index 4f913e71..045e4a46 100644 --- a/packages/essnmx/.pre-commit-config.yaml +++ b/packages/essnmx/.pre-commit-config.yaml @@ -10,7 +10,6 @@ repos: - id: check-merge-conflict - id: check-toml - id: check-yaml - exclude: conda/meta.yaml - id: detect-private-key - id: trailing-whitespace args: [ --markdown-linebreak-ext=md ] diff --git a/packages/essnmx/.python-version b/packages/essnmx/.python-version index c8cfe395..2c073331 100644 --- a/packages/essnmx/.python-version +++ b/packages/essnmx/.python-version @@ -1 +1 @@ -3.10 +3.11 diff --git a/packages/essnmx/README.md b/packages/essnmx/README.md index 85269345..a95ecc59 100644 --- a/packages/essnmx/README.md +++ b/packages/essnmx/README.md @@ -1,6 +1,6 @@ [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](CODE_OF_CONDUCT.md) [![PyPI badge](http://img.shields.io/pypi/v/essnmx.svg)](https://pypi.python.org/pypi/essnmx) -[![Anaconda-Server Badge](https://anaconda.org/scipp/essnmx/badges/version.svg)](https://anaconda.org/scipp/essnmx) +[![Anaconda-Server Badge](https://anaconda.org/conda-forge/essnmx/badges/version.svg)](https://anaconda.org/conda-forge/essnmx) [![License: BSD 3-Clause](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](LICENSE) # ESSnmx diff --git a/packages/essnmx/conda/meta.yaml b/packages/essnmx/conda/meta.yaml deleted file mode 100644 index 08c4160d..00000000 --- a/packages/essnmx/conda/meta.yaml +++ /dev/null @@ -1,67 +0,0 @@ -package: - name: essnmx - - version: {{ GIT_DESCRIBE_TAG }} - -source: - path: .. - - -{% set pyproject = load_file_data('pyproject.toml') %} -{% set dependencies = pyproject.get('project', {}).get('dependencies', {}) %} -{% set test_dependencies = pyproject.get('project', {}).get('optional-dependencies', {}).get('test', {}) %} - - -requirements: - build: - - setuptools - - setuptools_scm - run: - - dask - - python-graphviz - - plopp - - sciline>=23.9.1 - - scipp>=23.8.0 - - scippnexus>=23.9.0 - - pooch - - defusedxml - - python>=3.10 - - gemmi - - pandas - - {# Conda does not allow spaces between package name and version, so remove them #} - {% for package in dependencies %} - - {% if package == "graphviz" %}python-graphviz{% else %}{{ package|replace(" ", "") }}{% endif %} - {% endfor %} - - -test: - imports: - - ess.nmx - requires: - - {# Conda does not allow spaces between package name and version, so remove them #} - {% for package in test_dependencies %} - - {% if package == "graphviz" %}python-graphviz{% else %}{{ package|replace(" ", "") }}{% endif %} - {% endfor %} - - - source_files: - - pyproject.toml - - tests/ - commands: - # We ignore warnings during release package builds - - python -m pytest -Wignore tests - -build: - noarch: python - script: - - python -m pip install . - -about: - home: https://github.com/scipp/essnmx - license: BSD-3-Clause - summary: Data reduction for NMX at the European Spallation Source. - description: Data reduction for NMX at the European Spallation Source. - dev_url: https://github.com/scipp/essnmx - doc_url: https://scipp.github.io/essnmx diff --git a/packages/essnmx/docs/conf.py b/packages/essnmx/docs/conf.py index 6144897c..67f6fc3f 100644 --- a/packages/essnmx/docs/conf.py +++ b/packages/essnmx/docs/conf.py @@ -187,7 +187,7 @@ }, { "name": "Conda", - "url": "https://anaconda.org/scipp/essnmx", + "url": "https://anaconda.org/conda-forge/essnmx", "icon": "fa-custom fa-anaconda", "type": "fontawesome", }, @@ -267,5 +267,6 @@ def do_not_plot(*args, **kwargs): r'https?://github\.com/.*?/blob/[a-f0-9]+/.+?#', # Linkcheck seems to be denied access by some DOI resolvers. # Since DOIs are supposed to be permanent, we don't need to check them.' - r'https://doi\.org/', + r'https?://doi\.org/', + r'https?://dx\.doi\.org/', ] diff --git a/packages/essnmx/docs/developer/getting-started.md b/packages/essnmx/docs/developer/getting-started.md index a196f562..a7667511 100644 --- a/packages/essnmx/docs/developer/getting-started.md +++ b/packages/essnmx/docs/developer/getting-started.md @@ -40,7 +40,7 @@ Alternatively, if you want a different workflow, take a look at ``tox.ini`` or ` Run the tests using ```sh -tox -e py310 +tox -e py311 ``` (or just `tox` if you want to run all environments). diff --git a/packages/essnmx/docs/user-guide/installation.md b/packages/essnmx/docs/user-guide/installation.md index 412db9dc..f29e78c9 100644 --- a/packages/essnmx/docs/user-guide/installation.md +++ b/packages/essnmx/docs/user-guide/installation.md @@ -10,7 +10,7 @@ pip install essnmx ```` ````{tab-item} conda ```sh -conda install -c conda-forge -c scipp essnmx +conda install -c conda-forge essnmx ``` ```` ````` diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 6620b91b..bcf70f22 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -18,14 +18,13 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Topic :: Scientific/Engineering", "Typing :: Typed", ] -requires-python = ">=3.10" +requires-python = ">=3.11" # IMPORTANT: # Run 'tox -e deps' after making changes here. This will update requirement files. diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index a06d06c8..0eb5162a 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -9,7 +9,7 @@ annotated-types==0.7.0 # via pydantic bitshuffle==0.5.2 # via -r base.in -certifi==2025.7.14 +certifi==2025.8.3 # via requests charset-normalizer==3.4.2 # via requests @@ -17,7 +17,7 @@ click==8.2.1 # via dask cloudpickle==3.1.1 # via dask -contourpy==1.3.2 +contourpy==1.3.3 # via matplotlib cyclebane==24.10.0 # via sciline @@ -62,7 +62,7 @@ lazy-loader==0.4 # scippneutron locket==1.0.0 # via partd -matplotlib==3.10.3 +matplotlib==3.10.5 # via # mpltoolbox # plopp @@ -70,9 +70,9 @@ mpltoolbox==25.5.0 # via scippneutron msgpack==1.1.1 # via -r base.in -networkx==3.4.2 +networkx==3.5 # via cyclebane -numpy==2.2.6 +numpy==2.3.2 # via # bitshuffle # contourpy @@ -96,7 +96,7 @@ pillow==11.3.0 # via matplotlib platformdirs==4.3.8 # via pooch -plopp==25.7.0 +plopp==25.7.1 # via # -r base.in # scippneutron @@ -137,7 +137,7 @@ scippnexus==25.6.0 # -r base.in # essreduce # scippneutron -scipy==1.15.3 +scipy==1.16.1 # via # scippneutron # scippnexus diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt index 464e50dc..e37fea84 100644 --- a/packages/essnmx/requirements/basetest.txt +++ b/packages/essnmx/requirements/basetest.txt @@ -5,8 +5,6 @@ # # requirements upgrade # -exceptiongroup==1.3.0 - # via pytest iniconfig==2.1.0 # via pytest packaging==25.0 @@ -17,7 +15,3 @@ pygments==2.19.2 # via pytest pytest==8.4.1 # via -r basetest.in -tomli==2.2.1 - # via pytest -typing-extensions==4.14.1 - # via exceptiongroup diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index 629701a6..cf385f77 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -7,7 +7,7 @@ # cachetools==6.1.0 # via tox -certifi==2025.7.14 +certifi==2025.8.3 # via requests chardet==5.2.0 # via tox @@ -15,7 +15,7 @@ charset-normalizer==3.4.2 # via requests colorama==0.4.6 # via tox -distlib==0.3.9 +distlib==0.4.0 # via virtualenv filelock==3.18.0 # via @@ -23,7 +23,7 @@ filelock==3.18.0 # virtualenv gitdb==4.0.12 # via gitpython -gitpython==3.1.44 +gitpython==3.1.45 # via -r ci.in idna==3.10 # via requests @@ -44,15 +44,9 @@ requests==2.32.4 # via -r ci.in smmap==5.0.2 # via gitdb -tomli==2.2.1 - # via - # pyproject-api - # tox -tox==4.27.0 +tox==4.28.4 # via -r ci.in -typing-extensions==4.14.1 - # via tox urllib3==2.5.0 # via requests -virtualenv==20.31.2 +virtualenv==20.33.1 # via tox diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index 290f4dcc..94aff0a8 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -12,13 +12,13 @@ -r static.txt -r test.txt -r wheels.txt -anyio==4.9.0 +anyio==4.10.0 # via # httpx # jupyter-server argon2-cffi==25.1.0 # via jupyter-server -argon2-cffi-bindings==21.2.0 +argon2-cffi-bindings==25.1.0 # via argon2-cffi arrow==1.3.0 # via isoduration @@ -26,7 +26,7 @@ async-lru==2.0.5 # via jupyterlab cffi==1.17.1 # via argon2-cffi-bindings -copier==9.8.0 +copier==9.9.0 # via -r dev.in dunamai==1.25.0 # via copier @@ -48,14 +48,14 @@ json5==0.12.0 # via jupyterlab-server jsonpointer==3.0.0 # via jsonschema -jsonschema[format-nongpl]==4.24.0 +jsonschema[format-nongpl]==4.25.0 # via # jupyter-events # jupyterlab-server # nbformat jupyter-events==0.12.0 # via jupyter-server -jupyter-lsp==2.2.5 +jupyter-lsp==2.2.6 # via jupyterlab jupyter-server==2.16.0 # via @@ -65,17 +65,19 @@ jupyter-server==2.16.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.4.4 +jupyterlab==4.4.5 # via -r dev.in jupyterlab-server==2.27.3 # via jupyterlab +lark==1.2.2 + # via rfc3987-syntax notebook-shim==0.2.4 # via jupyterlab overrides==7.7.0 # via jupyter-server pip-compile-multi==3.2.1 # via -r dev.in -pip-tools==7.4.1 +pip-tools==7.5.0 # via pip-compile-multi plumbum==1.9.0 # via copier @@ -95,6 +97,8 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events +rfc3987-syntax==1.1.0 + # via jsonschema send2trash==1.8.3 # via jupyter-server sniffio==1.3.1 diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 9025cbb5..4535f7c0 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -10,6 +10,8 @@ accessible-pygments==0.0.5 # via pydata-sphinx-theme alabaster==1.0.0 # via sphinx +appnope==0.1.4 + # via ipykernel asttokens==3.0.0 # via stack-data attrs==25.3.0 @@ -28,7 +30,7 @@ beautifulsoup4==4.13.4 # pydata-sphinx-theme bleach[css]==6.2.0 # via nbconvert -comm==0.2.2 +comm==0.2.3 # via # ipykernel # ipywidgets @@ -42,8 +44,6 @@ docutils==0.21.2 # nbsphinx # pydata-sphinx-theme # sphinx -exceptiongroup==1.3.0 - # via ipython executing==2.2.0 # via stack-data fastjsonschema==2.21.1 @@ -52,13 +52,15 @@ imagesize==1.4.1 # via sphinx ipydatawidgets==4.3.5 # via pythreejs -ipykernel==6.29.5 +ipykernel==6.30.1 # via -r docs.in -ipython==8.37.0 +ipython==9.4.0 # via # -r docs.in # ipykernel # ipywidgets +ipython-pygments-lexers==1.1.1 + # via ipython ipywidgets==8.1.7 # via # ipydatawidgets @@ -71,7 +73,7 @@ jinja2==3.1.6 # nbconvert # nbsphinx # sphinx -jsonschema==4.24.0 +jsonschema==4.25.0 # via nbformat jsonschema-specifications==2025.4.1 # via jsonschema @@ -145,6 +147,7 @@ pygments==2.19.2 # via # accessible-pygments # ipython + # ipython-pygments-lexers # nbconvert # pydata-sphinx-theme # sphinx @@ -152,7 +155,7 @@ python-dotenv==1.1.1 # via pydantic-settings pythreejs==2.4.2 # via -r docs.in -pyzmq==27.0.0 +pyzmq==27.0.1 # via # ipykernel # jupyter-client @@ -200,15 +203,12 @@ stack-data==0.6.3 # via ipython tinycss2==1.4.0 # via bleach -tomli==2.2.1 - # via sphinx tornado==6.5.1 # via # ipykernel # jupyter-client traitlets==5.14.3 # via - # comm # ipykernel # ipython # ipywidgets diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index 99a887e2..0c527ae4 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -6,7 +6,7 @@ # requirements upgrade # -r test.txt -mypy==1.17.0 +mypy==1.17.1 # via -r mypy.in mypy-extensions==1.1.0 # via mypy diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 72138814..c5604ce6 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -12,7 +12,7 @@ annotated-types==0.7.0 # via pydantic bitshuffle==0.5.2 # via -r nightly.in -certifi==2025.7.14 +certifi==2025.8.3 # via requests charset-normalizer==3.4.2 # via requests @@ -20,7 +20,7 @@ click==8.2.1 # via dask cloudpickle==3.1.1 # via dask -contourpy==1.3.2 +contourpy==1.3.3 # via matplotlib cyclebane==24.10.0 # via sciline @@ -38,8 +38,6 @@ email-validator==2.2.0 # via scippneutron essreduce==25.7.1 # via -r nightly.in -exceptiongroup==1.3.0 - # via pytest fonttools==4.59.0 # via matplotlib fsspec==2025.7.0 @@ -69,7 +67,7 @@ lazy-loader==0.4 # scippneutron locket==1.0.0 # via partd -matplotlib==3.10.3 +matplotlib==3.10.5 # via # mpltoolbox # plopp @@ -77,9 +75,9 @@ mpltoolbox==25.5.0 # via scippneutron msgpack==1.1.1 # via -r nightly.in -networkx==3.4.2 +networkx==3.5 # via cyclebane -numpy==2.2.6 +numpy==2.3.2 # via # bitshuffle # contourpy @@ -112,9 +110,9 @@ pluggy==1.6.0 # via pytest pooch==1.8.2 # via -r nightly.in -pydantic==2.11.7 +pydantic==2.12.0a1 # via scippneutron -pydantic-core==2.33.2 +pydantic-core==2.37.2 # via pydantic pygments==2.19.2 # via pytest @@ -127,7 +125,6 @@ python-dateutil==2.9.0.post0 # matplotlib # pandas # scippneutron - # scippnexus pytz==2025.2 # via pandas pyyaml==6.0.2 @@ -151,21 +148,18 @@ scippnexus @ git+https://github.com/scipp/scippnexus@main # -r nightly.in # essreduce # scippneutron -scipy==1.15.3 +scipy==1.16.1 # via # scippneutron # scippnexus six==1.17.0 # via python-dateutil -tomli==2.2.1 - # via pytest toolz==1.0.0 # via # dask # partd typing-extensions==4.14.1 # via - # exceptiongroup # pydantic # pydantic-core # sciline diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index 7dbd6032..3e288557 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -7,7 +7,7 @@ # cfgv==3.4.0 # via pre-commit -distlib==0.3.9 +distlib==0.4.0 # via virtualenv filelock==3.18.0 # via virtualenv @@ -21,5 +21,5 @@ pre-commit==4.2.0 # via -r static.in pyyaml==6.0.2 # via pre-commit -virtualenv==20.31.2 +virtualenv==20.33.1 # via pre-commit diff --git a/packages/essnmx/requirements/wheels.txt b/packages/essnmx/requirements/wheels.txt index 651191e5..3558aae2 100644 --- a/packages/essnmx/requirements/wheels.txt +++ b/packages/essnmx/requirements/wheels.txt @@ -5,11 +5,9 @@ # # requirements upgrade # -build==1.2.2.post1 +build==1.3.0 # via -r wheels.in packaging==25.0 # via build pyproject-hooks==1.2.0 # via build -tomli==2.2.1 - # via build diff --git a/packages/essnmx/tox.ini b/packages/essnmx/tox.ini index b11d187c..bc04863e 100644 --- a/packages/essnmx/tox.ini +++ b/packages/essnmx/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py310,py311,py312} +envlist = py311 isolated_build = true [testenv] From a60d8af303cebba288f6fbd300e95ba737717136 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Sep 2025 07:53:56 +0000 Subject: [PATCH 266/403] Bump scipp from 25.5.1 to 25.8.0 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 25.5.1 to 25.8.0. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/25.05.1...25.08.0) --- updated-dependencies: - dependency-name: scipp dependency-version: 25.8.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 0eb5162a..44b2e1b2 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -124,7 +124,7 @@ sciline==25.5.2 # via # -r base.in # essreduce -scipp==25.5.1 +scipp==25.8.0 # via # -r base.in # essreduce From 3f6feb5fc4f3db5d4f439b3aba193bd2266dd5bf Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 4 Nov 2025 13:49:43 +0100 Subject: [PATCH 267/403] Reduction workflow for ess nmx files (#146) * Extract common part of command line helper. * Extract chunk size argument since it means different numbers in real reduction. * Add fallback compute position helper. * Update fallback compute position function. * Handle already folded data. * Add executable module for real data. * Return all result at the end. * Remove unecessary comment [skip ci] * Allow setting time bin edge from arguments. * Add docstring [skip ci] * Fix fast axis decision function and update compression option argument type * Remove unecessary copy and add logging. * Hardcode event time offset unit and make chunk size default to -1 * Apply scipp lower pin. * fixed unit issue on even_time_offset * nbins to be a number of bins instead of number of bin-edges [skip cip] * Use enum instead of integer. * "--compression" arg should be a string * Add comment about tof/wavelength * Add small nexus file for testing. * Add command line CLI testing. * Add command line CLI testing. * Test with non-default values. --------- Co-authored-by: Aaron Finke --- packages/essnmx/pyproject.toml | 3 +- .../essnmx/src/ess/nmx/_executable_helper.py | 59 +++ packages/essnmx/src/ess/nmx/data/__init__.py | 7 + packages/essnmx/src/ess/nmx/executables.py | 402 ++++++++++++++++++ .../essnmx/src/ess/nmx/mcstas/executables.py | 76 ++-- packages/essnmx/src/ess/nmx/nexus.py | 85 +++- packages/essnmx/src/ess/nmx/types.py | 11 + packages/essnmx/tests/executable_test.py | 72 ++++ 8 files changed, 659 insertions(+), 56 deletions(-) create mode 100644 packages/essnmx/src/ess/nmx/_executable_helper.py create mode 100644 packages/essnmx/src/ess/nmx/executables.py create mode 100644 packages/essnmx/tests/executable_test.py diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index bcf70f22..4535426d 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "graphviz", "plopp>=24.7.0", "sciline>=24.06.0", - "scipp>=23.8.0", + "scipp>=25.3.0", "scippnexus>=23.12.0", "pooch>=1.5", "pandas>=2.1.2", @@ -49,6 +49,7 @@ dynamic = ["version"] [project.scripts] essnmx_reduce_mcstas = "ess.nmx.mcstas.executables:main" +essnmx-reduce = "ess.nmx.executables:main" [project.optional-dependencies] test = [ diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py new file mode 100644 index 00000000..bf8aec32 --- /dev/null +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2025 Scipp contributors (https://github.com/scipp) +import argparse +import logging +import sys + +from .types import Compression + + +def build_reduction_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Command line arguments for the NMX reduction. " + "It assumes 14 Hz pulse speed." + ) + input_arg_group = parser.add_argument_group("Input Options") + input_arg_group.add_argument( + "--input_file", type=str, help="Path to the input file", required=True + ) + input_arg_group.add_argument( + "--nbins", + type=int, + default=50, + help="Number of TOF bins", + ) + input_arg_group.add_argument( + "--detector_ids", + type=int, + nargs="+", + default=[0, 1, 2], + help="Detector indices to process", + ) + + output_arg_group = parser.add_argument_group("Output Options") + output_arg_group.add_argument( + "--output_file", + type=str, + default="scipp_output.h5", + help="Path to the output file", + ) + output_arg_group.add_argument( + "--compression", + type=str, + default=Compression.BITSHUFFLE_LZ4.name, + choices=[compression_key.name for compression_key in Compression], + help="Compress option of reduced output file. Default: BITSHUFFLE_LZ4", + ) + output_arg_group.add_argument( + "--verbose", "-v", action="store_true", help="Increase output verbosity" + ) + + return parser + + +def build_logger(args: argparse.Namespace) -> logging.Logger: + logger = logging.getLogger(__name__) + if args.verbose: + logger.setLevel(logging.INFO) + logger.addHandler(logging.StreamHandler(sys.stdout)) + return logger diff --git a/packages/essnmx/src/ess/nmx/data/__init__.py b/packages/essnmx/src/ess/nmx/data/__init__.py index f8803dbb..36919908 100644 --- a/packages/essnmx/src/ess/nmx/data/__init__.py +++ b/packages/essnmx/src/ess/nmx/data/__init__.py @@ -21,6 +21,7 @@ def _make_pooch() -> pooch.Pooch: "small_mcstas_3_sample.h5": "md5:2afaac205d13ee857ee5364e3f1957a7", "mtz_samples.tar.gz": "md5:bed1eaf604bbe8725c1f6a20ca79fcc0", "mtz_random_samples.tar.gz": "md5:c8259ae2e605560ab88959e7109613b6", + "small_nmx_nexus.hdf": "md5:42cffb85e4ce7c1aaa5f7e81469b865e", }, ) @@ -89,3 +90,9 @@ def get_small_random_mtz_samples() -> list[pathlib.Path]: pathlib.Path(file_path) for file_path in _pooch.fetch("mtz_random_samples.tar.gz", processor=Untar()) ] + + +def get_small_nmx_nexus() -> str: + """Return the path to a small NMX NeXus file.""" + + return _pooch.fetch("small_nmx_nexus.hdf") diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py new file mode 100644 index 00000000..61d8e634 --- /dev/null +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -0,0 +1,402 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2025 Scipp contributors (https://github.com/scipp) +import argparse +import logging +import pathlib +from collections.abc import Callable +from dataclasses import dataclass +from typing import Literal + +import scipp as sc +import scippnexus as snx + +from .nexus import ( + _compute_positions, + _export_detector_metadata_as_nxlauetof, + _export_reduced_data_as_nxlauetof, + _export_static_metadata_as_nxlauetof, +) +from .streaming import _validate_chunk_size +from .types import Compression, NMXDetectorMetadata, NMXExperimentMetadata + + +def _retrieve_source_position(file: snx.File) -> sc.Variable: + da = file['entry/instrument/source'][()] + return _compute_positions(da, auto_fix_transformations=True)['position'] + + +def _retrieve_sample_position(file: snx.File) -> sc.Variable: + da = file['entry/sample'][()] + return _compute_positions(da, auto_fix_transformations=True)['position'] + + +def _decide_fast_axis(da: sc.DataArray) -> str: + x_slice = da['x_pixel_offset', 0].coords['detector_number'] + y_slice = da['y_pixel_offset', 0].coords['detector_number'] + + if (x_slice.max() < y_slice.max()).value: + return 'y' + elif (x_slice.max() > y_slice.max()).value: + return 'x' + else: + raise ValueError( + "Cannot decide fast axis based on pixel offsets. " + "Please specify the fast axis explicitly." + ) + + +def _decide_step(offsets: sc.Variable) -> sc.Variable: + """Decide the step size based on the offsets assuming at least 2 values.""" + sorted_offsets = sc.sort(offsets, key=offsets.dim, order='ascending') + return sorted_offsets[1] - sorted_offsets[0] + + +@dataclass +class DetectorDesc: + """Detector information extracted from McStas instrument xml description.""" + + name: str + id_start: int # 'idstart' + num_x: int # 'xpixels' + num_y: int # 'ypixels' + step_x: sc.Variable # 'xstep' + step_y: sc.Variable # 'ystep' + start_x: float # 'xstart' + start_y: float # 'ystart' + position: sc.Variable # 'x', 'y', 'z' + # Calculated fields + rotation_matrix: sc.Variable + fast_axis_name: str + slow_axis_name: str + fast_axis: sc.Variable + slow_axis: sc.Variable + + +def build_detector_desc( + name: str, dg: sc.DataGroup, *, fast_axis: Literal['x', 'y'] | None = None +) -> DetectorDesc: + da: sc.DataArray = dg['data'] + _fast_axis = fast_axis if fast_axis is not None else _decide_fast_axis(da) + transformation_matrix = dg['transform_matrix'] + t_unit = transformation_matrix.unit + fast_axis_vector = ( + sc.vector([1, 0, 0], unit=t_unit) + if _fast_axis == 'x' + else sc.vector([0, 1, 0], unit=t_unit) + ) + slow_axis_vector = ( + sc.vector([0, 1, 0], unit=t_unit) + if _fast_axis == 'x' + else sc.vector([1, 0, 0], unit=t_unit) + ) + return DetectorDesc( + name=name, + id_start=da.coords['detector_number'].min().value, + num_x=da.sizes['x_pixel_offset'], + num_y=da.sizes['y_pixel_offset'], + start_x=da.coords['x_pixel_offset'].min().value, + start_y=da.coords['y_pixel_offset'].min().value, + position=dg['position'], + rotation_matrix=dg['transform_matrix'], + fast_axis_name=_fast_axis, + slow_axis_name='x' if _fast_axis == 'y' else 'y', + fast_axis=fast_axis_vector, + slow_axis=slow_axis_vector, + step_x=_decide_step(da.coords['x_pixel_offset']), + step_y=_decide_step(da.coords['y_pixel_offset']), + ) + + +def calculate_number_of_chunks(detector_gr: snx.Group, *, chunk_size: int = 0) -> int: + _validate_chunk_size(chunk_size) + event_time_zero_size = detector_gr.sizes['event_time_zero'] + if chunk_size == -1: + return 1 # Read all at once + else: + return event_time_zero_size // chunk_size + int( + event_time_zero_size % chunk_size != 0 + ) + + +def build_toa_bin_edges( + *, + min_toa: sc.Variable | int = 0, + max_toa: sc.Variable | int = int((1 / 14) * 1_000), # Default for ESS NMX + toa_bin_edges: sc.Variable | int = 250, +) -> sc.Variable: + if isinstance(toa_bin_edges, sc.Variable): + return toa_bin_edges + elif isinstance(toa_bin_edges, int): + min_toa = sc.scalar(min_toa, unit='ms') if isinstance(min_toa, int) else min_toa + max_toa = sc.scalar(max_toa, unit='ms') if isinstance(max_toa, int) else max_toa + return sc.linspace( + dim='event_time_offset', + start=min_toa.value, + stop=max_toa.to(unit=min_toa.unit).value, + unit=min_toa.unit, + num=toa_bin_edges + 1, + ) + + +def reduction( + *, + input_file: pathlib.Path, + output_file: pathlib.Path, + chunk_size: int = 1_000, + detector_ids: list[int | str], + compression: Compression = Compression.BITSHUFFLE_LZ4, + logger: logging.Logger | None = None, + min_toa: sc.Variable | int = 0, + max_toa: sc.Variable | int = int((1 / 14) * 1_000), # Default for ESS NMX + toa_bin_edges: sc.Variable | int = 250, + fast_axis: Literal['x', 'y'] | None = None, # 'x', 'y', or None to auto-detect + display: Callable | None = None, # For Jupyter notebook display +) -> sc.DataGroup: + """Reduce NMX data from a Nexus file and export to NXLauetof(ESS NMX specific) file. + + This workflow is written as a flatten function without using sciline Pipeline. + It is because the first part of NMX reduction only requires + a few steps of processing and it is overkill to use a Pipeline or GenericWorkflow. + + We also do not apply frame unwrapping or pulse skipping here, + as it is not expected from NMX experiments. + + Frame unwrapping may be applied later on the result of this function if needed + however, then the whole range of `event_time_offset` should have been histogrammed + so that the unwrapping can be applied. + i.e. `min_toa` should be 0 and `max_toa` should be 1/14 seconds + for 14 Hz pulse frequency. + TODO: Implement tof/wavelength workflow for NMX. + + Parameters + ---------- + input_file: + Path to the input Nexus file containing NMX data. + output_file: + Path to the output file where reduced data will be saved. + chunk_size: + Number of pulses to process in each chunk. If <= 0, all data is processed + at once. It represents the number of event_time_zero entries to read at once. + detector_ids: + List of detector IDs (as integers or names) to process. + compression: + If True, the output data will be compressed. + logger: + Logger to use for logging messages. If None, a default logger is created. + min_toa: + Minimum time of arrival (TOA) in milliseconds. Default is 0 ms. + max_toa: + Maximum time of arrival (TOA) in milliseconds. Default is 1/14 seconds, + typical for ESS NMX. + toa_bin_edges: + Number of time of arrival (TOA) bin edges or a scipp Variable defining the + edges. Default is 250 edges. + display: + Callable for displaying messages, useful in Jupyter notebooks. If None, + defaults to logger.info. + + Returns + ------- + sc.DataGroup: + A DataGroup containing the reduced data for each selected detector. + + """ + import scippnexus as snx + + if logger is None: + logger = logging.getLogger(__name__) + if display is None: + display = logger.info + + toa_bin_edges = build_toa_bin_edges( + min_toa=min_toa, max_toa=max_toa, toa_bin_edges=toa_bin_edges + ) + with snx.File(input_file) as f: + intrument_group = f['entry/instrument'] + dets = intrument_group[snx.NXdetector] + detector_group_keys = list(dets.keys()) + display(f"Found NXdetectors: {detector_group_keys}") + detector_id_map = { + det_name: dets[det_name] + for i, det_name in enumerate(detector_group_keys) + if i in detector_ids or det_name in detector_ids + } + if len(detector_id_map) != len(detector_ids): + raise ValueError( + f"Requested detector ids {detector_ids} not found in the file.\n" + f"Found {detector_group_keys}\n" + f"Try using integer indices instead of names." + ) + display(f"Selected detectors: {list(detector_id_map.keys())}") + source_position = _retrieve_source_position(f) + sample_position = _retrieve_sample_position(f) + experiment_metadata = NMXExperimentMetadata( + sc.DataGroup( + { + # Placeholder for crystal rotation + 'crystal_rotation': sc.vector([0, 0, 0], unit='deg'), + 'sample_position': sample_position, + 'source_position': source_position, + 'sample_name': sc.scalar(f['entry/sample/name'][()]), + } + ) + ) + display(experiment_metadata) + display("Experiment metadata component:") + for name, component in experiment_metadata.items(): + display(f"{name}: {component}") + + _export_static_metadata_as_nxlauetof( + experiment_metadata=experiment_metadata, + output_file=output_file, + ) + detector_grs = {} + for det_name, det_group in detector_id_map.items(): + display(f"Processing {det_name}") + if chunk_size <= 0: + dg = det_group[()] + else: + # Slice the first chunk for metadata extraction + dg = det_group['event_time_zero', 0:chunk_size] + + display("Computing detector positions...") + display(dg := _compute_positions(dg, auto_fix_transformations=True)) + detector = build_detector_desc(det_name, dg, fast_axis=fast_axis) + detector_meta = sc.DataGroup( + { + 'fast_axis': detector.fast_axis, + 'slow_axis': detector.slow_axis, + 'origin_position': sc.vector([0, 0, 0], unit='m'), + 'position': detector.position, + 'detector_shape': sc.scalar( + ( + dg['data'].sizes['x_pixel_offset'], + dg['data'].sizes['y_pixel_offset'], + ) + ), + 'x_pixel_size': detector.step_x, + 'y_pixel_size': detector.step_y, + 'detector_name': sc.scalar(detector.name), + } + ) + _export_detector_metadata_as_nxlauetof( + NMXDetectorMetadata(detector_meta), output_file=output_file + ) + + da: sc.DataArray = dg['data'] + event_time_offset_unit = da.bins.coords['event_time_offset'].bins.unit + display("Event time offset unit: %s", event_time_offset_unit) + toa_bin_edges = toa_bin_edges.to(unit=event_time_offset_unit, copy=False) + if chunk_size <= 0: + counts = da.hist(event_time_offset=toa_bin_edges).rename_dims( + x_pixel_offset='x', y_pixel_offset='y', event_time_offset='t' + ) + counts.coords['t'] = counts.coords['event_time_offset'] + + else: + num_chunks = calculate_number_of_chunks( + det_group, chunk_size=chunk_size + ) + display(f"Number of chunks: {num_chunks}") + counts = da.hist(event_time_offset=toa_bin_edges).rename_dims( + x_pixel_offset='x', y_pixel_offset='y', event_time_offset='t' + ) + counts.coords['t'] = counts.coords['event_time_offset'] + for chunk_index in range(1, num_chunks): + cur_chunk = det_group[ + 'event_time_zero', + chunk_index * chunk_size : (chunk_index + 1) * chunk_size, + ] + display(f"Processing chunk {chunk_index + 1} of {num_chunks}") + cur_chunk = _compute_positions( + cur_chunk, auto_fix_transformations=True + ) + cur_counts = ( + cur_chunk['data'] + .hist(event_time_offset=toa_bin_edges) + .rename_dims( + x_pixel_offset='x', + y_pixel_offset='y', + event_time_offset='t', + ) + ) + cur_counts.coords['t'] = cur_counts.coords['event_time_offset'] + counts += cur_counts + display("Accumulated counts:") + display(counts.sum().data) + + dg = sc.DataGroup( + counts=counts, + detector_shape=detector_meta['detector_shape'], + detector_name=detector_meta['detector_name'], + ) + display("Final data group:") + display(dg) + display("Saving reduced data to Nexus file...") + _export_reduced_data_as_nxlauetof( + dg, + output_file=output_file, + compress_counts=(compression == Compression.NONE), + ) + detector_grs[det_name] = dg + + display("Reduction completed successfully.") + return sc.DataGroup(detector_grs) + + +def _add_ess_reduction_args(arg: argparse.ArgumentParser) -> None: + argument_group = arg.add_argument_group("ESS Reduction Options") + argument_group.add_argument( + "--chunk_size", + type=int, + default=-1, + help="Chunk size for processing (number of pulses per chunk).", + ) + argument_group.add_argument( + "--min-toa", + type=int, + default=0, + help="Minimum time of arrival (TOA) in ms.", + ) + argument_group.add_argument( + "--max-toa", + type=int, + default=int((1 / 14) * 1_000), + help="Maximum time of arrival (TOA) in ms.", + ) + argument_group.add_argument( + "--fast-axis", + type=str, + choices=['x', 'y', None], + default=None, + help="Specify the fast axis of the detector. If None, it will be determined " + "automatically based on the pixel offsets.", + ) + + +def main() -> None: + from ._executable_helper import build_logger, build_reduction_arg_parser + + parser = build_reduction_arg_parser() + _add_ess_reduction_args(parser) + args = parser.parse_args() + + input_file = pathlib.Path(args.input_file).resolve() + output_file = pathlib.Path(args.output_file).resolve() + logger = build_logger(args) + + logger.info("Input file: %s", input_file) + logger.info("Output file: %s", output_file) + + reduction( + input_file=input_file, + output_file=output_file, + chunk_size=args.chunk_size, + detector_ids=args.detector_ids, + compression=Compression[args.compression], + toa_bin_edges=args.nbins, + min_toa=sc.scalar(args.min_toa, unit='ms'), + max_toa=sc.scalar(args.max_toa, unit='ms'), + fast_axis=args.fast_axis, + logger=logger, + ) diff --git a/packages/essnmx/src/ess/nmx/mcstas/executables.py b/packages/essnmx/src/ess/nmx/mcstas/executables.py index 47abe419..14ec7f68 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/executables.py +++ b/packages/essnmx/src/ess/nmx/mcstas/executables.py @@ -3,7 +3,6 @@ import argparse import logging import pathlib -import sys from collections.abc import Callable from functools import partial @@ -24,6 +23,7 @@ ) from ..streaming import calculate_number_of_chunks from ..types import ( + Compression, DetectorIndex, DetectorName, FilePath, @@ -139,10 +139,10 @@ def reduction( input_file: pathlib.Path, output_file: pathlib.Path, chunk_size: int = 10_000_000, - nbins: int = 51, + nbins: int = 50, max_counts: int | None = None, detector_ids: list[int | str], - compression: bool = True, + compression: Compression = Compression.BITSHUFFLE_LZ4, wf: sl.Pipeline | None = None, logger: logging.Logger | None = None, toa_min_max_prob: tuple[float] | None = None, @@ -161,7 +161,10 @@ def reduction( logger.info("Metadata retrieved: %s", data_metadata) toa_bin_edges = sc.linspace( - dim='t', start=data_metadata.min_toa, stop=data_metadata.max_toa, num=nbins + dim='t', + start=data_metadata.min_toa, + stop=data_metadata.max_toa, + num=nbins + 1, ) scale_factor = mcstas_weight_to_probability_scalefactor( max_counts=wf.compute(MaximumCounts), @@ -173,7 +176,7 @@ def reduction( toa_min = sc.scalar(toa_min_max_prob[0], unit='s') toa_max = sc.scalar(toa_min_max_prob[1], unit='s') prob_max = sc.scalar(toa_min_max_prob[2]) - toa_bin_edges = sc.linspace(dim='t', start=toa_min, stop=toa_max, num=nbins) + toa_bin_edges = sc.linspace(dim='t', start=toa_min, stop=toa_max, num=nbins + 1) scale_factor = mcstas_weight_to_probability_scalefactor( max_counts=wf.compute(MaximumCounts), max_probability=prob_max, @@ -253,69 +256,44 @@ def reduction( result_list.append(result) if logger is not None: logger.info("Appending reduced data into the output file %s", output_file) + _export_reduced_data_as_nxlauetof( - result, output_file=output_file, compress_counts=compression + result, + output_file=output_file, + compress_counts=(compression == Compression.NONE), ) from ess.nmx.reduction import merge_panels return merge_panels(*result_list) -def main() -> None: - parser = argparse.ArgumentParser(description="McStas Data Reduction.") - parser.add_argument( - "--input_file", type=str, help="Path to the input file", required=True - ) - parser.add_argument( - "--output_file", - type=str, - default="scipp_output.h5", - help="Path to the output file", - ) - parser.add_argument( - "--verbose", action="store_true", help="Increase output verbosity" - ) - parser.add_argument( - "--chunk_size", - type=int, - default=10_000_000, - help="Chunk size for processing", - ) - parser.add_argument( - "--nbins", - type=int, - default=51, - help="Number of TOF bins", - ) - parser.add_argument( +def _add_mcstas_args(parser: argparse.ArgumentParser) -> None: + mcstas_arg_group = parser.add_argument_group("McStas Data Reduction Options") + mcstas_arg_group.add_argument( "--max_counts", type=int, default=None, help="Maximum Counts", ) - parser.add_argument( - "--detector_ids", + mcstas_arg_group.add_argument( + "--chunk_size", type=int, - nargs="+", - default=[0, 1, 2], - help="Detector indices to process", - ) - parser.add_argument( - "--compression", - type=bool, - default=True, - help="Compress reduced output with bitshuffle/lz4", + default=10_000_000, + help="Chunk size for processing (number of events per chunk)", ) + +def main() -> None: + from .._executable_helper import build_logger, build_reduction_arg_parser + + parser = build_reduction_arg_parser() + _add_mcstas_args(parser) args = parser.parse_args() input_file = pathlib.Path(args.input_file).resolve() output_file = pathlib.Path(args.output_file).resolve() - logger = logging.getLogger(__name__) - if args.verbose: - logger.setLevel(logging.INFO) - logger.addHandler(logging.StreamHandler(sys.stdout)) + logger = build_logger(args) wf = McStasWorkflow() reduction( @@ -325,7 +303,7 @@ def main() -> None: nbins=args.nbins, max_counts=args.max_counts, detector_ids=args.detector_ids, - compression=args.compression, + compression=Compression[args.compression], logger=logger, wf=wf, ) diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/nexus.py index 42a11156..64f31b36 100644 --- a/packages/essnmx/src/ess/nmx/nexus.py +++ b/packages/essnmx/src/ess/nmx/nexus.py @@ -22,6 +22,73 @@ ) +def _fallback_compute_positions(dg: sc.DataGroup) -> sc.DataGroup: + import warnings + + import scippnexus as snx + + warnings.warn( + "Using fallback compute_positions due to empty log entries. " + "This may lead to incorrect results. Please check the data carefully." + "The fallback will replace empty logs with a scalar value of zero.", + UserWarning, + stacklevel=2, + ) + + empty_transformations = [ + transformation + for transformation in dg['depends_on'].transformations.values() + if 'time' in transformation.value.dims + and transformation.sizes['time'] == 0 # empty log + ] + for transformation in empty_transformations: + orig_value = transformation.value + orig_value = sc.scalar(0, unit=orig_value.unit, dtype=orig_value.dtype) + transformation.value = orig_value + return snx.compute_positions(dg, store_transform='transform_matrix') + + +def _compute_positions( + dg: sc.DataGroup, auto_fix_transformations: bool = False +) -> sc.DataGroup: + """Compute positions of the data group from transformations. + + Wraps the `scippnexus.compute_positions` function + and provides a fallback for cases where the transformations + contain empty logs. + + Parameters + ---------- + dg: + Data group containing the transformations and data. + auto_fix_transformations: + If `True`, it will attempt to fix empty transformations. + It will replace them with a scalar value of zero. + It is because adding a time dimension will make it not possible + to compute positions of children due to time-dependent transformations. + + Returns + ------- + : + Data group with computed positions. + + Warnings + -------- + If `auto_fix_transformations` is `True`, it will warn about the fallback + being used due to empty logs or scalar transformations. + This is because the fallback may lead to incorrect results. + + """ + import scippnexus as snx + + try: + return snx.compute_positions(dg, store_transform='transform_matrix') + except ValueError as e: + if auto_fix_transformations: + return _fallback_compute_positions(dg) + raise e + + def _create_dataset_from_string(*, root_entry: h5py.Group, name: str, var: str) -> None: root_entry.create_dataset(name, dtype=h5py.string_dtype(), data=var) @@ -428,6 +495,16 @@ def _export_detector_metadata_as_nxlauetof( _add_lauetof_detector_group(detector_metadata, nx_instrument) +def _extract_counts(dg: sc.DataGroup) -> sc.Variable: + counts: sc.DataArray = dg['counts'].data + if 'id' in counts.dims: + num_x, num_y = dg["detector_shape"].value + return sc.fold(counts, dim='id', sizes={'x': num_x, 'y': num_y}) + else: + # If there is no 'id' dimension, we assume it is already in the correct shape + return counts + + def _export_reduced_data_as_nxlauetof( dg: NMXReducedDataGroup, output_file: str | pathlib.Path | io.BytesIO, @@ -471,9 +548,7 @@ def _export_reduced_data_as_nxlauetof( data_dset = _create_compressed_dataset( name="data", root_entry=nx_detector, - var=sc.fold( - dg['counts'].data, dim='id', sizes={'x': num_x, 'y': num_y} - ), + var=_extract_counts(dg), chunks=(num_x, num_y, 1), dtype=np.uint, ) @@ -481,9 +556,7 @@ def _export_reduced_data_as_nxlauetof( data_dset = _create_dataset_from_var( name="data", root_entry=nx_detector, - var=sc.fold( - dg['counts'].data, dim='id', sizes={'x': num_x, 'y': num_y} - ), + var=_extract_counts(dg), dtype=np.uint, ) data_dset.attrs["signal"] = 1 diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 0d629021..356117e0 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -1,3 +1,4 @@ +import enum from dataclasses import dataclass from typing import Any, NewType @@ -76,3 +77,13 @@ class NMXRawDataMetadata: max_probability: MaximumProbability min_toa: MinimumTimeOfArrival max_toa: MaximumTimeOfArrival + + +class Compression(enum.Enum): + """Compression type of the output file. + + These options are written as enum for future extensibility. + """ + + NONE = 0 + BITSHUFFLE_LZ4 = 1 diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py new file mode 100644 index 00000000..f96a56f9 --- /dev/null +++ b/packages/essnmx/tests/executable_test.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2025 Scipp contributors (https://github.com/scipp) + +import pathlib +import subprocess + +import pytest +import scipp as sc +import scippnexus as snx +from scipp.testing import assert_allclose + + +@pytest.fixture(scope="session") +def small_nmx_nexus_path(): + """Fixture to provide the path to the small NMX NeXus file.""" + from ess.nmx.data import get_small_nmx_nexus + + return get_small_nmx_nexus() + + +def _check_output_file( + output_file_path: pathlib.Path, expected_toa_output: sc.Variable +): + detector_names = [f'detector_panel_{i}' for i in range(3)] + with snx.File(output_file_path, 'r') as f: + # Test + for name in detector_names: + det_gr = f[f'entry/instrument/{name}'] + assert det_gr is not None + toa_edges = det_gr['time_of_flight'][()] + assert_allclose(toa_edges, expected_toa_output) + + +def test_executable_runs(small_nmx_nexus_path, tmp_path: pathlib.Path): + """Test that the executable runs and returns the expected output.""" + output_file = tmp_path / "output.h5" + assert not output_file.exists() + + nbins = 20 # Small number of bins for testing. + # The output has 1280x1280 pixels per detector per time bin. + expected_toa_bins = sc.linspace( + dim='dim_0', + start=2, # Unrealistic number for testing + stop=int((1 / 15) * 1_000), # Unrealistic number for testing + num=nbins + 1, + unit='ms', + ) + expected_toa_output = sc.midpoints(expected_toa_bins, dim='dim_0').to(unit='ns') + + commands = ( + 'essnmx-reduce', + '--input_file', + small_nmx_nexus_path, + '--nbins', + str(nbins), + '--output_file', + output_file.as_posix(), + '--min-toa', + str(int(expected_toa_bins.min().value)), + '--max-toa', + str(int(expected_toa_bins.max().value)), + ) + # Validate that all commands are strings and contain no unsafe characters + result = subprocess.run( # noqa: S603 - We are not accepting arbitrary input here. + commands, + text=True, + capture_output=True, + check=False, + ) + assert result.returncode == 0 + assert output_file.exists() + _check_output_file(output_file, expected_toa_output=expected_toa_output) From f4c18ed5be518338d78f2ed75bb00042ade714f3 Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 10 Nov 2025 12:21:31 +0100 Subject: [PATCH 268/403] Bump up essreduce minimum version to use new domain types. (#154) --- packages/essnmx/pyproject.toml | 2 +- packages/essnmx/requirements/base.in | 4 +- packages/essnmx/requirements/base.txt | 73 +++++++++++------------ packages/essnmx/requirements/basetest.txt | 4 +- packages/essnmx/requirements/ci.txt | 20 +++---- packages/essnmx/requirements/dev.txt | 44 +++++++------- packages/essnmx/requirements/docs.txt | 60 +++++++++---------- packages/essnmx/requirements/mypy.txt | 2 +- packages/essnmx/requirements/nightly.in | 2 +- packages/essnmx/requirements/nightly.txt | 71 ++++++++++------------ packages/essnmx/requirements/static.txt | 12 ++-- 11 files changed, 140 insertions(+), 154 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 4535426d..ba1be8e6 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -31,7 +31,7 @@ requires-python = ">=3.11" # Make sure to list one dependency per line. dependencies = [ "dask>=2022.1.0", - "essreduce>=24.03.0", + "essreduce>=25.11.0", # New domain types implemented, see https://scipp.github.io/essreduce/user-guide/reduction-workflow-guidelines.html#c-6-domain-types-for-data-products "graphviz", "plopp>=24.7.0", "sciline>=24.06.0", diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index 8826f94f..36150236 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -3,11 +3,11 @@ # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask>=2022.1.0 -essreduce>=24.03.0 +essreduce>=25.11.0 graphviz plopp>=24.7.0 sciline>=24.06.0 -scipp>=23.8.0 +scipp>=25.3.0 scippnexus>=23.12.0 pooch>=1.5 pandas>=2.1.2 diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 44b2e1b2..5406ed10 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:57dea2fd04558cbe6e1a72f838773564fbe6cb3c +# SHA1:e5e735f23415c4cc7855c37cba24934ee1d0a01d # # This file was generated by pip-compile-multi. # To update, run: @@ -9,13 +9,13 @@ annotated-types==0.7.0 # via pydantic bitshuffle==0.5.2 # via -r base.in -certifi==2025.8.3 +certifi==2025.10.5 # via requests -charset-normalizer==3.4.2 +charset-normalizer==3.4.4 # via requests -click==8.2.1 +click==8.3.0 # via dask -cloudpickle==3.1.1 +cloudpickle==3.1.2 # via dask contourpy==1.3.3 # via matplotlib @@ -23,38 +23,36 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -cython==3.1.2 +cython==3.1.6 # via bitshuffle -dask==2025.7.0 +dask==2025.10.0 # via -r base.in defusedxml==0.7.1 # via -r base.in -dnspython==2.7.0 +dnspython==2.8.0 # via email-validator -email-validator==2.2.0 +email-validator==2.3.0 # via scippneutron -essreduce==25.7.1 +essreduce==25.11.0 # via -r base.in -fonttools==4.59.0 +fonttools==4.60.1 # via matplotlib -fsspec==2025.7.0 +fsspec==2025.10.0 # via dask gemmi==0.7.3 # via -r base.in graphviz==0.21 # via -r base.in -h5py==3.14.0 +h5py==3.15.1 # via # bitshuffle # scippneutron # scippnexus -idna==3.10 +idna==3.11 # via # email-validator # requests -importlib-metadata==8.7.0 - # via dask -kiwisolver==1.4.8 +kiwisolver==1.4.9 # via matplotlib lazy-loader==0.4 # via @@ -62,17 +60,17 @@ lazy-loader==0.4 # scippneutron locket==1.0.0 # via partd -matplotlib==3.10.5 +matplotlib==3.10.7 # via # mpltoolbox # plopp -mpltoolbox==25.5.0 +mpltoolbox==25.10.0 # via scippneutron -msgpack==1.1.1 +msgpack==1.1.2 # via -r base.in networkx==3.5 # via cyclebane -numpy==2.3.2 +numpy==2.3.4 # via # bitshuffle # contourpy @@ -88,25 +86,25 @@ packaging==25.0 # lazy-loader # matplotlib # pooch -pandas==2.3.1 +pandas==2.3.3 # via -r base.in partd==1.4.2 # via dask -pillow==11.3.0 +pillow==12.0.0 # via matplotlib -platformdirs==4.3.8 +platformdirs==4.5.0 # via pooch -plopp==25.7.1 +plopp==25.10.0 # via # -r base.in # scippneutron pooch==1.8.2 # via -r base.in -pydantic==2.11.7 +pydantic==2.12.3 # via scippneutron -pydantic-core==2.33.2 +pydantic-core==2.41.4 # via pydantic -pyparsing==3.2.3 +pyparsing==3.2.5 # via matplotlib python-dateutil==2.9.0.post0 # via @@ -116,15 +114,15 @@ python-dateutil==2.9.0.post0 # scippnexus pytz==2025.2 # via pandas -pyyaml==6.0.2 +pyyaml==6.0.3 # via dask -requests==2.32.4 +requests==2.32.5 # via pooch -sciline==25.5.2 +sciline==25.11.1 # via # -r base.in # essreduce -scipp==25.8.0 +scipp==25.11.0 # via # -r base.in # essreduce @@ -137,30 +135,27 @@ scippnexus==25.6.0 # -r base.in # essreduce # scippneutron -scipy==1.16.1 +scipy==1.16.3 # via # scippneutron # scippnexus six==1.17.0 # via python-dateutil -toolz==1.0.0 +toolz==1.1.0 # via # dask # partd -typing-extensions==4.14.1 +typing-extensions==4.15.0 # via # pydantic # pydantic-core - # sciline # typing-inspection -typing-inspection==0.4.1 +typing-inspection==0.4.2 # via pydantic tzdata==2025.2 # via pandas urllib3==2.5.0 # via requests -zipp==3.23.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt index e37fea84..85cf4015 100644 --- a/packages/essnmx/requirements/basetest.txt +++ b/packages/essnmx/requirements/basetest.txt @@ -5,7 +5,7 @@ # # requirements upgrade # -iniconfig==2.1.0 +iniconfig==2.3.0 # via pytest packaging==25.0 # via pytest @@ -13,5 +13,5 @@ pluggy==1.6.0 # via pytest pygments==2.19.2 # via pytest -pytest==8.4.1 +pytest==8.4.2 # via -r basetest.in diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index cf385f77..5bc02f40 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -5,19 +5,19 @@ # # requirements upgrade # -cachetools==6.1.0 +cachetools==6.2.1 # via tox -certifi==2025.8.3 +certifi==2025.10.5 # via requests chardet==5.2.0 # via tox -charset-normalizer==3.4.2 +charset-normalizer==3.4.4 # via requests colorama==0.4.6 # via tox distlib==0.4.0 # via virtualenv -filelock==3.18.0 +filelock==3.20.0 # via # tox # virtualenv @@ -25,28 +25,28 @@ gitdb==4.0.12 # via gitpython gitpython==3.1.45 # via -r ci.in -idna==3.10 +idna==3.11 # via requests packaging==25.0 # via # -r ci.in # pyproject-api # tox -platformdirs==4.3.8 +platformdirs==4.5.0 # via # tox # virtualenv pluggy==1.6.0 # via tox -pyproject-api==1.9.1 +pyproject-api==1.10.0 # via tox -requests==2.32.4 +requests==2.32.5 # via -r ci.in smmap==5.0.2 # via gitdb -tox==4.28.4 +tox==4.32.0 # via -r ci.in urllib3==2.5.0 # via requests -virtualenv==20.33.1 +virtualenv==20.35.4 # via tox diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index 94aff0a8..a2d2832e 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -12,7 +12,7 @@ -r static.txt -r test.txt -r wheels.txt -anyio==4.10.0 +anyio==4.11.0 # via # httpx # jupyter-server @@ -20,13 +20,13 @@ argon2-cffi==25.1.0 # via jupyter-server argon2-cffi-bindings==25.1.0 # via argon2-cffi -arrow==1.3.0 +arrow==1.4.0 # via isoduration async-lru==2.0.5 # via jupyterlab -cffi==1.17.1 +cffi==2.0.0 # via argon2-cffi-bindings -copier==9.9.0 +copier==9.10.3 # via -r dev.in dunamai==1.25.0 # via copier @@ -44,20 +44,20 @@ isoduration==20.11.0 # via jsonschema jinja2-ansible-filters==1.3.2 # via copier -json5==0.12.0 +json5==0.12.1 # via jupyterlab-server jsonpointer==3.0.0 # via jsonschema -jsonschema[format-nongpl]==4.25.0 +jsonschema[format-nongpl]==4.25.1 # via # jupyter-events # jupyterlab-server # nbformat jupyter-events==0.12.0 # via jupyter-server -jupyter-lsp==2.2.6 +jupyter-lsp==2.3.0 # via jupyterlab -jupyter-server==2.16.0 +jupyter-server==2.17.0 # via # jupyter-lsp # jupyterlab @@ -65,29 +65,27 @@ jupyter-server==2.16.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.4.5 +jupyterlab==4.4.10 # via -r dev.in -jupyterlab-server==2.27.3 +jupyterlab-server==2.28.0 # via jupyterlab -lark==1.2.2 +lark==1.3.1 # via rfc3987-syntax notebook-shim==0.2.4 # via jupyterlab -overrides==7.7.0 - # via jupyter-server -pip-compile-multi==3.2.1 +pip-compile-multi==3.2.2 # via -r dev.in -pip-tools==7.5.0 +pip-tools==7.5.1 # via pip-compile-multi -plumbum==1.9.0 +plumbum==1.10.0 # via copier -prometheus-client==0.22.1 +prometheus-client==0.23.1 # via jupyter-server -pycparser==2.22 +pycparser==2.23 # via cffi -python-json-logger==3.3.0 +python-json-logger==4.0.0 # via jupyter-events -questionary==2.1.0 +questionary==2.1.1 # via copier rfc3339-validator==0.1.4 # via @@ -109,13 +107,11 @@ terminado==0.18.1 # jupyter-server-terminals toposort==1.10 # via pip-compile-multi -types-python-dateutil==2.9.0.20250708 - # via arrow uri-template==1.3.0 # via jsonschema -webcolors==24.11.1 +webcolors==25.10.0 # via jsonschema -websocket-client==1.8.0 +websocket-client==1.9.0 # via jupyter-server wheel==0.45.1 # via pip-tools diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 4535f7c0..44c31e87 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -14,7 +14,7 @@ appnope==0.1.4 # via ipykernel asttokens==3.0.0 # via stack-data -attrs==25.3.0 +attrs==25.4.0 # via # jsonschema # referencing @@ -24,17 +24,17 @@ babel==2.17.0 # via # pydata-sphinx-theme # sphinx -beautifulsoup4==4.13.4 +beautifulsoup4==4.14.2 # via # nbconvert # pydata-sphinx-theme -bleach[css]==6.2.0 +bleach[css]==6.3.0 # via nbconvert comm==0.2.3 # via # ipykernel # ipywidgets -debugpy==1.8.15 +debugpy==1.8.17 # via ipykernel decorator==5.2.1 # via ipython @@ -44,24 +44,24 @@ docutils==0.21.2 # nbsphinx # pydata-sphinx-theme # sphinx -executing==2.2.0 +executing==2.2.1 # via stack-data -fastjsonschema==2.21.1 +fastjsonschema==2.21.2 # via nbformat imagesize==1.4.1 # via sphinx ipydatawidgets==4.3.5 # via pythreejs -ipykernel==6.30.1 +ipykernel==7.1.0 # via -r docs.in -ipython==9.4.0 +ipython==9.6.0 # via # -r docs.in # ipykernel # ipywidgets ipython-pygments-lexers==1.1.1 # via ipython -ipywidgets==8.1.7 +ipywidgets==8.1.8 # via # ipydatawidgets # pythreejs @@ -73,15 +73,15 @@ jinja2==3.1.6 # nbconvert # nbsphinx # sphinx -jsonschema==4.25.0 +jsonschema==4.25.1 # via nbformat -jsonschema-specifications==2025.4.1 +jsonschema-specifications==2025.9.1 # via jsonschema jupyter-client==8.6.3 # via # ipykernel # nbclient -jupyter-core==5.8.1 +jupyter-core==5.9.1 # via # ipykernel # jupyter-client @@ -90,25 +90,25 @@ jupyter-core==5.8.1 # nbformat jupyterlab-pygments==0.3.0 # via nbconvert -jupyterlab-widgets==3.0.15 +jupyterlab-widgets==3.0.16 # via ipywidgets markdown-it-py==3.0.0 # via # mdit-py-plugins # myst-parser -markupsafe==3.0.2 +markupsafe==3.0.3 # via # jinja2 # nbconvert -matplotlib-inline==0.1.7 +matplotlib-inline==0.2.1 # via # ipykernel # ipython -mdit-py-plugins==0.4.2 +mdit-py-plugins==0.5.0 # via myst-parser mdurl==0.1.2 # via markdown-it-py -mistune==3.1.3 +mistune==3.1.4 # via nbconvert myst-parser==4.0.1 # via -r docs.in @@ -127,19 +127,19 @@ nest-asyncio==1.6.0 # via ipykernel pandocfilters==1.5.1 # via nbconvert -parso==0.8.4 +parso==0.8.5 # via jedi pexpect==4.9.0 # via ipython -prompt-toolkit==3.0.51 +prompt-toolkit==3.0.52 # via ipython -psutil==7.0.0 +psutil==7.1.3 # via ipykernel ptyprocess==0.7.0 # via pexpect pure-eval==0.2.3 # via stack-data -pydantic-settings==2.10.1 +pydantic-settings==2.11.0 # via autodoc-pydantic pydata-sphinx-theme==0.16.1 # via -r docs.in @@ -151,25 +151,25 @@ pygments==2.19.2 # nbconvert # pydata-sphinx-theme # sphinx -python-dotenv==1.1.1 +python-dotenv==1.2.1 # via pydantic-settings pythreejs==2.4.2 # via -r docs.in -pyzmq==27.0.1 +pyzmq==27.1.0 # via # ipykernel # jupyter-client -referencing==0.36.2 +referencing==0.37.0 # via # jsonschema # jsonschema-specifications -rpds-py==0.26.0 +rpds-py==0.28.0 # via # jsonschema # referencing snowballstemmer==3.0.1 # via sphinx -soupsieve==2.7 +soupsieve==2.8 # via beautifulsoup4 sphinx==8.1.3 # via @@ -203,7 +203,7 @@ stack-data==0.6.3 # via ipython tinycss2==1.4.0 # via bleach -tornado==6.5.1 +tornado==6.5.2 # via # ipykernel # jupyter-client @@ -221,15 +221,15 @@ traitlets==5.14.3 # nbsphinx # pythreejs # traittypes -traittypes==0.2.1 +traittypes==0.2.3 # via ipydatawidgets -wcwidth==0.2.13 +wcwidth==0.2.14 # via prompt-toolkit webencodings==0.5.1 # via # bleach # tinycss2 -widgetsnbextension==4.0.14 +widgetsnbextension==4.0.15 # via ipywidgets # The following packages are considered to be unsafe in a requirements file: diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index 0c527ae4..74e2fd20 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -6,7 +6,7 @@ # requirements upgrade # -r test.txt -mypy==1.17.1 +mypy==1.18.2 # via -r mypy.in mypy-extensions==1.1.0 # via mypy diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index a2f5be19..d78c28a2 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -2,7 +2,7 @@ # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask>=2022.1.0 -essreduce>=24.03.0 +essreduce>=25.11.0 graphviz pooch>=1.5 pandas>=2.1.2 diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index c5604ce6..6c5504ad 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:3aa4a12a5ca5339b013c64f1d999b73adb5b3ca1 +# SHA1:c3fe0a59eb91d120f06ad2064b100fd7397b9055 # # This file was generated by pip-compile-multi. # To update, run: @@ -12,13 +12,13 @@ annotated-types==0.7.0 # via pydantic bitshuffle==0.5.2 # via -r nightly.in -certifi==2025.8.3 +certifi==2025.10.5 # via requests -charset-normalizer==3.4.2 +charset-normalizer==3.4.4 # via requests -click==8.2.1 +click==8.3.0 # via dask -cloudpickle==3.1.1 +cloudpickle==3.1.2 # via dask contourpy==1.3.3 # via matplotlib @@ -26,40 +26,38 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -cython==3.1.2 +cython==3.2.0b3 # via bitshuffle -dask==2025.7.0 +dask==2025.10.0 # via -r nightly.in defusedxml==0.8.0rc2 # via -r nightly.in -dnspython==2.7.0 +dnspython==2.8.0 # via email-validator -email-validator==2.2.0 +email-validator==2.3.0 # via scippneutron -essreduce==25.7.1 +essreduce==25.11.0 # via -r nightly.in -fonttools==4.59.0 +fonttools==4.60.1 # via matplotlib -fsspec==2025.7.0 +fsspec==2025.10.0 # via dask gemmi==0.7.3 # via -r nightly.in graphviz==0.21 # via -r nightly.in -h5py==3.14.0 +h5py==3.15.1 # via # bitshuffle # scippneutron # scippnexus -idna==3.10 +idna==3.11 # via # email-validator # requests -importlib-metadata==8.7.0 - # via dask -iniconfig==2.1.0 +iniconfig==2.3.0 # via pytest -kiwisolver==1.4.8 +kiwisolver==1.4.10rc0 # via matplotlib lazy-loader==0.4 # via @@ -67,17 +65,17 @@ lazy-loader==0.4 # scippneutron locket==1.0.0 # via partd -matplotlib==3.10.5 +matplotlib==3.10.7 # via # mpltoolbox # plopp -mpltoolbox==25.5.0 +mpltoolbox==25.10.0 # via scippneutron -msgpack==1.1.1 +msgpack==1.1.2 # via -r nightly.in networkx==3.5 # via cyclebane -numpy==2.3.2 +numpy==2.3.4 # via # bitshuffle # contourpy @@ -94,13 +92,13 @@ packaging==25.0 # matplotlib # pooch # pytest -pandas==2.3.1 +pandas==2.3.3 # via -r nightly.in partd==1.4.2 # via dask -pillow==11.3.0 +pillow==12.0.0 # via matplotlib -platformdirs==4.3.8 +platformdirs==4.5.0 # via pooch plopp @ git+https://github.com/scipp/plopp@main # via @@ -110,15 +108,15 @@ pluggy==1.6.0 # via pytest pooch==1.8.2 # via -r nightly.in -pydantic==2.12.0a1 +pydantic==2.12.3 # via scippneutron -pydantic-core==2.37.2 +pydantic-core==2.41.4 # via pydantic pygments==2.19.2 # via pytest -pyparsing==3.2.3 +pyparsing==3.3.0a1 # via matplotlib -pytest==8.4.1 +pytest==8.4.2 # via -r nightly.in python-dateutil==2.9.0.post0 # via @@ -127,9 +125,9 @@ python-dateutil==2.9.0.post0 # scippneutron pytz==2025.2 # via pandas -pyyaml==6.0.2 +pyyaml==6.0.3 # via dask -requests==2.32.4 +requests==2.32.5 # via pooch sciline @ git+https://github.com/scipp/sciline@main # via @@ -148,30 +146,27 @@ scippnexus @ git+https://github.com/scipp/scippnexus@main # -r nightly.in # essreduce # scippneutron -scipy==1.16.1 +scipy==1.16.3 # via # scippneutron # scippnexus six==1.17.0 # via python-dateutil -toolz==1.0.0 +toolz==1.1.0 # via # dask # partd -typing-extensions==4.14.1 +typing-extensions==4.15.0 # via # pydantic # pydantic-core - # sciline # typing-inspection -typing-inspection==0.4.1 +typing-inspection==0.4.2 # via pydantic tzdata==2025.2 # via pandas urllib3==2.5.0 # via requests -zipp==3.23.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index 3e288557..991a8921 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -9,17 +9,17 @@ cfgv==3.4.0 # via pre-commit distlib==0.4.0 # via virtualenv -filelock==3.18.0 +filelock==3.20.0 # via virtualenv -identify==2.6.12 +identify==2.6.15 # via pre-commit nodeenv==1.9.1 # via pre-commit -platformdirs==4.3.8 +platformdirs==4.5.0 # via virtualenv -pre-commit==4.2.0 +pre-commit==4.3.0 # via -r static.in -pyyaml==6.0.2 +pyyaml==6.0.3 # via pre-commit -virtualenv==20.33.1 +virtualenv==20.35.4 # via pre-commit From 831a094cb48a0dd59202c3c394d439268ab1ad47 Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 10 Nov 2025 19:41:37 +0100 Subject: [PATCH 269/403] Retrieve crystal rotation from nexus file. (#155) --- packages/essnmx/src/ess/nmx/executables.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 61d8e634..c423eec6 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -30,6 +30,24 @@ def _retrieve_sample_position(file: snx.File) -> sc.Variable: return _compute_positions(da, auto_fix_transformations=True)['position'] +def _retrieve_crystal_rotation(file: snx.File) -> sc.Variable: + if 'crystal_rotation' not in file['entry/sample']: + import warnings + + warnings.warn( + "No crystal rotation found in the Nexus file under " + "'entry/sample/crystal_rotation'. Returning zero rotation.", + RuntimeWarning, + stacklevel=2, + ) + return sc.vector([0, 0, 0], unit='deg') + + # Temporary way of storing crystal rotation. + # streaming-sample-mcstas module writes crystal rotation under + # 'entry/sample/crystal_rotation' as an array of three values. + return file['entry/sample/crystal_rotation'][()] + + def _decide_fast_axis(da: sc.DataArray) -> str: x_slice = da['x_pixel_offset', 0].coords['detector_number'] y_slice = da['y_pixel_offset', 0].coords['detector_number'] @@ -230,11 +248,11 @@ def reduction( display(f"Selected detectors: {list(detector_id_map.keys())}") source_position = _retrieve_source_position(f) sample_position = _retrieve_sample_position(f) + crystal_rotation = _retrieve_crystal_rotation(f) experiment_metadata = NMXExperimentMetadata( sc.DataGroup( { - # Placeholder for crystal rotation - 'crystal_rotation': sc.vector([0, 0, 0], unit='deg'), + 'crystal_rotation': crystal_rotation, 'sample_position': sample_position, 'source_position': source_position, 'sample_name': sc.scalar(f['entry/sample/name'][()]), From ab70fa9649b524260c09a092be2b53379dde3dcc Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Thu, 13 Nov 2025 08:51:20 +0100 Subject: [PATCH 270/403] fix: enable compression when bitshuffle is set, disable otherwise --- packages/essnmx/src/ess/nmx/executables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index c423eec6..c5c0ae95 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -354,7 +354,7 @@ def reduction( _export_reduced_data_as_nxlauetof( dg, output_file=output_file, - compress_counts=(compression == Compression.NONE), + compress_counts=(compression == Compression.BITSHUFFLE_LZ4), ) detector_grs[det_name] = dg From d677e5531c5891d15584554e4ae6b9f06d71ebbf Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Sun, 16 Nov 2025 21:27:46 +0100 Subject: [PATCH 271/403] Add mtz data file to be ignored. --- packages/essnmx/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/essnmx/.gitignore b/packages/essnmx/.gitignore index 720318ba..c6e47900 100644 --- a/packages/essnmx/.gitignore +++ b/packages/essnmx/.gitignore @@ -46,3 +46,4 @@ docs/generated/ *.zip *.sqw *.nxspe +*.mtz From 383e4244aee54b303092a5d2e6f40df76f8320e1 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Sun, 16 Nov 2025 22:01:36 +0100 Subject: [PATCH 272/403] Use pydantic model to build argparser. --- .../essnmx/src/ess/nmx/_executable_helper.py | 153 +++++++++++++++++- packages/essnmx/src/ess/nmx/executables.py | 59 ++----- 2 files changed, 165 insertions(+), 47 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index bf8aec32..8a63d205 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -3,11 +3,162 @@ import argparse import logging import sys +from typing import Literal + +from pydantic import BaseModel from .types import Compression +class InputConfig(BaseModel): + input_file: str + detector_ids: list[int | str] = [0, 1, 2] + chunk_size_pulse: int = 1 + + @classmethod + def add_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + group = parser.add_argument_group("Input Options") + group.add_argument( + "--input-file", type=str, help="Path to the input file", required=True + ) + group.add_argument( + "--detector-ids", + type=int, + nargs="+", + default=[0, 1, 2], + help="Detector indices to process", + ) + group.add_argument( + "--chunk-size-pulse", + type=int, + default=1, + help="Number of pulses to process in each chunk", + ) + return parser + + @classmethod + def from_args(cls, args: argparse.Namespace) -> "InputConfig": + return cls( + input_file=args.input_file, + detector_ids=args.detector_ids, + ) + + +class WorkflowConfig(BaseModel): + nbins: int = 50 + min_toa: int = 0 + max_toa: int = int((1 / 14) * 1_000) + fast_axis: Literal['x', 'y'] | None = None + + @classmethod + def add_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + group = parser.add_argument_group("Workflow Options") + group.add_argument( + "--nbins", + type=int, + default=50, + help="Number of TOF bins", + ) + group.add_argument( + "--min-toa", + type=int, + default=0, + help="Minimum time of arrival (TOA) in [ms].", + ) + group.add_argument( + "--max-toa", + type=int, + default=int((1 / 14) * 1_000), + help="Maximum time of arrival (TOA) in [ms].", + ) + group.add_argument( + "--fast-axis", + type=str, + choices=['x', 'y', None], + default=None, + help="Specify the fast axis of the detector. " + "If None, it will be determined " + "automatically based on the pixel offsets.", + ) + return parser + + @classmethod + def from_args(cls, args: argparse.Namespace) -> "WorkflowConfig": + return cls(nbins=args.nbins) + + +class OutputConfig(BaseModel): + # Log verbosity + verbose: bool = False + # File output + output_file: str = "scipp_output.h5" + compression: Compression = Compression.BITSHUFFLE_LZ4 + + @classmethod + def add_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: + group = parser.add_argument_group("Output Options") + group.add_argument( + "--verbose", "-v", action="store_true", help="Increase output verbosity" + ) + group.add_argument( + "--output-file", + type=str, + default="scipp_output.h5", + help="Path to the output file", + ) + group.add_argument( + "--compression", + type=str, + default=Compression.BITSHUFFLE_LZ4.name, + choices=[compression_key.name for compression_key in Compression], + help="Compress option of reduced output file. Default: BITSHUFFLE_LZ4", + ) + return parser + + @classmethod + def from_args(cls, args: argparse.Namespace) -> "OutputConfig": + return cls( + verbose=args.verbose, + output_file=args.output_file, + compression=Compression[args.compression], + ) + + +class ReductionConfig(BaseModel): + inputs: InputConfig + workflow: WorkflowConfig + output: OutputConfig + + @classmethod + def build_argument_parser(cls) -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Command line arguments for the ESS NMX reduction. " + "It assumes 14 Hz pulse speed." + ) + parser = InputConfig.add_args(parser) + parser = WorkflowConfig.add_args(parser) + parser = OutputConfig.add_args(parser) + return parser + + @classmethod + def from_args(cls, args: argparse.Namespace) -> "ReductionConfig": + return cls( + inputs=InputConfig.from_args(args), + workflow=WorkflowConfig.from_args(args), + output=OutputConfig.from_args(args), + ) + + def build_reduction_arg_parser() -> argparse.ArgumentParser: + import warnings + + warnings.warn( + "build_reduction_arg_parser is deprecated and will be removed " + "in the future release (>=26.11.0) " + "Please use the config classes to handle command line arguments.", + DeprecationWarning, + stacklevel=2, + ) parser = argparse.ArgumentParser( description="Command line arguments for the NMX reduction. " "It assumes 14 Hz pulse speed." @@ -51,7 +202,7 @@ def build_reduction_arg_parser() -> argparse.ArgumentParser: return parser -def build_logger(args: argparse.Namespace) -> logging.Logger: +def build_logger(args: argparse.Namespace | OutputConfig) -> logging.Logger: logger = logging.getLogger(__name__) if args.verbose: logger.setLevel(logging.INFO) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index c5c0ae95..a3780d4d 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) -import argparse import logging import pathlib from collections.abc import Callable @@ -10,6 +9,7 @@ import scipp as sc import scippnexus as snx +from ._executable_helper import ReductionConfig, build_logger from .nexus import ( _compute_positions, _export_detector_metadata_as_nxlauetof, @@ -362,46 +362,13 @@ def reduction( return sc.DataGroup(detector_grs) -def _add_ess_reduction_args(arg: argparse.ArgumentParser) -> None: - argument_group = arg.add_argument_group("ESS Reduction Options") - argument_group.add_argument( - "--chunk_size", - type=int, - default=-1, - help="Chunk size for processing (number of pulses per chunk).", - ) - argument_group.add_argument( - "--min-toa", - type=int, - default=0, - help="Minimum time of arrival (TOA) in ms.", - ) - argument_group.add_argument( - "--max-toa", - type=int, - default=int((1 / 14) * 1_000), - help="Maximum time of arrival (TOA) in ms.", - ) - argument_group.add_argument( - "--fast-axis", - type=str, - choices=['x', 'y', None], - default=None, - help="Specify the fast axis of the detector. If None, it will be determined " - "automatically based on the pixel offsets.", - ) - - def main() -> None: - from ._executable_helper import build_logger, build_reduction_arg_parser - - parser = build_reduction_arg_parser() - _add_ess_reduction_args(parser) - args = parser.parse_args() + parser = ReductionConfig.build_argument_parser() + config = ReductionConfig.from_args(parser.parse_args()) - input_file = pathlib.Path(args.input_file).resolve() - output_file = pathlib.Path(args.output_file).resolve() - logger = build_logger(args) + input_file = pathlib.Path(config.inputs.input_file).resolve() + output_file = pathlib.Path(config.output.output_file).resolve() + logger = build_logger(config.output) logger.info("Input file: %s", input_file) logger.info("Output file: %s", output_file) @@ -409,12 +376,12 @@ def main() -> None: reduction( input_file=input_file, output_file=output_file, - chunk_size=args.chunk_size, - detector_ids=args.detector_ids, - compression=Compression[args.compression], - toa_bin_edges=args.nbins, - min_toa=sc.scalar(args.min_toa, unit='ms'), - max_toa=sc.scalar(args.max_toa, unit='ms'), - fast_axis=args.fast_axis, + chunk_size=config.inputs.chunk_size_pulse, + detector_ids=config.inputs.detector_ids, + compression=config.output.compression, + toa_bin_edges=config.workflow.nbins, + min_toa=sc.scalar(config.workflow.min_toa, unit='ms'), + max_toa=sc.scalar(config.workflow.max_toa, unit='ms'), + fast_axis=config.workflow.fast_axis, logger=logger, ) From eabac554974e67fb3417af8ae2e9778b2d1ac2ab Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 17 Nov 2025 09:45:59 +0100 Subject: [PATCH 273/403] Add event-based chunk size option and iter-chunk option. --- packages/essnmx/src/ess/nmx/_executable_helper.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index 8a63d205..c8967b22 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -31,8 +31,18 @@ def add_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: group.add_argument( "--chunk-size-pulse", type=int, - default=1, - help="Number of pulses to process in each chunk", + default=0, + help="Number of pulses to process in each chunk. " + "If 0 or negative, process all pulses at once.", + ) + group.add_argument( + "--chunk-size-events", + type=int, + default=0, + help="Number of events to process in each chunk. " + "If 0 or negative, process all events at once." + "If both chunk-size-pulse and chunk-size-events are set, " + "chunk-size-pulse is preferred.", ) return parser From da92f81fcaa3f4b5da6a9b217cc5392648c74198 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 17 Nov 2025 09:46:04 +0100 Subject: [PATCH 274/403] Add event-based chunk size option and iter-chunk option. --- packages/essnmx/src/ess/nmx/_executable_helper.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index c8967b22..9e39935e 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -28,14 +28,24 @@ def add_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: default=[0, 1, 2], help="Detector indices to process", ) - group.add_argument( + chunk_option_group = parser.add_argument_group("Chunking Options") + chunk_option_group.add_argument( + "--iter-chunk", + type=bool, + default=False, + help="Whether to process the input file in chunks " + " based on the hdf5 dataset chunk size. " + "It is ignored if hdf5 dataset is not chunked. " + "If True, it overrides chunk-size-pulse and chunk-size-events options.", + ) + chunk_option_group.add_argument( "--chunk-size-pulse", type=int, default=0, help="Number of pulses to process in each chunk. " "If 0 or negative, process all pulses at once.", ) - group.add_argument( + chunk_option_group.add_argument( "--chunk-size-events", type=int, default=0, From 60706660564ad78d62dcc07fbd6c65974e5ed997 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 17 Nov 2025 11:04:42 +0100 Subject: [PATCH 275/403] Fix argument parser. --- .../essnmx/src/ess/nmx/_executable_helper.py | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index 9e39935e..99ecbc19 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -13,7 +13,10 @@ class InputConfig(BaseModel): input_file: str detector_ids: list[int | str] = [0, 1, 2] + # Chunking options + iter_chunk: bool = False chunk_size_pulse: int = 1 + chunk_size_events: int = 0 @classmethod def add_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: @@ -61,6 +64,9 @@ def from_args(cls, args: argparse.Namespace) -> "InputConfig": return cls( input_file=args.input_file, detector_ids=args.detector_ids, + chunk_size_pulse=args.chunk_size_pulse, + chunk_size_events=args.chunk_size_events, + iter_chunk=args.iter_chunk, ) @@ -104,7 +110,12 @@ def add_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: @classmethod def from_args(cls, args: argparse.Namespace) -> "WorkflowConfig": - return cls(nbins=args.nbins) + return cls( + nbins=args.nbins, + min_toa=args.min_toa, + max_toa=args.max_toa, + fast_axis=args.fast_axis, + ) class OutputConfig(BaseModel): @@ -168,6 +179,31 @@ def from_args(cls, args: argparse.Namespace) -> "ReductionConfig": output=OutputConfig.from_args(args), ) + @property + def _children(self) -> list[BaseModel]: + return [self.inputs, self.workflow, self.output] + + def to_command_arguments(self) -> list[str]: + args = {} + for instance in self._children: + args.update(instance.model_dump(mode='python')) + args = {f"--{k.replace('_', '-')}": v for k, v in args.items()} + + arg_list = [] + for k, v in args.items(): + if not isinstance(v, bool): + arg_list.append(k) + if isinstance(v, list): + arg_list.extend(str(item) for item in v) + elif isinstance(v, Compression): + arg_list.append(v.name) + else: + arg_list.append(str(v)) + elif v is True: + arg_list.append(k) + + return arg_list + def build_reduction_arg_parser() -> argparse.ArgumentParser: import warnings From 18d6ccf661cb6b2081339f8cabf5ef41c13dc327 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 17 Nov 2025 11:04:55 +0100 Subject: [PATCH 276/403] Add argument parsing test. --- packages/essnmx/tests/executable_test.py | 63 +++++++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index f96a56f9..bd42f03a 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -4,11 +4,70 @@ import pathlib import subprocess +import pydantic import pytest import scipp as sc import scippnexus as snx from scipp.testing import assert_allclose +from ess.nmx._executable_helper import ( + InputConfig, + OutputConfig, + ReductionConfig, + WorkflowConfig, +) +from ess.nmx.types import Compression + + +def _build_arg_list_from_pydantic_instance(*instances: pydantic.BaseModel) -> list[str]: + args = {} + for instance in instances: + args.update(instance.model_dump(mode='python')) + args = {f"--{k.replace('_', '-')}": v for k, v in args.items()} + + arg_list = [] + for k, v in args.items(): + if not isinstance(v, bool): + arg_list.append(k) + if isinstance(v, list): + arg_list.extend(str(item) for item in v) + elif isinstance(v, Compression): + arg_list.append(v.name) + else: + arg_list.append(str(v)) + elif v is True: + arg_list.append(k) + + return arg_list + + +def test_reduction_config() -> None: + input_options = InputConfig( + input_file='test-input.h5', detector_ids=[0, 1, 2, 3], chunk_size_pulse=10 + ) + workflow_options = WorkflowConfig( + nbins=100, min_toa=10, max_toa=5000, fast_axis='y' + ) + output_options = OutputConfig( + output_file='test-output.h5', compression=Compression.NONE, verbose=True + ) + # Building argument list manually, not using `to_command_arguments` to test it. + arg_list = _build_arg_list_from_pydantic_instance( + input_options, workflow_options, output_options + ) + + expected_config = ReductionConfig( + inputs=input_options, workflow=workflow_options, output=output_options + ) + assert arg_list == expected_config.to_command_arguments() + + parser = ReductionConfig.build_argument_parser() + args = parser.parse_args(arg_list) + + config = ReductionConfig.from_args(args) + + assert expected_config == config + @pytest.fixture(scope="session") def small_nmx_nexus_path(): @@ -49,11 +108,11 @@ def test_executable_runs(small_nmx_nexus_path, tmp_path: pathlib.Path): commands = ( 'essnmx-reduce', - '--input_file', + '--input-file', small_nmx_nexus_path, '--nbins', str(nbins), - '--output_file', + '--output-file', output_file.as_posix(), '--min-toa', str(int(expected_toa_bins.min().value)), From cd146f5460867e032bb50d0fe1b3254ecf2d0022 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 17 Nov 2025 11:08:33 +0100 Subject: [PATCH 277/403] Check the expected result before using it in the test. --- packages/essnmx/tests/executable_test.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index bd42f03a..0ea4da73 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -42,6 +42,8 @@ def _build_arg_list_from_pydantic_instance(*instances: pydantic.BaseModel) -> li def test_reduction_config() -> None: + """Test ReductionConfig argument parsing.""" + # Build config instances with non-default values. input_options = InputConfig( input_file='test-input.h5', detector_ids=[0, 1, 2, 3], chunk_size_pulse=10 ) @@ -51,21 +53,24 @@ def test_reduction_config() -> None: output_options = OutputConfig( output_file='test-output.h5', compression=Compression.NONE, verbose=True ) - # Building argument list manually, not using `to_command_arguments` to test it. - arg_list = _build_arg_list_from_pydantic_instance( - input_options, workflow_options, output_options - ) - expected_config = ReductionConfig( inputs=input_options, workflow=workflow_options, output=output_options ) + # Check if all instances have at least one non-default value. + assert expected_config.inputs != InputConfig(input_file='') + assert expected_config.workflow != WorkflowConfig() + assert expected_config.output != OutputConfig() + + # Build argument list manually, not using `to_command_arguments` to test it. + arg_list = _build_arg_list_from_pydantic_instance( + input_options, workflow_options, output_options + ) assert arg_list == expected_config.to_command_arguments() + # Parse arguments and build config from them. parser = ReductionConfig.build_argument_parser() args = parser.parse_args(arg_list) - config = ReductionConfig.from_args(args) - assert expected_config == config From 6b4072a02c4295892f71da0c7daf0becd119e2db Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 17 Nov 2025 11:26:18 +0100 Subject: [PATCH 278/403] Add swmr mode in the argument and add more strict sanity check for testing configuration. --- .../essnmx/src/ess/nmx/_executable_helper.py | 10 ++++- packages/essnmx/tests/executable_test.py | 43 +++++++++++++++---- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index 99ecbc19..7d25f227 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -11,7 +11,10 @@ class InputConfig(BaseModel): + # File IO input_file: str + swmr: bool = False + # Detector selection detector_ids: list[int | str] = [0, 1, 2] # Chunking options iter_chunk: bool = False @@ -24,6 +27,9 @@ def add_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: group.add_argument( "--input-file", type=str, help="Path to the input file", required=True ) + group.add_argument( + "--swmr", action="store_true", help="Open the input file in SWMR mode" + ) group.add_argument( "--detector-ids", type=int, @@ -34,8 +40,7 @@ def add_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: chunk_option_group = parser.add_argument_group("Chunking Options") chunk_option_group.add_argument( "--iter-chunk", - type=bool, - default=False, + action="store_true", help="Whether to process the input file in chunks " " based on the hdf5 dataset chunk size. " "It is ignored if hdf5 dataset is not chunked. " @@ -63,6 +68,7 @@ def add_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: def from_args(cls, args: argparse.Namespace) -> "InputConfig": return cls( input_file=args.input_file, + swmr=args.swmr, detector_ids=args.detector_ids, chunk_size_pulse=args.chunk_size_pulse, chunk_size_events=args.chunk_size_events, diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index 0ea4da73..6fab6cf4 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -41,25 +41,52 @@ def _build_arg_list_from_pydantic_instance(*instances: pydantic.BaseModel) -> li return arg_list +def _default_config() -> ReductionConfig: + """Helper to create a default ReductionConfig instance.""" + return ReductionConfig( + inputs=InputConfig(input_file=''), + workflow=WorkflowConfig(), + output=OutputConfig(), + ) + + +def _check_non_default_config(testing_config: ReductionConfig) -> None: + """Helper to check that all values in the config are non-default.""" + default_config = _default_config() + testing_children = testing_config._children + default_children = default_config._children + for testing_child, default_child in zip( + testing_children, default_children, strict=True + ): + testing_model = testing_child.model_dump(mode='python') + default_model = default_child.model_dump(mode='python') + for key, testing_value in testing_model.items(): + default_value = default_model[key] + assert ( + testing_value != default_value + ), f"Value for '{key}' is default: {testing_value}" + + def test_reduction_config() -> None: """Test ReductionConfig argument parsing.""" # Build config instances with non-default values. input_options = InputConfig( - input_file='test-input.h5', detector_ids=[0, 1, 2, 3], chunk_size_pulse=10 - ) - workflow_options = WorkflowConfig( - nbins=100, min_toa=10, max_toa=5000, fast_axis='y' + input_file='test-input.h5', + swmr=True, + detector_ids=[0, 1, 2, 3], + iter_chunk=True, + chunk_size_pulse=10, + chunk_size_events=100000, ) + workflow_options = WorkflowConfig(nbins=100, min_toa=10, max_toa=100, fast_axis='y') output_options = OutputConfig( output_file='test-output.h5', compression=Compression.NONE, verbose=True ) expected_config = ReductionConfig( inputs=input_options, workflow=workflow_options, output=output_options ) - # Check if all instances have at least one non-default value. - assert expected_config.inputs != InputConfig(input_file='') - assert expected_config.workflow != WorkflowConfig() - assert expected_config.output != OutputConfig() + # Check if all values are non-default. + _check_non_default_config(expected_config) # Build argument list manually, not using `to_command_arguments` to test it. arg_list = _build_arg_list_from_pydantic_instance( From 7bc843b2ba7c0ac6b1c56deb79ffb93d9bf30af9 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 17 Nov 2025 11:33:57 +0100 Subject: [PATCH 279/403] Command argument dumping in one line by default. --- .../essnmx/src/ess/nmx/_executable_helper.py | 16 ++++++++++++++-- packages/essnmx/tests/executable_test.py | 2 +- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index 7d25f227..9b70673e 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -189,7 +189,16 @@ def from_args(cls, args: argparse.Namespace) -> "ReductionConfig": def _children(self) -> list[BaseModel]: return [self.inputs, self.workflow, self.output] - def to_command_arguments(self) -> list[str]: + def to_command_arguments(self, one_line: bool = True) -> list[str] | str: + """Convert the config to a list of command line arguments. + + Parameters + ---------- + one_line: + If True, return a single string with all arguments joined by spaces. + If False, return a list of argument strings. + + """ args = {} for instance in self._children: args.update(instance.model_dump(mode='python')) @@ -208,7 +217,10 @@ def to_command_arguments(self) -> list[str]: elif v is True: arg_list.append(k) - return arg_list + if one_line: + return ' '.join(arg_list) + else: + return arg_list def build_reduction_arg_parser() -> argparse.ArgumentParser: diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index 6fab6cf4..ecf72a5a 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -92,7 +92,7 @@ def test_reduction_config() -> None: arg_list = _build_arg_list_from_pydantic_instance( input_options, workflow_options, output_options ) - assert arg_list == expected_config.to_command_arguments() + assert arg_list == expected_config.to_command_arguments(one_line=False) # Parse arguments and build config from them. parser = ReductionConfig.build_argument_parser() From e98734eab78f93a4469fc4eb4fb7fa4461e260aa Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 17 Nov 2025 11:39:41 +0100 Subject: [PATCH 280/403] Allow multiple file names in the arguments. --- packages/essnmx/src/ess/nmx/_executable_helper.py | 8 ++++++-- packages/essnmx/src/ess/nmx/executables.py | 7 ++++++- packages/essnmx/tests/executable_test.py | 4 ++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index 9b70673e..763f5f53 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -12,7 +12,7 @@ class InputConfig(BaseModel): # File IO - input_file: str + input_file: list[str] swmr: bool = False # Detector selection detector_ids: list[int | str] = [0, 1, 2] @@ -25,7 +25,11 @@ class InputConfig(BaseModel): def add_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: group = parser.add_argument_group("Input Options") group.add_argument( - "--input-file", type=str, help="Path to the input file", required=True + "--input-file", + type=str, + nargs="+", + help="Path to the input file", + required=True, ) group.add_argument( "--swmr", action="store_true", help="Open the input file in SWMR mode" diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index a3780d4d..ba610434 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -366,7 +366,12 @@ def main() -> None: parser = ReductionConfig.build_argument_parser() config = ReductionConfig.from_args(parser.parse_args()) - input_file = pathlib.Path(config.inputs.input_file).resolve() + if len(config.inputs.input_file) > 1: + raise NotImplementedError( + "Multiple input files are not supported yet in this executable." + ) + + input_file = pathlib.Path(config.inputs.input_file[0]).resolve() output_file = pathlib.Path(config.output.output_file).resolve() logger = build_logger(config.output) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index ecf72a5a..6abb3681 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -44,7 +44,7 @@ def _build_arg_list_from_pydantic_instance(*instances: pydantic.BaseModel) -> li def _default_config() -> ReductionConfig: """Helper to create a default ReductionConfig instance.""" return ReductionConfig( - inputs=InputConfig(input_file=''), + inputs=InputConfig(input_file=['']), workflow=WorkflowConfig(), output=OutputConfig(), ) @@ -71,7 +71,7 @@ def test_reduction_config() -> None: """Test ReductionConfig argument parsing.""" # Build config instances with non-default values. input_options = InputConfig( - input_file='test-input.h5', + input_file=['test-input.h5'], swmr=True, detector_ids=[0, 1, 2, 3], iter_chunk=True, From ffd146b86749f1cb70e3af27a991c0848b856939 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 17 Nov 2025 11:59:06 +0100 Subject: [PATCH 281/403] Allow pattern style input path. --- .../essnmx/src/ess/nmx/_executable_helper.py | 13 +++++++++++ packages/essnmx/src/ess/nmx/executables.py | 22 +++++++++++-------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index 763f5f53..68874d9f 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -1,7 +1,9 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) import argparse +import glob import logging +import pathlib import sys from typing import Literal @@ -286,3 +288,14 @@ def build_logger(args: argparse.Namespace | OutputConfig) -> logging.Logger: logger.setLevel(logging.INFO) logger.addHandler(logging.StreamHandler(sys.stdout)) return logger + + +def collect_matching_input_files(*input_file_patterns: str) -> list[pathlib.Path]: + """Helper to collect input files matching the given patterns.""" + + input_files: list[str] = [] + for pattern in input_file_patterns: + input_files.extend(glob.glob(pattern)) + + # Remove duplicates and sort + return sorted({pathlib.Path(f).resolve() for f in input_files}) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index ba610434..c744ff5e 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -9,7 +9,11 @@ import scipp as sc import scippnexus as snx -from ._executable_helper import ReductionConfig, build_logger +from ._executable_helper import ( + ReductionConfig, + build_logger, + collect_matching_input_files, +) from .nexus import ( _compute_positions, _export_detector_metadata_as_nxlauetof, @@ -158,7 +162,7 @@ def build_toa_bin_edges( def reduction( *, - input_file: pathlib.Path, + input_file: list[pathlib.Path], output_file: pathlib.Path, chunk_size: int = 1_000, detector_ids: list[int | str], @@ -219,6 +223,11 @@ def reduction( A DataGroup containing the reduced data for each selected detector. """ + if len(input_file) != 1: + raise NotImplementedError( + "Currently, only a single input file is supported for reduction." + ) + import scippnexus as snx if logger is None: @@ -229,7 +238,7 @@ def reduction( toa_bin_edges = build_toa_bin_edges( min_toa=min_toa, max_toa=max_toa, toa_bin_edges=toa_bin_edges ) - with snx.File(input_file) as f: + with snx.File(input_file[0]) as f: intrument_group = f['entry/instrument'] dets = intrument_group[snx.NXdetector] detector_group_keys = list(dets.keys()) @@ -366,12 +375,7 @@ def main() -> None: parser = ReductionConfig.build_argument_parser() config = ReductionConfig.from_args(parser.parse_args()) - if len(config.inputs.input_file) > 1: - raise NotImplementedError( - "Multiple input files are not supported yet in this executable." - ) - - input_file = pathlib.Path(config.inputs.input_file[0]).resolve() + input_file = collect_matching_input_files(*config.inputs.input_file) output_file = pathlib.Path(config.output.output_file).resolve() logger = build_logger(config.output) From 853f065509b2c020bf6032e51c2eb80bb55fcd6b Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 17 Nov 2025 13:01:15 +0100 Subject: [PATCH 282/403] Allow single file path to reduction interface. --- packages/essnmx/src/ess/nmx/executables.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index c744ff5e..c26bdba6 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -162,7 +162,7 @@ def build_toa_bin_edges( def reduction( *, - input_file: list[pathlib.Path], + input_file: list[pathlib.Path] | pathlib.Path, output_file: pathlib.Path, chunk_size: int = 1_000, detector_ids: list[int | str], @@ -223,10 +223,14 @@ def reduction( A DataGroup containing the reduced data for each selected detector. """ - if len(input_file) != 1: + if isinstance(input_file, list) and len(input_file) != 1: raise NotImplementedError( "Currently, only a single input file is supported for reduction." ) + elif isinstance(input_file, list): + input_file_path = input_file[0] + else: + input_file_path = input_file import scippnexus as snx @@ -238,7 +242,7 @@ def reduction( toa_bin_edges = build_toa_bin_edges( min_toa=min_toa, max_toa=max_toa, toa_bin_edges=toa_bin_edges ) - with snx.File(input_file[0]) as f: + with snx.File(input_file_path) as f: intrument_group = f['entry/instrument'] dets = intrument_group[snx.NXdetector] detector_group_keys = list(dets.keys()) From ef82de920d4f8291c98e8b553f8af861cbadfa45 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 17 Nov 2025 13:03:11 +0100 Subject: [PATCH 283/403] Cleanup code. --- packages/essnmx/src/ess/nmx/executables.py | 26 ++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index c26bdba6..34562137 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -160,6 +160,22 @@ def build_toa_bin_edges( ) +def _retrieve_input_file(input_file: list[pathlib.Path] | pathlib.Path) -> pathlib.Path: + """Temporary helper to retrieve a single input file from the list + Until multiple input file support is implemented. + """ + if isinstance(input_file, list) and len(input_file) != 1: + raise NotImplementedError( + "Currently, only a single input file is supported for reduction." + ) + elif isinstance(input_file, list): + input_file_path = input_file[0] + else: + input_file_path = input_file + + return input_file_path + + def reduction( *, input_file: list[pathlib.Path] | pathlib.Path, @@ -223,15 +239,6 @@ def reduction( A DataGroup containing the reduced data for each selected detector. """ - if isinstance(input_file, list) and len(input_file) != 1: - raise NotImplementedError( - "Currently, only a single input file is supported for reduction." - ) - elif isinstance(input_file, list): - input_file_path = input_file[0] - else: - input_file_path = input_file - import scippnexus as snx if logger is None: @@ -242,6 +249,7 @@ def reduction( toa_bin_edges = build_toa_bin_edges( min_toa=min_toa, max_toa=max_toa, toa_bin_edges=toa_bin_edges ) + input_file_path = _retrieve_input_file(input_file) with snx.File(input_file_path) as f: intrument_group = f['entry/instrument'] dets = intrument_group[snx.NXdetector] From 969fc0785f6454bdbad64dd1e3428756d29a541e Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 17 Nov 2025 13:11:00 +0100 Subject: [PATCH 284/403] Add more precise helper text. --- packages/essnmx/src/ess/nmx/_executable_helper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index 68874d9f..fe4cd3f4 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -30,7 +30,9 @@ def add_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: "--input-file", type=str, nargs="+", - help="Path to the input file", + help="Path to the input file. If multiple file paths are given," + " the output(histogram) will be merged(summed) " + "and will not save individual outputs per input file. ", required=True, ) group.add_argument( From 9d905c4f4c7d620fa12423e0b78f11d4ae3be354 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 19 Nov 2025 13:39:19 +0100 Subject: [PATCH 285/403] Use pydantic field to automate argument building and parsing. --- .../essnmx/src/ess/nmx/_executable_helper.py | 331 ++++++++++-------- 1 file changed, 188 insertions(+), 143 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index fe4cd3f4..029aa2dd 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -5,168 +5,213 @@ import logging import pathlib import sys -from typing import Literal +from enum import Enum +from functools import partial +from types import UnionType +from typing import Literal, Self, TypeGuard, Union, get_args, get_origin -from pydantic import BaseModel +from pydantic import BaseModel, Field +from pydantic.fields import FieldInfo +from pydantic_core import PydanticUndefined from .types import Compression -class InputConfig(BaseModel): - # File IO - input_file: list[str] - swmr: bool = False - # Detector selection - detector_ids: list[int | str] = [0, 1, 2] - # Chunking options - iter_chunk: bool = False - chunk_size_pulse: int = 1 - chunk_size_events: int = 0 +def _validate_annotation(annotation) -> TypeGuard[type]: + return not ( + isinstance(annotation, type) + or isinstance((origin_type := get_origin(annotation)), type) + or (origin_type is UnionType) + or (origin_type is Union) # typing.Optional is Union[X, NoneType] + ) - @classmethod - def add_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: - group = parser.add_argument_group("Input Options") - group.add_argument( - "--input-file", - type=str, - nargs="+", - help="Path to the input file. If multiple file paths are given," - " the output(histogram) will be merged(summed) " - "and will not save individual outputs per input file. ", - required=True, - ) - group.add_argument( - "--swmr", action="store_true", help="Open the input file in SWMR mode" - ) - group.add_argument( - "--detector-ids", - type=int, - nargs="+", - default=[0, 1, 2], - help="Detector indices to process", - ) - chunk_option_group = parser.add_argument_group("Chunking Options") - chunk_option_group.add_argument( - "--iter-chunk", - action="store_true", - help="Whether to process the input file in chunks " - " based on the hdf5 dataset chunk size. " - "It is ignored if hdf5 dataset is not chunked. " - "If True, it overrides chunk-size-pulse and chunk-size-events options.", - ) - chunk_option_group.add_argument( - "--chunk-size-pulse", - type=int, - default=0, - help="Number of pulses to process in each chunk. " - "If 0 or negative, process all pulses at once.", - ) - chunk_option_group.add_argument( - "--chunk-size-events", - type=int, - default=0, - help="Number of events to process in each chunk. " - "If 0 or negative, process all events at once." - "If both chunk-size-pulse and chunk-size-events are set, " - "chunk-size-pulse is preferred.", - ) - return parser - @classmethod - def from_args(cls, args: argparse.Namespace) -> "InputConfig": - return cls( - input_file=args.input_file, - swmr=args.swmr, - detector_ids=args.detector_ids, - chunk_size_pulse=args.chunk_size_pulse, - chunk_size_events=args.chunk_size_events, - iter_chunk=args.iter_chunk, - ) +def _get_no_nonetype_args(annotation) -> type: + origin_type = get_origin(annotation) + if (origin_type is UnionType or origin_type is Union) and type(None) in ( + union_args := get_args(annotation) + ): + arg_types = set(union_args) - {type(None)} + if len(arg_types) > 1: + raise TypeError( + "Optional type with single non-None type is not supported: " + f"{annotation}" + ) + return next(iter(arg_types)) + return annotation -class WorkflowConfig(BaseModel): - nbins: int = 50 - min_toa: int = 0 - max_toa: int = int((1 / 14) * 1_000) - fast_axis: Literal['x', 'y'] | None = None +def _is_appendable_type(annotation) -> bool: + return get_origin(annotation) in (list, tuple, set) + +def _retrieve_field_value( + field_name: str, field_info: FieldInfo, args: argparse.Namespace +): + if isinstance(field_info.annotation, type) and issubclass( + field_info.annotation, Enum + ): + return field_info.annotation[getattr(args, field_name)] + return getattr(args, field_name) + + +class CommandArgument(BaseModel): @classmethod def add_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: - group = parser.add_argument_group("Workflow Options") - group.add_argument( - "--nbins", - type=int, - default=50, - help="Number of TOF bins", - ) - group.add_argument( - "--min-toa", - type=int, - default=0, - help="Minimum time of arrival (TOA) in [ms].", - ) - group.add_argument( - "--max-toa", - type=int, - default=int((1 / 14) * 1_000), - help="Maximum time of arrival (TOA) in [ms].", - ) - group.add_argument( - "--fast-axis", - type=str, - choices=['x', 'y', None], - default=None, - help="Specify the fast axis of the detector. " - "If None, it will be determined " - "automatically based on the pixel offsets.", - ) + group = parser.add_argument_group(cls.model_config.get("title", cls.__name__)) + for field_name, field_info in cls.model_fields.items(): + add_argument = partial( + group.add_argument, f"--{field_name.replace('_', '-')}" + ) + + if _validate_annotation(field_info.annotation): + raise TypeError(f"Unsupported annotation type: {field_info.annotation}") + + arg_type = _get_no_nonetype_args(field_info.annotation) + if _is_appendable_type(arg_type): + nargs = '+' + arg_type = get_args(field_info.annotation)[0] + else: + nargs = None + arg_type = arg_type + + required = field_info.default is PydanticUndefined + default = ... if required else field_info.default + + if arg_type is bool: + add_argument = partial(add_argument, action='store_true') + elif isinstance(arg_type, type) and issubclass(arg_type, Enum): + add_argument = partial( + add_argument, + type=str, + choices=[e.name for e in arg_type], + ) + default = default.name if isinstance(default, Enum) else default + elif get_origin(arg_type) is Literal: + add_argument = partial( + add_argument, + type=str, + choices=[str(lit) for lit in get_args(arg_type)], + ) + else: + add_argument = partial(add_argument, type=arg_type, nargs=nargs) + + help_text = ' '.join( + [field_info.description or '', f"(default: {default})"] + ) + add_argument(default=default, required=required, help=help_text) + return parser @classmethod - def from_args(cls, args: argparse.Namespace) -> "WorkflowConfig": - return cls( - nbins=args.nbins, - min_toa=args.min_toa, - max_toa=args.max_toa, - fast_axis=args.fast_axis, - ) + def from_args(cls, args: argparse.Namespace) -> Self: + kwargs = { + field_name: _retrieve_field_value(field_name, field_info, args) + for field_name, field_info in cls.model_fields.items() + } + return cls(**kwargs) -class OutputConfig(BaseModel): - # Log verbosity - verbose: bool = False - # File output - output_file: str = "scipp_output.h5" - compression: Compression = Compression.BITSHUFFLE_LZ4 +class InputConfig(CommandArgument, BaseModel): + # Add title of the basemodel + model_config = {"title": "Input Configuration"} + # File IO + input_file: list[str] = Field( + title="Input File", + description="Path to the input file. If multiple file paths are given," + " the output(histogram) will be merged(summed) " + "and will not save individual outputs per input file. ", + ) + swmr: bool = Field( + title="SWMR Mode", + description="Open the input file in SWMR mode", + default=False, + ) + # Detector selection + detector_ids: list[int] = Field( + title="Detector IDs", + description="Detector indices to process", + default=[0, 1, 2], + ) + # Chunking options + iter_chunk: bool = Field( + title="Iterate in Chunks", + description="Whether to process the input file in chunks " + " based on the hdf5 dataset chunk size. " + "It is ignored if hdf5 dataset is not chunked. " + "If True, it overrides chunk-size-pulse and chunk-size-events options.", + default=False, + ) + chunk_size_pulse: int = Field( + title="Chunk Size Pulse", + description="Number of pulses to process in each chunk. " + "If 0 or negative, process all pulses at once.", + default=0, + ) + chunk_size_events: int = Field( + title="Chunk Size Events", + description="Number of events to process in each chunk. " + "If 0 or negative, process all events at once." + "If both chunk-size-pulse and chunk-size-events are set, " + "chunk-size-pulse is preferred.", + default=0, + ) - @classmethod - def add_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: - group = parser.add_argument_group("Output Options") - group.add_argument( - "--verbose", "-v", action="store_true", help="Increase output verbosity" - ) - group.add_argument( - "--output-file", - type=str, - default="scipp_output.h5", - help="Path to the output file", - ) - group.add_argument( - "--compression", - type=str, - default=Compression.BITSHUFFLE_LZ4.name, - choices=[compression_key.name for compression_key in Compression], - help="Compress option of reduced output file. Default: BITSHUFFLE_LZ4", - ) - return parser - @classmethod - def from_args(cls, args: argparse.Namespace) -> "OutputConfig": - return cls( - verbose=args.verbose, - output_file=args.output_file, - compression=Compression[args.compression], - ) +class TOAUnit(Enum): + ms = 'ms' + us = 'us' + ns = 'ns' + + +class WorkflowConfig(CommandArgument, BaseModel): + nbins: int = Field( + title="Number of TOF Bins", + description="Number of TOF bins", + default=50, + ) + min_toa: int = Field( + title="Minimum Time of Arrival", + description="Minimum time of arrival (TOA) in [toa_unit].", + default=0, + ) + max_toa: int = Field( + title="Maximum Time of Arrival", + description="Maximum time of arrival (TOA) in [toa_unit].", + default=int((1 / 14) * 1_000), + ) + toa_unit: TOAUnit = Field( + title="Maximum Time of Arrival", + description="Unit of TOA.", + default=TOAUnit.ms, + ) + fast_axis: Literal['x', 'y'] | None = Field( + title="Fast Axis", + description="Specify the fast axis of the detector. " + "If None, it will be determined " + "automatically based on the pixel offsets.", + default=None, + ) + + +class OutputConfig(CommandArgument, BaseModel): + # Log verbosity + verbose: bool = Field( + title="Verbose Logging", + description="Increase output verbosity.", + default=False, + ) + # File output + output_file: str = Field( + title="Output File", + description="Path to the output file.", + default="scipp_output.h5", + ) + compression: Compression = Field( + title="Compression", + description="Compress option of reduced output file.", + default=Compression.BITSHUFFLE_LZ4, + ) class ReductionConfig(BaseModel): From cde4422878acab7e5913a2edaa7cda7321d266c6 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 19 Nov 2025 13:42:42 +0100 Subject: [PATCH 286/403] Add model config attribute. --- packages/essnmx/src/ess/nmx/_executable_helper.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index 029aa2dd..00777617 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -165,6 +165,8 @@ class TOAUnit(Enum): class WorkflowConfig(CommandArgument, BaseModel): + # Add title of the basemodel + model_config = {"title": "Workflow Configuration"} nbins: int = Field( title="Number of TOF Bins", description="Number of TOF bins", @@ -195,6 +197,8 @@ class WorkflowConfig(CommandArgument, BaseModel): class OutputConfig(CommandArgument, BaseModel): + # Add title of the basemodel + model_config = {"title": "Output Configuration"} # Log verbosity verbose: bool = Field( title="Verbose Logging", From 7ba7c3dd4e1596d4d2ed61abbf0cc97fddb96103 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 19 Nov 2025 13:47:22 +0100 Subject: [PATCH 287/403] Update tests. --- packages/essnmx/src/ess/nmx/_executable_helper.py | 2 +- packages/essnmx/tests/executable_test.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index 00777617..3720492b 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -267,7 +267,7 @@ def to_command_arguments(self, one_line: bool = True) -> list[str] | str: arg_list.append(k) if isinstance(v, list): arg_list.extend(str(item) for item in v) - elif isinstance(v, Compression): + elif isinstance(v, Enum): arg_list.append(v.name) else: arg_list.append(str(v)) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index 6abb3681..aa20c1d5 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -3,6 +3,7 @@ import pathlib import subprocess +from enum import Enum import pydantic import pytest @@ -14,6 +15,7 @@ InputConfig, OutputConfig, ReductionConfig, + TOAUnit, WorkflowConfig, ) from ess.nmx.types import Compression @@ -31,7 +33,7 @@ def _build_arg_list_from_pydantic_instance(*instances: pydantic.BaseModel) -> li arg_list.append(k) if isinstance(v, list): arg_list.extend(str(item) for item in v) - elif isinstance(v, Compression): + elif isinstance(v, Enum): arg_list.append(v.name) else: arg_list.append(str(v)) @@ -78,7 +80,9 @@ def test_reduction_config() -> None: chunk_size_pulse=10, chunk_size_events=100000, ) - workflow_options = WorkflowConfig(nbins=100, min_toa=10, max_toa=100, fast_axis='y') + workflow_options = WorkflowConfig( + nbins=100, min_toa=10, max_toa=100_000, toa_unit=TOAUnit.us, fast_axis='y' + ) output_options = OutputConfig( output_file='test-output.h5', compression=Compression.NONE, verbose=True ) From e7eeec428d5a15e1c1eb72cee4196b99245149a1 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 20 Nov 2025 09:37:17 +0100 Subject: [PATCH 288/403] Fix typo --- packages/essnmx/src/ess/nmx/_executable_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index 3720492b..1fcf7795 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -183,7 +183,7 @@ class WorkflowConfig(CommandArgument, BaseModel): default=int((1 / 14) * 1_000), ) toa_unit: TOAUnit = Field( - title="Maximum Time of Arrival", + title="Unit of TOA", description="Unit of TOA.", default=TOAUnit.ms, ) From 4adcbf1cc3b71b28846297a36fa218ceb5ab40d2 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 20 Nov 2025 09:44:30 +0100 Subject: [PATCH 289/403] Use free function instead of inheriting a parent class. [skip ci] --- .../essnmx/src/ess/nmx/_executable_helper.py | 176 +++++++----------- 1 file changed, 63 insertions(+), 113 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index 1fcf7795..8984dee3 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -8,7 +8,7 @@ from enum import Enum from functools import partial from types import UnionType -from typing import Literal, Self, TypeGuard, Union, get_args, get_origin +from typing import Literal, TypeGuard, TypeVar, Union, get_args, get_origin from pydantic import BaseModel, Field from pydantic.fields import FieldInfo @@ -55,64 +55,65 @@ def _retrieve_field_value( return getattr(args, field_name) -class CommandArgument(BaseModel): - @classmethod - def add_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser: - group = parser.add_argument_group(cls.model_config.get("title", cls.__name__)) - for field_name, field_info in cls.model_fields.items(): +def add_args_from_pydantic_model( + model_cls: type[BaseModel], parser: argparse.ArgumentParser +) -> argparse.ArgumentParser: + group = parser.add_argument_group( + model_cls.model_config.get("title", model_cls.__name__) + ) + for field_name, field_info in model_cls.model_fields.items(): + add_argument = partial(group.add_argument, f"--{field_name.replace('_', '-')}") + + if _validate_annotation(field_info.annotation): + raise TypeError(f"Unsupported annotation type: {field_info.annotation}") + + arg_type = _get_no_nonetype_args(field_info.annotation) + if _is_appendable_type(arg_type): + nargs = '+' + arg_type = get_args(field_info.annotation)[0] + else: + nargs = None + arg_type = arg_type + + required = field_info.default is PydanticUndefined + default = ... if required else field_info.default + + if arg_type is bool: + add_argument = partial(add_argument, action='store_true') + elif isinstance(arg_type, type) and issubclass(arg_type, Enum): add_argument = partial( - group.add_argument, f"--{field_name.replace('_', '-')}" + add_argument, + type=str, + choices=[e.name for e in arg_type], ) - - if _validate_annotation(field_info.annotation): - raise TypeError(f"Unsupported annotation type: {field_info.annotation}") - - arg_type = _get_no_nonetype_args(field_info.annotation) - if _is_appendable_type(arg_type): - nargs = '+' - arg_type = get_args(field_info.annotation)[0] - else: - nargs = None - arg_type = arg_type - - required = field_info.default is PydanticUndefined - default = ... if required else field_info.default - - if arg_type is bool: - add_argument = partial(add_argument, action='store_true') - elif isinstance(arg_type, type) and issubclass(arg_type, Enum): - add_argument = partial( - add_argument, - type=str, - choices=[e.name for e in arg_type], - ) - default = default.name if isinstance(default, Enum) else default - elif get_origin(arg_type) is Literal: - add_argument = partial( - add_argument, - type=str, - choices=[str(lit) for lit in get_args(arg_type)], - ) - else: - add_argument = partial(add_argument, type=arg_type, nargs=nargs) - - help_text = ' '.join( - [field_info.description or '', f"(default: {default})"] + default = default.name if isinstance(default, Enum) else default + elif get_origin(arg_type) is Literal: + add_argument = partial( + add_argument, + type=str, + choices=[str(lit) for lit in get_args(arg_type)], ) - add_argument(default=default, required=required, help=help_text) + else: + add_argument = partial(add_argument, type=arg_type, nargs=nargs) - return parser + help_text = ' '.join([field_info.description or '', f"(default: {default})"]) + add_argument(default=default, required=required, help=help_text) + + return parser - @classmethod - def from_args(cls, args: argparse.Namespace) -> Self: - kwargs = { - field_name: _retrieve_field_value(field_name, field_info, args) - for field_name, field_info in cls.model_fields.items() - } - return cls(**kwargs) +T = TypeVar('T', bound=BaseModel) -class InputConfig(CommandArgument, BaseModel): + +def from_args(cls: type[T], args: argparse.Namespace) -> T: + kwargs = { + field_name: _retrieve_field_value(field_name, field_info, args) + for field_name, field_info in cls.model_fields.items() + } + return cls(**kwargs) + + +class InputConfig(BaseModel): # Add title of the basemodel model_config = {"title": "Input Configuration"} # File IO @@ -164,7 +165,7 @@ class TOAUnit(Enum): ns = 'ns' -class WorkflowConfig(CommandArgument, BaseModel): +class WorkflowConfig(BaseModel): # Add title of the basemodel model_config = {"title": "Workflow Configuration"} nbins: int = Field( @@ -196,7 +197,7 @@ class WorkflowConfig(CommandArgument, BaseModel): ) -class OutputConfig(CommandArgument, BaseModel): +class OutputConfig(BaseModel): # Add title of the basemodel model_config = {"title": "Output Configuration"} # Log verbosity @@ -219,6 +220,8 @@ class OutputConfig(CommandArgument, BaseModel): class ReductionConfig(BaseModel): + """Container for all reduction configurations.""" + inputs: InputConfig workflow: WorkflowConfig output: OutputConfig @@ -229,17 +232,17 @@ def build_argument_parser(cls) -> argparse.ArgumentParser: description="Command line arguments for the ESS NMX reduction. " "It assumes 14 Hz pulse speed." ) - parser = InputConfig.add_args(parser) - parser = WorkflowConfig.add_args(parser) - parser = OutputConfig.add_args(parser) + parser = add_args_from_pydantic_model(model_cls=InputConfig, parser=parser) + parser = add_args_from_pydantic_model(model_cls=WorkflowConfig, parser=parser) + parser = add_args_from_pydantic_model(model_cls=OutputConfig, parser=parser) return parser @classmethod def from_args(cls, args: argparse.Namespace) -> "ReductionConfig": return cls( - inputs=InputConfig.from_args(args), - workflow=WorkflowConfig.from_args(args), - output=OutputConfig.from_args(args), + inputs=from_args(InputConfig, args), + workflow=from_args(WorkflowConfig, args), + output=from_args(OutputConfig, args), ) @property @@ -280,59 +283,6 @@ def to_command_arguments(self, one_line: bool = True) -> list[str] | str: return arg_list -def build_reduction_arg_parser() -> argparse.ArgumentParser: - import warnings - - warnings.warn( - "build_reduction_arg_parser is deprecated and will be removed " - "in the future release (>=26.11.0) " - "Please use the config classes to handle command line arguments.", - DeprecationWarning, - stacklevel=2, - ) - parser = argparse.ArgumentParser( - description="Command line arguments for the NMX reduction. " - "It assumes 14 Hz pulse speed." - ) - input_arg_group = parser.add_argument_group("Input Options") - input_arg_group.add_argument( - "--input_file", type=str, help="Path to the input file", required=True - ) - input_arg_group.add_argument( - "--nbins", - type=int, - default=50, - help="Number of TOF bins", - ) - input_arg_group.add_argument( - "--detector_ids", - type=int, - nargs="+", - default=[0, 1, 2], - help="Detector indices to process", - ) - - output_arg_group = parser.add_argument_group("Output Options") - output_arg_group.add_argument( - "--output_file", - type=str, - default="scipp_output.h5", - help="Path to the output file", - ) - output_arg_group.add_argument( - "--compression", - type=str, - default=Compression.BITSHUFFLE_LZ4.name, - choices=[compression_key.name for compression_key in Compression], - help="Compress option of reduced output file. Default: BITSHUFFLE_LZ4", - ) - output_arg_group.add_argument( - "--verbose", "-v", action="store_true", help="Increase output verbosity" - ) - - return parser - - def build_logger(args: argparse.Namespace | OutputConfig) -> logging.Logger: logger = logging.getLogger(__name__) if args.verbose: From 03cf13eb35c197a8ddd124f2f9de1c4e58c581c5 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 20 Nov 2025 10:44:57 +0100 Subject: [PATCH 290/403] Use StrEnum instead of Enum --- .../essnmx/src/ess/nmx/_executable_helper.py | 45 +++++++++++++------ packages/essnmx/src/ess/nmx/types.py | 6 +-- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index 8984dee3..8e91719e 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -1,11 +1,11 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) import argparse +import enum import glob import logging import pathlib import sys -from enum import Enum from functools import partial from types import UnionType from typing import Literal, TypeGuard, TypeVar, Union, get_args, get_origin @@ -18,11 +18,28 @@ def _validate_annotation(annotation) -> TypeGuard[type]: - return not ( - isinstance(annotation, type) - or isinstance((origin_type := get_origin(annotation)), type) - or (origin_type is UnionType) - or (origin_type is Union) # typing.Optional is Union[X, NoneType] + # Supported annotation for command arguments: + # - Atomic types: int, float, str, bool, enum.StrEnum, Literal + # - Optional[AtomicType] + # - List[AtomicType], Tuple[AtomicType, ...], Set[AtomicType] + def _validate_atomic_type(annotation) -> bool: + return ( + (annotation in (int, float, str, bool)) + or (isinstance(annotation, type) and issubclass(annotation, enum.StrEnum)) + or (get_origin(annotation) is Literal) + ) + + return ( + _validate_atomic_type(annotation) + or ( + (origin := get_origin(annotation)) in (Union, UnionType) + and _validate_atomic_type(_get_no_nonetype_args(annotation)) + ) + or ( + origin in (list, tuple, set) + and len(args := get_args(annotation)) > 0 + and _validate_atomic_type(args[0]) + ) ) @@ -49,7 +66,7 @@ def _retrieve_field_value( field_name: str, field_info: FieldInfo, args: argparse.Namespace ): if isinstance(field_info.annotation, type) and issubclass( - field_info.annotation, Enum + field_info.annotation, enum.StrEnum ): return field_info.annotation[getattr(args, field_name)] return getattr(args, field_name) @@ -64,7 +81,7 @@ def add_args_from_pydantic_model( for field_name, field_info in model_cls.model_fields.items(): add_argument = partial(group.add_argument, f"--{field_name.replace('_', '-')}") - if _validate_annotation(field_info.annotation): + if not _validate_annotation(field_info.annotation): raise TypeError(f"Unsupported annotation type: {field_info.annotation}") arg_type = _get_no_nonetype_args(field_info.annotation) @@ -80,13 +97,13 @@ def add_args_from_pydantic_model( if arg_type is bool: add_argument = partial(add_argument, action='store_true') - elif isinstance(arg_type, type) and issubclass(arg_type, Enum): + elif isinstance(arg_type, type) and issubclass(arg_type, enum.StrEnum): add_argument = partial( add_argument, type=str, - choices=[e.name for e in arg_type], + choices=[str(e) for e in arg_type], ) - default = default.name if isinstance(default, Enum) else default + default = default.name if isinstance(default, enum.StrEnum) else default elif get_origin(arg_type) is Literal: add_argument = partial( add_argument, @@ -159,7 +176,7 @@ class InputConfig(BaseModel): ) -class TOAUnit(Enum): +class TOAUnit(enum.StrEnum): ms = 'ms' us = 'us' ns = 'ns' @@ -270,8 +287,8 @@ def to_command_arguments(self, one_line: bool = True) -> list[str] | str: arg_list.append(k) if isinstance(v, list): arg_list.extend(str(item) for item in v) - elif isinstance(v, Enum): - arg_list.append(v.name) + elif isinstance(v, enum.StrEnum): + arg_list.append(v.value) else: arg_list.append(str(v)) elif v is True: diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 356117e0..baa7c3b8 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -79,11 +79,11 @@ class NMXRawDataMetadata: max_toa: MaximumTimeOfArrival -class Compression(enum.Enum): +class Compression(enum.StrEnum): """Compression type of the output file. These options are written as enum for future extensibility. """ - NONE = 0 - BITSHUFFLE_LZ4 = 1 + NONE = 'NONE' + BITSHUFFLE_LZ4 = 'BITSHUFFLE_LZ4' From 5f8cfb50236cb198a3a50394d57052e28cf730a1 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 20 Nov 2025 11:31:06 +0100 Subject: [PATCH 291/403] Add docstring for argument parser related helpers. --- .../essnmx/src/ess/nmx/_executable_helper.py | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index 8e91719e..8ce2f653 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -18,10 +18,6 @@ def _validate_annotation(annotation) -> TypeGuard[type]: - # Supported annotation for command arguments: - # - Atomic types: int, float, str, bool, enum.StrEnum, Literal - # - Optional[AtomicType] - # - List[AtomicType], Tuple[AtomicType, ...], Set[AtomicType] def _validate_atomic_type(annotation) -> bool: return ( (annotation in (int, float, str, bool)) @@ -73,8 +69,33 @@ def _retrieve_field_value( def add_args_from_pydantic_model( - model_cls: type[BaseModel], parser: argparse.ArgumentParser + *, model_cls: type[BaseModel], parser: argparse.ArgumentParser ) -> argparse.ArgumentParser: + """Add arguments to the parser from the pydantic model class. + + Each field in the model class is added as a command line argument + with the name `--{field-name}`. + Arguments are added based on fields' information: + - type annotation (type, choices, nargs) + - description (help text) + - default value (default, required and help text) + + Supported annotation for command arguments: + - Atomic types: int, float, str, bool, enum.StrEnum, Literal + - Optional[AtomicType] + - List[AtomicType], Tuple[AtomicType, ...], Set[AtomicType] + + Parameters + ---------- + model_cls: + Pydantic model class to extract the arguments from. + parser: + Argument parser to add the arguments to. + It adds a new argument group for the model. + The group name is taken from the model's title config if available, + otherwise the model class name is used. + + """ group = parser.add_argument_group( model_cls.model_config.get("title", model_cls.__name__) ) @@ -123,6 +144,10 @@ def add_args_from_pydantic_model( def from_args(cls: type[T], args: argparse.Namespace) -> T: + """Create an instance of the pydantic model from the argparse namespace. + + It ignores any extra arguments in the namespace that are not part of the model. + """ kwargs = { field_name: _retrieve_field_value(field_name, field_info, args) for field_name, field_info in cls.model_fields.items() From d148d4d661026792a2a29c4fe81ac8dff6f3a603 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 25 Nov 2025 16:52:16 +0100 Subject: [PATCH 292/403] Add tof as a dependency to run tof simulation if needed. --- packages/essnmx/pyproject.toml | 1 + packages/essnmx/requirements/base.in | 1 + packages/essnmx/requirements/base.txt | 38 +++++++++++++---------- packages/essnmx/requirements/basetest.txt | 2 +- packages/essnmx/requirements/ci.txt | 4 +-- packages/essnmx/requirements/dev.txt | 6 ++-- packages/essnmx/requirements/docs.txt | 8 ++--- packages/essnmx/requirements/nightly.in | 1 + packages/essnmx/requirements/nightly.txt | 35 ++++++++++++--------- packages/essnmx/requirements/static.txt | 4 +-- 10 files changed, 58 insertions(+), 42 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index ba1be8e6..6bb45907 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -43,6 +43,7 @@ dependencies = [ "defusedxml>=0.7.1", "bitshuffle>=0.5.2", "msgpack>=1.0.8", + "tof>=25.12.0", ] dynamic = ["version"] diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index 36150236..76bb7361 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -15,3 +15,4 @@ gemmi>=0.6.6 defusedxml>=0.7.1 bitshuffle>=0.5.2 msgpack>=1.0.8 +tof>=25.12.0 diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 5406ed10..ccfe91fa 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:e5e735f23415c4cc7855c37cba24934ee1d0a01d +# SHA1:b8bb22bde983f5da456b1499bb8c783f42ba2bec # # This file was generated by pip-compile-multi. # To update, run: @@ -9,11 +9,11 @@ annotated-types==0.7.0 # via pydantic bitshuffle==0.5.2 # via -r base.in -certifi==2025.10.5 +certifi==2025.11.12 # via requests charset-normalizer==3.4.4 # via requests -click==8.3.0 +click==8.3.1 # via dask cloudpickle==3.1.2 # via dask @@ -23,9 +23,9 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -cython==3.1.6 +cython==3.2.1 # via bitshuffle -dask==2025.10.0 +dask==2025.11.0 # via -r base.in defusedxml==0.7.1 # via -r base.in @@ -33,13 +33,13 @@ dnspython==2.8.0 # via email-validator email-validator==2.3.0 # via scippneutron -essreduce==25.11.0 +essreduce==25.11.3 # via -r base.in fonttools==4.60.1 # via matplotlib fsspec==2025.10.0 # via dask -gemmi==0.7.3 +gemmi==0.7.4 # via -r base.in graphviz==0.21 # via -r base.in @@ -58,6 +58,7 @@ lazy-loader==0.4 # via # plopp # scippneutron + # tof locket==1.0.0 # via partd matplotlib==3.10.7 @@ -68,9 +69,9 @@ mpltoolbox==25.10.0 # via scippneutron msgpack==1.1.2 # via -r base.in -networkx==3.5 +networkx==3.6 # via cyclebane -numpy==2.3.4 +numpy==2.3.5 # via # bitshuffle # contourpy @@ -94,15 +95,18 @@ pillow==12.0.0 # via matplotlib platformdirs==4.5.0 # via pooch -plopp==25.10.0 +plopp==25.11.0 # via # -r base.in # scippneutron + # tof pooch==1.8.2 - # via -r base.in -pydantic==2.12.3 + # via + # -r base.in + # tof +pydantic==2.12.4 # via scippneutron -pydantic-core==2.41.4 +pydantic-core==2.41.5 # via pydantic pyparsing==3.2.5 # via matplotlib @@ -111,7 +115,6 @@ python-dateutil==2.9.0.post0 # matplotlib # pandas # scippneutron - # scippnexus pytz==2025.2 # via pandas pyyaml==6.0.3 @@ -128,9 +131,10 @@ scipp==25.11.0 # essreduce # scippneutron # scippnexus -scippneutron==25.7.0 + # tof +scippneutron==25.11.0 # via essreduce -scippnexus==25.6.0 +scippnexus==25.11.0 # via # -r base.in # essreduce @@ -141,6 +145,8 @@ scipy==1.16.3 # scippnexus six==1.17.0 # via python-dateutil +tof==25.12.0 + # via -r base.in toolz==1.1.0 # via # dask diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt index 85cf4015..acbe0b32 100644 --- a/packages/essnmx/requirements/basetest.txt +++ b/packages/essnmx/requirements/basetest.txt @@ -13,5 +13,5 @@ pluggy==1.6.0 # via pytest pygments==2.19.2 # via pytest -pytest==8.4.2 +pytest==9.0.1 # via -r basetest.in diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index 5bc02f40..4b49511f 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -5,9 +5,9 @@ # # requirements upgrade # -cachetools==6.2.1 +cachetools==6.2.2 # via tox -certifi==2025.10.5 +certifi==2025.11.12 # via requests chardet==5.2.0 # via tox diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index a2d2832e..9f4a3f52 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -26,7 +26,7 @@ async-lru==2.0.5 # via jupyterlab cffi==2.0.0 # via argon2-cffi-bindings -copier==9.10.3 +copier==9.11.0 # via -r dev.in dunamai==1.25.0 # via copier @@ -65,7 +65,7 @@ jupyter-server==2.17.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.4.10 +jupyterlab==4.5.0 # via -r dev.in jupyterlab-server==2.28.0 # via jupyterlab @@ -75,7 +75,7 @@ notebook-shim==0.2.4 # via jupyterlab pip-compile-multi==3.2.2 # via -r dev.in -pip-tools==7.5.1 +pip-tools==7.5.2 # via pip-compile-multi plumbum==1.10.0 # via copier diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 44c31e87..4971a08f 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -12,7 +12,7 @@ alabaster==1.0.0 # via sphinx appnope==0.1.4 # via ipykernel -asttokens==3.0.0 +asttokens==3.0.1 # via stack-data attrs==25.4.0 # via @@ -54,7 +54,7 @@ ipydatawidgets==4.3.5 # via pythreejs ipykernel==7.1.0 # via -r docs.in -ipython==9.6.0 +ipython==9.7.0 # via # -r docs.in # ipykernel @@ -139,7 +139,7 @@ ptyprocess==0.7.0 # via pexpect pure-eval==0.2.3 # via stack-data -pydantic-settings==2.11.0 +pydantic-settings==2.12.0 # via autodoc-pydantic pydata-sphinx-theme==0.16.1 # via -r docs.in @@ -163,7 +163,7 @@ referencing==0.37.0 # via # jsonschema # jsonschema-specifications -rpds-py==0.28.0 +rpds-py==0.29.0 # via # jsonschema # referencing diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index d78c28a2..9fedb039 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -10,6 +10,7 @@ gemmi>=0.6.6 defusedxml>=0.7.1 bitshuffle>=0.5.2 msgpack>=1.0.8 +tof>=25.12.0 pytest>=7.0 scipp --index-url=https://pypi.anaconda.org/scipp-nightly-wheels/simple/ diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 6c5504ad..198ddbb4 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:c3fe0a59eb91d120f06ad2064b100fd7397b9055 +# SHA1:3d2f4d2f209cbd357fbb592e05b6a7f8201d89ee # # This file was generated by pip-compile-multi. # To update, run: @@ -12,11 +12,11 @@ annotated-types==0.7.0 # via pydantic bitshuffle==0.5.2 # via -r nightly.in -certifi==2025.10.5 +certifi==2025.11.12 # via requests charset-normalizer==3.4.4 # via requests -click==8.3.0 +click==8.3.1 # via dask cloudpickle==3.1.2 # via dask @@ -26,9 +26,9 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -cython==3.2.0b3 +cython==3.2.1 # via bitshuffle -dask==2025.10.0 +dask==2025.11.0 # via -r nightly.in defusedxml==0.8.0rc2 # via -r nightly.in @@ -36,13 +36,13 @@ dnspython==2.8.0 # via email-validator email-validator==2.3.0 # via scippneutron -essreduce==25.11.0 +essreduce==25.11.3 # via -r nightly.in fonttools==4.60.1 # via matplotlib fsspec==2025.10.0 # via dask -gemmi==0.7.3 +gemmi==0.7.4 # via -r nightly.in graphviz==0.21 # via -r nightly.in @@ -63,6 +63,7 @@ lazy-loader==0.4 # via # plopp # scippneutron + # tof locket==1.0.0 # via partd matplotlib==3.10.7 @@ -73,9 +74,9 @@ mpltoolbox==25.10.0 # via scippneutron msgpack==1.1.2 # via -r nightly.in -networkx==3.5 +networkx==3.6 # via cyclebane -numpy==2.3.4 +numpy==2.3.5 # via # bitshuffle # contourpy @@ -104,19 +105,22 @@ plopp @ git+https://github.com/scipp/plopp@main # via # -r nightly.in # scippneutron + # tof pluggy==1.6.0 # via pytest pooch==1.8.2 - # via -r nightly.in -pydantic==2.12.3 + # via + # -r nightly.in + # tof +pydantic==2.12.4 # via scippneutron -pydantic-core==2.41.4 +pydantic-core==2.41.5 # via pydantic pygments==2.19.2 # via pytest pyparsing==3.3.0a1 # via matplotlib -pytest==8.4.2 +pytest==9.0.1 # via -r nightly.in python-dateutil==2.9.0.post0 # via @@ -139,7 +143,8 @@ scipp==100.0.0.dev0 # essreduce # scippneutron # scippnexus -scippneutron==25.7.0 + # tof +scippneutron==25.11.0 # via essreduce scippnexus @ git+https://github.com/scipp/scippnexus@main # via @@ -152,6 +157,8 @@ scipy==1.16.3 # scippnexus six==1.17.0 # via python-dateutil +tof==25.12.0 + # via -r nightly.in toolz==1.1.0 # via # dask diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index 991a8921..4cc95f11 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -5,7 +5,7 @@ # # requirements upgrade # -cfgv==3.4.0 +cfgv==3.5.0 # via pre-commit distlib==0.4.0 # via virtualenv @@ -17,7 +17,7 @@ nodeenv==1.9.1 # via pre-commit platformdirs==4.5.0 # via virtualenv -pre-commit==4.3.0 +pre-commit==4.5.0 # via -r static.in pyyaml==6.0.3 # via pre-commit From 024750c3b48f2d28a51a295ce499616e2bc7f84c Mon Sep 17 00:00:00 2001 From: Jan-Lukas Wynen Date: Wed, 26 Nov 2025 15:11:32 +0100 Subject: [PATCH 293/403] Use common data registry from essreduce --- packages/essnmx/pyproject.toml | 2 +- packages/essnmx/src/ess/nmx/data/__init__.py | 61 ++++++++------------ 2 files changed, 25 insertions(+), 38 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 6bb45907..c43086ba 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -31,7 +31,7 @@ requires-python = ">=3.11" # Make sure to list one dependency per line. dependencies = [ "dask>=2022.1.0", - "essreduce>=25.11.0", # New domain types implemented, see https://scipp.github.io/essreduce/user-guide/reduction-workflow-guidelines.html#c-6-domain-types-for-data-products + "essreduce>25.11.0", # New domain types implemented, see https://scipp.github.io/essreduce/user-guide/reduction-workflow-guidelines.html#c-6-domain-types-for-data-products "graphviz", "plopp>=24.7.0", "sciline>=24.06.0", diff --git a/packages/essnmx/src/ess/nmx/data/__init__.py b/packages/essnmx/src/ess/nmx/data/__init__.py index 36919908..ff594429 100644 --- a/packages/essnmx/src/ess/nmx/data/__init__.py +++ b/packages/essnmx/src/ess/nmx/data/__init__.py @@ -2,34 +2,31 @@ # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) import pathlib -import pooch +from ess.reduce.data import Entry, make_registry _version = "0" __all__ = ["get_path", "small_mcstas_2_sample", "small_mcstas_3_sample"] -def _make_pooch() -> pooch.Pooch: - return pooch.create( - path=pooch.os_cache("essnmx"), - env="ESSNMX_DATA_DIR", - retry_if_failed=3, - base_url="https://public.esss.dk/groups/scipp/ess/nmx/", - version=_version, - registry={ - "small_mcstas_2_sample.h5": "md5:c3affe636397f8c9eea1d9c10a2bf487", - "small_mcstas_3_sample.h5": "md5:2afaac205d13ee857ee5364e3f1957a7", - "mtz_samples.tar.gz": "md5:bed1eaf604bbe8725c1f6a20ca79fcc0", - "mtz_random_samples.tar.gz": "md5:c8259ae2e605560ab88959e7109613b6", - "small_nmx_nexus.hdf": "md5:42cffb85e4ce7c1aaa5f7e81469b865e", - }, - ) - - -_pooch = _make_pooch() +_registry = make_registry( + "ess/nmx", + version="0", + files={ + "small_mcstas_2_sample.h5": "md5:c3affe636397f8c9eea1d9c10a2bf487", + "small_mcstas_3_sample.h5": "md5:2afaac205d13ee857ee5364e3f1957a7", + "mtz_samples.tar.gz": Entry( + alg="md5", chk="bed1eaf604bbe8725c1f6a20ca79fcc0", extractor="untar" + ), + "mtz_random_samples.tar.gz": Entry( + alg="md5", chk="c8259ae2e605560ab88959e7109613b6", extractor="untar" + ), + "small_nmx_nexus.hdf": "md5:42cffb85e4ce7c1aaa5f7e81469b865e", + }, +) -def small_mcstas_2_sample(): +def small_mcstas_2_sample() -> pathlib.Path: """McStas 2 file containing small number of events.""" import warnings @@ -45,7 +42,7 @@ def small_mcstas_2_sample(): return get_path("small_mcstas_2_sample.h5") -def small_mcstas_3_sample(): +def small_mcstas_3_sample() -> pathlib.Path: """McStas 3 file that contains only ``bank0(1-3)`` in the ``data`` group. Real McStas 3 file should contain more dataset under ``data`` group. @@ -53,14 +50,14 @@ def small_mcstas_3_sample(): return get_path("small_mcstas_3_sample.h5") -def get_path(name: str) -> str: +def get_path(name: str) -> pathlib.Path: """ Return the path to a data file bundled with ess nmx. This function only works with example data and cannot handle paths to custom files. """ - return _pooch.fetch(name) + return _registry.get_path(name) def get_small_mtz_samples() -> list[pathlib.Path]: @@ -68,12 +65,7 @@ def get_small_mtz_samples() -> list[pathlib.Path]: This samples also contain optional columns. """ - from pooch.processors import Untar - - return [ - pathlib.Path(file_path) - for file_path in _pooch.fetch("mtz_samples.tar.gz", processor=Untar()) - ] + return _registry.get_paths("mtz_samples.tar.gz") def get_small_random_mtz_samples() -> list[pathlib.Path]: @@ -84,15 +76,10 @@ def get_small_random_mtz_samples() -> list[pathlib.Path]: Use ``get_small_mtz_samples`` for testing since they are more representative of real data. """ - from pooch.processors import Untar - - return [ - pathlib.Path(file_path) - for file_path in _pooch.fetch("mtz_random_samples.tar.gz", processor=Untar()) - ] + return _registry.get_paths("mtz_random_samples.tar.gz") -def get_small_nmx_nexus() -> str: +def get_small_nmx_nexus() -> pathlib.Path: """Return the path to a small NMX NeXus file.""" - return _pooch.fetch("small_nmx_nexus.hdf") + return get_path("small_nmx_nexus.hdf") From 2e5e44dc931d948b024efe79c98ba7c2319bb093 Mon Sep 17 00:00:00 2001 From: Mridul Seth Date: Wed, 26 Nov 2025 18:03:59 +0100 Subject: [PATCH 294/403] tox -e deps with essreduce --- packages/essnmx/pyproject.toml | 2 +- packages/essnmx/requirements/base.in | 2 +- packages/essnmx/requirements/base.txt | 10 +++++++--- packages/essnmx/requirements/dev.txt | 2 ++ packages/essnmx/requirements/nightly.in | 2 +- packages/essnmx/requirements/nightly.txt | 12 ++++++++---- 6 files changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index c43086ba..6d67ab07 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -31,7 +31,7 @@ requires-python = ">=3.11" # Make sure to list one dependency per line. dependencies = [ "dask>=2022.1.0", - "essreduce>25.11.0", # New domain types implemented, see https://scipp.github.io/essreduce/user-guide/reduction-workflow-guidelines.html#c-6-domain-types-for-data-products + "essreduce>=25.11.5", # New domain types implemented, see https://scipp.github.io/essreduce/user-guide/reduction-workflow-guidelines.html#c-6-domain-types-for-data-products "graphviz", "plopp>=24.7.0", "sciline>=24.06.0", diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index 76bb7361..7f14a721 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -3,7 +3,7 @@ # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask>=2022.1.0 -essreduce>=25.11.0 +essreduce>=25.11.5 graphviz plopp>=24.7.0 sciline>=24.06.0 diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index ccfe91fa..12864a3e 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:b8bb22bde983f5da456b1499bb8c783f42ba2bec +# SHA1:60fc64eb056a0e8e93d0a6ebf0875a2e341193b5 # # This file was generated by pip-compile-multi. # To update, run: @@ -33,7 +33,7 @@ dnspython==2.8.0 # via email-validator email-validator==2.3.0 # via scippneutron -essreduce==25.11.3 +essreduce==25.11.5 # via -r base.in fonttools==4.60.1 # via matplotlib @@ -52,6 +52,8 @@ idna==3.11 # via # email-validator # requests +importlib-metadata==8.7.0 + # via dask kiwisolver==1.4.9 # via matplotlib lazy-loader==0.4 @@ -104,7 +106,7 @@ pooch==1.8.2 # via # -r base.in # tof -pydantic==2.12.4 +pydantic==2.12.5 # via scippneutron pydantic-core==2.41.5 # via pydantic @@ -162,6 +164,8 @@ tzdata==2025.2 # via pandas urllib3==2.5.0 # via requests +zipp==3.23.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index 9f4a3f52..6c15809f 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -73,6 +73,8 @@ lark==1.3.1 # via rfc3987-syntax notebook-shim==0.2.4 # via jupyterlab +overrides==7.7.0 + # via jupyter-server pip-compile-multi==3.2.2 # via -r dev.in pip-tools==7.5.2 diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index 9fedb039..29b77c0f 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -2,7 +2,7 @@ # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask>=2022.1.0 -essreduce>=25.11.0 +essreduce>=25.11.5 graphviz pooch>=1.5 pandas>=2.1.2 diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 198ddbb4..625fda38 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:3d2f4d2f209cbd357fbb592e05b6a7f8201d89ee +# SHA1:82815c7a739545007523e0e1a48d57f0f7e427db # # This file was generated by pip-compile-multi. # To update, run: @@ -36,7 +36,7 @@ dnspython==2.8.0 # via email-validator email-validator==2.3.0 # via scippneutron -essreduce==25.11.3 +essreduce==25.11.5 # via -r nightly.in fonttools==4.60.1 # via matplotlib @@ -55,6 +55,8 @@ idna==3.11 # via # email-validator # requests +importlib-metadata==8.7.0 + # via dask iniconfig==2.3.0 # via pytest kiwisolver==1.4.10rc0 @@ -112,13 +114,13 @@ pooch==1.8.2 # via # -r nightly.in # tof -pydantic==2.12.4 +pydantic==2.12.5 # via scippneutron pydantic-core==2.41.5 # via pydantic pygments==2.19.2 # via pytest -pyparsing==3.3.0a1 +pyparsing==3.3.0b1 # via matplotlib pytest==9.0.1 # via -r nightly.in @@ -174,6 +176,8 @@ tzdata==2025.2 # via pandas urllib3==2.5.0 # via requests +zipp==3.23.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From 5f5d5b212fab3a365aada8b290f2c8ef0aff0331 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 2 Dec 2025 13:52:18 +0100 Subject: [PATCH 295/403] Update data registry and remove deprecated mcstas version(2) file. --- packages/essnmx/docs/api-reference/index.md | 5 --- packages/essnmx/src/ess/nmx/__init__.py | 4 +- packages/essnmx/src/ess/nmx/data/__init__.py | 46 ++++++++------------ packages/essnmx/tests/conftest.py | 9 ---- packages/essnmx/tests/loader_test.py | 46 ++------------------ packages/essnmx/tests/mcstas_io_test.py | 12 ++--- packages/essnmx/tests/workflow_test.py | 12 ++--- 7 files changed, 30 insertions(+), 104 deletions(-) diff --git a/packages/essnmx/docs/api-reference/index.md b/packages/essnmx/docs/api-reference/index.md index cb30925c..4c32bbab 100644 --- a/packages/essnmx/docs/api-reference/index.md +++ b/packages/essnmx/docs/api-reference/index.md @@ -10,9 +10,6 @@ :template: class-template.rst :recursive: - NMXRawEventCountsDataGroup - NMXReducedDataGroup - ``` ## Top-level functions @@ -22,8 +19,6 @@ :toctree: ../generated/functions :recursive: - small_mcstas_3_sample - ``` ## Submodules diff --git a/packages/essnmx/src/ess/nmx/__init__.py b/packages/essnmx/src/ess/nmx/__init__.py index 57586b2e..7abf7bd0 100644 --- a/packages/essnmx/src/ess/nmx/__init__.py +++ b/packages/essnmx/src/ess/nmx/__init__.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) -# ruff: noqa: E402, F401, I +# ruff: noqa: E402, I import importlib.metadata @@ -11,7 +11,6 @@ del importlib -from .data import small_mcstas_3_sample from .reduction import NMXReducedDataGroup from .types import MaximumCounts, NMXRawEventCountsDataGroup @@ -23,5 +22,4 @@ "NMXRawEventCountsDataGroup", "NMXReducedDataGroup", "default_parameters", - "small_mcstas_3_sample", ] diff --git a/packages/essnmx/src/ess/nmx/data/__init__.py b/packages/essnmx/src/ess/nmx/data/__init__.py index ff594429..81546da8 100644 --- a/packages/essnmx/src/ess/nmx/data/__init__.py +++ b/packages/essnmx/src/ess/nmx/data/__init__.py @@ -4,50 +4,42 @@ from ess.reduce.data import Entry, make_registry -_version = "0" +_version = "1" -__all__ = ["get_path", "small_mcstas_2_sample", "small_mcstas_3_sample"] +__all__ = [ + "get_path", + "get_small_mtz_samples", + "get_small_nmx_nexus", + "get_small_random_mtz_samples", + "small_mcstas_sample", +] _registry = make_registry( "ess/nmx", - version="0", + version=_version, files={ - "small_mcstas_2_sample.h5": "md5:c3affe636397f8c9eea1d9c10a2bf487", - "small_mcstas_3_sample.h5": "md5:2afaac205d13ee857ee5364e3f1957a7", + "small_mcstas_sample.h5": "md5:2afaac205d13ee857ee5364e3f1957a7", "mtz_samples.tar.gz": Entry( alg="md5", chk="bed1eaf604bbe8725c1f6a20ca79fcc0", extractor="untar" ), "mtz_random_samples.tar.gz": Entry( alg="md5", chk="c8259ae2e605560ab88959e7109613b6", extractor="untar" ), - "small_nmx_nexus.hdf": "md5:42cffb85e4ce7c1aaa5f7e81469b865e", + "small_nmx_nexus.hdf.zip": Entry( + alg="md5", chk="96877cddc9f6392c96890069657710ca", extractor="unzip" + ), }, ) -def small_mcstas_2_sample() -> pathlib.Path: - """McStas 2 file containing small number of events.""" - import warnings - - warnings.warn( - DeprecationWarning( - "``essnmx`` will not support loading files " - "made by McStas with version less than 3 from ``25.0.0``. " - "Use ``small_mcstas_3_sample`` instead." - ), - stacklevel=2, - ) - - return get_path("small_mcstas_2_sample.h5") - - -def small_mcstas_3_sample() -> pathlib.Path: - """McStas 3 file that contains only ``bank0(1-3)`` in the ``data`` group. +def small_mcstas_sample() -> pathlib.Path: + """McStas file that contains only ``bank0(1-3)`` in the ``data`` group. - Real McStas 3 file should contain more dataset under ``data`` group. + Real McStas file should contain more dataset under ``data`` group. + McStas version >=3. """ - return get_path("small_mcstas_3_sample.h5") + return get_path("small_mcstas_sample.h5") def get_path(name: str) -> pathlib.Path: @@ -82,4 +74,4 @@ def get_small_random_mtz_samples() -> list[pathlib.Path]: def get_small_nmx_nexus() -> pathlib.Path: """Return the path to a small NMX NeXus file.""" - return get_path("small_nmx_nexus.hdf") + return get_path("small_nmx_nexus.hdf.zip") diff --git a/packages/essnmx/tests/conftest.py b/packages/essnmx/tests/conftest.py index 33d8e057..8710000b 100644 --- a/packages/essnmx/tests/conftest.py +++ b/packages/essnmx/tests/conftest.py @@ -2,12 +2,3 @@ # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) # These fixtures cannot be found by pytest, # if they are not defined in `conftest.py` under `tests` directory. -from contextlib import AbstractContextManager -from functools import partial - -import pytest - - -@pytest.fixture -def mcstas_2_deprecation_warning_context() -> partial[AbstractContextManager]: - return partial(pytest.warns, DeprecationWarning, match="McStas") diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 622652d0..7f38dd6d 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -11,7 +11,7 @@ from scipp.testing import assert_allclose, assert_identical from ess.nmx import default_parameters -from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample +from ess.nmx.data import small_mcstas_sample from ess.nmx.mcstas.load import bank_names_to_detector_names, load_crystal_rotation from ess.nmx.mcstas.load import providers as loader_providers from ess.nmx.types import ( @@ -51,39 +51,6 @@ def check_nmxdata_properties( assert_identical(dg['slow_axis'], slow_axis) -@pytest.mark.parametrize( - ('detector_index', 'fast_axis', 'slow_axis'), - [ - # Expected values are provided by the IDS - # based on the simulation settings of the sample file. - (0, (1.0, 0.0, -0.01), (0.0, 1.0, 0.0)), - (1, (-0.01, 0.0, -1.0), (0.0, 1.0, 0.0)), - (2, (0.01, 0.0, 1.0), (0.0, 1.0, 0.0)), - ], -) -def test_file_reader_mcstas2( - detector_index, fast_axis, slow_axis, mcstas_2_deprecation_warning_context -) -> None: - with mcstas_2_deprecation_warning_context(): - file_path = small_mcstas_2_sample() - - fast_axis = sc.vector(fast_axis) - slow_axis = sc.vector(slow_axis) - - pl = sl.Pipeline( - loader_providers, - params={ - FilePath: file_path, - DetectorIndex: detector_index, - **default_parameters, - }, - ) - dg = pl.compute(NMXRawEventCountsDataGroup) - - check_scalar_properties_mcstas_2(dg) - check_nmxdata_properties(dg, fast_axis, slow_axis) - - def check_scalar_properties_mcstas_3(dg: NMXRawEventCountsDataGroup): """Test helper for NMXData loaded from McStas 3. @@ -108,7 +75,7 @@ def check_scalar_properties_mcstas_3(dg: NMXRawEventCountsDataGroup): ], ) def test_file_reader_mcstas3(detector_index, fast_axis, slow_axis) -> None: - file_path = small_mcstas_3_sample() + file_path = small_mcstas_sample() pl = sl.Pipeline( loader_providers, @@ -130,20 +97,15 @@ def test_file_reader_mcstas3(detector_index, fast_axis, slow_axis) -> None: check_nmxdata_properties(dg, sc.vector(fast_axis), sc.vector(slow_axis)) -@pytest.fixture(params=[small_mcstas_2_sample, small_mcstas_3_sample]) +@pytest.fixture(params=[small_mcstas_sample]) def tmp_mcstas_file( tmp_path: pathlib.Path, request: pytest.FixtureRequest, - mcstas_2_deprecation_warning_context, ) -> Generator[pathlib.Path, None, None]: import os import shutil - if request.param == small_mcstas_2_sample: - with mcstas_2_deprecation_warning_context(): - original_file_path = request.param() - else: - original_file_path = request.param() + original_file_path = request.param() tmp_file = tmp_path / pathlib.Path('file.h5') shutil.copy(original_file_path, tmp_file) diff --git a/packages/essnmx/tests/mcstas_io_test.py b/packages/essnmx/tests/mcstas_io_test.py index bf193c5e..3999ff8c 100644 --- a/packages/essnmx/tests/mcstas_io_test.py +++ b/packages/essnmx/tests/mcstas_io_test.py @@ -3,18 +3,12 @@ import pytest import scipp as sc -from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample +from ess.nmx.data import small_mcstas_sample from ess.nmx.mcstas.load import load_raw_event_data, raw_event_data_chunk_generator -@pytest.fixture(params=[small_mcstas_2_sample, small_mcstas_3_sample]) -def mcstas_file_path( - request: pytest.FixtureRequest, mcstas_2_deprecation_warning_context -) -> str: - if request.param == small_mcstas_2_sample: - with mcstas_2_deprecation_warning_context(): - return request.param() - +@pytest.fixture(params=[small_mcstas_sample]) +def mcstas_file_path(request: pytest.FixtureRequest) -> str: return request.param() diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index f0c5c8a4..5c0ff07c 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -6,7 +6,7 @@ import scipp as sc from ess.nmx import default_parameters -from ess.nmx.data import small_mcstas_2_sample, small_mcstas_3_sample +from ess.nmx.data import small_mcstas_sample from ess.nmx.mcstas.load import providers as load_providers from ess.nmx.reduction import ( NMXReducedDataGroup, @@ -25,14 +25,8 @@ ) -@pytest.fixture(params=[small_mcstas_2_sample, small_mcstas_3_sample]) -def mcstas_file_path( - request: pytest.FixtureRequest, mcstas_2_deprecation_warning_context -) -> str: - if request.param == small_mcstas_2_sample: - with mcstas_2_deprecation_warning_context(): - return request.param() - +@pytest.fixture(params=[small_mcstas_sample]) +def mcstas_file_path(request: pytest.FixtureRequest) -> str: return request.param() From b5541ca9ef26c573d00a8a8ea9f010f80ab0206c Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 2 Dec 2025 13:55:32 +0100 Subject: [PATCH 296/403] Update the mcstas file fetching function name to match the other function names. --- packages/essnmx/src/ess/nmx/data/__init__.py | 4 ++-- packages/essnmx/tests/loader_test.py | 6 +++--- packages/essnmx/tests/mcstas_io_test.py | 4 ++-- packages/essnmx/tests/workflow_test.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/data/__init__.py b/packages/essnmx/src/ess/nmx/data/__init__.py index 81546da8..ece282a9 100644 --- a/packages/essnmx/src/ess/nmx/data/__init__.py +++ b/packages/essnmx/src/ess/nmx/data/__init__.py @@ -8,10 +8,10 @@ __all__ = [ "get_path", + "get_small_mcstas", "get_small_mtz_samples", "get_small_nmx_nexus", "get_small_random_mtz_samples", - "small_mcstas_sample", ] @@ -33,7 +33,7 @@ ) -def small_mcstas_sample() -> pathlib.Path: +def get_small_mcstas() -> pathlib.Path: """McStas file that contains only ``bank0(1-3)`` in the ``data`` group. Real McStas file should contain more dataset under ``data`` group. diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 7f38dd6d..56e48681 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -11,7 +11,7 @@ from scipp.testing import assert_allclose, assert_identical from ess.nmx import default_parameters -from ess.nmx.data import small_mcstas_sample +from ess.nmx.data import get_small_mcstas from ess.nmx.mcstas.load import bank_names_to_detector_names, load_crystal_rotation from ess.nmx.mcstas.load import providers as loader_providers from ess.nmx.types import ( @@ -75,7 +75,7 @@ def check_scalar_properties_mcstas_3(dg: NMXRawEventCountsDataGroup): ], ) def test_file_reader_mcstas3(detector_index, fast_axis, slow_axis) -> None: - file_path = small_mcstas_sample() + file_path = get_small_mcstas() pl = sl.Pipeline( loader_providers, @@ -97,7 +97,7 @@ def test_file_reader_mcstas3(detector_index, fast_axis, slow_axis) -> None: check_nmxdata_properties(dg, sc.vector(fast_axis), sc.vector(slow_axis)) -@pytest.fixture(params=[small_mcstas_sample]) +@pytest.fixture(params=[get_small_mcstas]) def tmp_mcstas_file( tmp_path: pathlib.Path, request: pytest.FixtureRequest, diff --git a/packages/essnmx/tests/mcstas_io_test.py b/packages/essnmx/tests/mcstas_io_test.py index 3999ff8c..4c927cfc 100644 --- a/packages/essnmx/tests/mcstas_io_test.py +++ b/packages/essnmx/tests/mcstas_io_test.py @@ -3,11 +3,11 @@ import pytest import scipp as sc -from ess.nmx.data import small_mcstas_sample +from ess.nmx.data import get_small_mcstas from ess.nmx.mcstas.load import load_raw_event_data, raw_event_data_chunk_generator -@pytest.fixture(params=[small_mcstas_sample]) +@pytest.fixture(params=[get_small_mcstas]) def mcstas_file_path(request: pytest.FixtureRequest) -> str: return request.param() diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index 5c0ff07c..8e3912b7 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -6,7 +6,7 @@ import scipp as sc from ess.nmx import default_parameters -from ess.nmx.data import small_mcstas_sample +from ess.nmx.data import get_small_mcstas from ess.nmx.mcstas.load import providers as load_providers from ess.nmx.reduction import ( NMXReducedDataGroup, @@ -25,7 +25,7 @@ ) -@pytest.fixture(params=[small_mcstas_sample]) +@pytest.fixture(params=[get_small_mcstas]) def mcstas_file_path(request: pytest.FixtureRequest) -> str: return request.param() From 0a003f535b71c33ed663004282edbc98d477da9e Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 2 Dec 2025 14:05:51 +0100 Subject: [PATCH 297/403] Fix relative path. --- packages/essnmx/docs/about/data_workflow_overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/docs/about/data_workflow_overview.md b/packages/essnmx/docs/about/data_workflow_overview.md index 78af4d56..159574d6 100644 --- a/packages/essnmx/docs/about/data_workflow_overview.md +++ b/packages/essnmx/docs/about/data_workflow_overview.md @@ -21,7 +21,7 @@ Then the single events get binned into pixels and then histogramed in the TOF di This result can be exported to an HDF5 file along with additional metadata and instrument coordinates (pixel IDs). -See [workflow example](../examples/workflow) for more details. +See [workflow example](../user-guide/workflow) for more details. ### Spot finding and integration (DIALS) For the next five steps of the data reduction from spot finding to spot integration, From a20597cc3a17d8b1fd804bf4f15fded3f5dd28cf Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 2 Dec 2025 14:06:49 +0100 Subject: [PATCH 298/403] Fix type hints. --- packages/essnmx/tests/mcstas_io_test.py | 4 +++- packages/essnmx/tests/workflow_test.py | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/essnmx/tests/mcstas_io_test.py b/packages/essnmx/tests/mcstas_io_test.py index 4c927cfc..39eb8426 100644 --- a/packages/essnmx/tests/mcstas_io_test.py +++ b/packages/essnmx/tests/mcstas_io_test.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) +import pathlib + import pytest import scipp as sc @@ -8,7 +10,7 @@ @pytest.fixture(params=[get_small_mcstas]) -def mcstas_file_path(request: pytest.FixtureRequest) -> str: +def mcstas_file_path(request: pytest.FixtureRequest) -> pathlib.Path: return request.param() diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/workflow_test.py index 8e3912b7..cbd2964b 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/workflow_test.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2023 Scipp contributors (https://github.com/scipp) +import pathlib + import pandas as pd import pytest import sciline as sl @@ -26,12 +28,12 @@ @pytest.fixture(params=[get_small_mcstas]) -def mcstas_file_path(request: pytest.FixtureRequest) -> str: +def mcstas_file_path(request: pytest.FixtureRequest) -> pathlib.Path: return request.param() @pytest.fixture -def mcstas_workflow(mcstas_file_path: str) -> sl.Pipeline: +def mcstas_workflow(mcstas_file_path: pathlib.Path) -> sl.Pipeline: return sl.Pipeline( [ *load_providers, @@ -59,7 +61,9 @@ def multi_bank_mcstas_workflow(mcstas_workflow: sl.Pipeline) -> sl.Pipeline: return pl -def test_pipeline_builder(mcstas_workflow: sl.Pipeline, mcstas_file_path: str) -> None: +def test_pipeline_builder( + mcstas_workflow: sl.Pipeline, mcstas_file_path: pathlib.Path +) -> None: assert mcstas_workflow.get(FilePath).compute() == mcstas_file_path From 0d7ea8bbd9b9fc9c3688b37b799669a10486929e Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 2 Dec 2025 14:11:59 +0100 Subject: [PATCH 299/403] Fix userguide according to the updated data registry. --- packages/essnmx/docs/user-guide/workflow.ipynb | 8 ++++---- packages/essnmx/docs/user-guide/workflow_chunk.ipynb | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/essnmx/docs/user-guide/workflow.ipynb b/packages/essnmx/docs/user-guide/workflow.ipynb index 833c1fdb..5c60e6d0 100644 --- a/packages/essnmx/docs/user-guide/workflow.ipynb +++ b/packages/essnmx/docs/user-guide/workflow.ipynb @@ -20,7 +20,7 @@ "outputs": [], "source": [ "from ess.nmx.mcstas import McStasWorkflow\n", - "from ess.nmx.data import small_mcstas_3_sample\n", + "from ess.nmx.data import get_small_mcstas\n", "\n", "from ess.nmx.types import *\n", "from ess.nmx.reduction import merge_panels\n", @@ -28,7 +28,7 @@ "\n", "wf = McStasWorkflow()\n", "# Replace with the path to your own file\n", - "wf[FilePath] = small_mcstas_3_sample()\n", + "wf[FilePath] = get_small_mcstas()\n", "wf[MaximumCounts] = 10000\n", "wf[TimeBinSteps] = 50" ] @@ -170,7 +170,7 @@ ], "metadata": { "kernelspec": { - "display_name": "scipp", + "display_name": "nmx-dev-313", "language": "python", "name": "python3" }, @@ -184,7 +184,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.0" + "version": "3.13.5" } }, "nbformat": 4, diff --git a/packages/essnmx/docs/user-guide/workflow_chunk.ipynb b/packages/essnmx/docs/user-guide/workflow_chunk.ipynb index 0892b8bd..f0eb7f82 100644 --- a/packages/essnmx/docs/user-guide/workflow_chunk.ipynb +++ b/packages/essnmx/docs/user-guide/workflow_chunk.ipynb @@ -22,12 +22,12 @@ "outputs": [], "source": [ "from ess.nmx.mcstas import McStasWorkflow\n", - "from ess.nmx.data import small_mcstas_3_sample\n", + "from ess.nmx.data import get_small_mcstas\n", "from ess.nmx.types import *\n", "\n", "wf = McStasWorkflow()\n", "# Replace with the path to your own file\n", - "wf[FilePath] = small_mcstas_3_sample()\n", + "wf[FilePath] = get_small_mcstas()\n", "wf[MaximumCounts] = 10_000\n", "wf[TimeBinSteps] = 50\n", "wf.visualize(NMXReducedDataGroup, graph_attr={\"rankdir\": \"TD\"}, compact=True)" From 614e0e3ebfa8b292ee5ccab32fb97676bbe6f5db Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 2 Dec 2025 14:12:39 +0100 Subject: [PATCH 300/403] Remove unnecessary helper. --- packages/essnmx/tests/loader_test.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/loader_test.py index 56e48681..b112257e 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/loader_test.py @@ -30,19 +30,6 @@ ) -def check_scalar_properties_mcstas_2(dg: NMXRawEventCountsDataGroup): - """Test helper for NMXData loaded from McStas 2. - - Expected numbers are hard-coded based on the sample file. - """ - assert_identical(dg['crystal_rotation'], sc.vector([20, 0, 90], unit='deg')) - assert_identical(dg['sample_position'], sc.vector(value=[0, 0, 0], unit='m')) - assert_identical( - dg['source_position'], sc.vector(value=[-0.53123, 0.0, -157.405], unit='m') - ) - assert dg['sample_name'] == sc.scalar("sampleMantid") - - def check_nmxdata_properties( dg: NMXRawEventCountsDataGroup, fast_axis, slow_axis ) -> None: From 5574288506c35545c941ae36f920f033c5df138f Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 2 Dec 2025 15:19:29 +0100 Subject: [PATCH 301/403] Isolate all mcstas specific modules and helpers. --- packages/essnmx/docs/api-reference/index.md | 3 - .../essnmx/docs/user-guide/workflow.ipynb | 10 +-- .../docs/user-guide/workflow_chunk.ipynb | 11 ++- packages/essnmx/src/ess/nmx/__init__.py | 13 +--- packages/essnmx/src/ess/nmx/executables.py | 18 ++++- .../essnmx/src/ess/nmx/mcstas/__init__.py | 9 +-- .../essnmx/src/ess/nmx/mcstas/executables.py | 68 +++++++++++++--- packages/essnmx/src/ess/nmx/mcstas/load.py | 2 +- .../essnmx/src/ess/nmx/{ => mcstas}/nexus.py | 3 +- .../src/ess/nmx/{ => mcstas}/reduction.py | 0 .../src/ess/nmx/{ => mcstas}/streaming.py | 2 +- packages/essnmx/src/ess/nmx/mcstas/types.py | 78 +++++++++++++++++++ packages/essnmx/src/ess/nmx/mcstas/xml.py | 2 +- packages/essnmx/src/ess/nmx/types.py | 78 ------------------- .../tests/{ => mcstas}/exporter_test.py | 4 +- .../essnmx/tests/{ => mcstas}/loader_test.py | 28 ++----- .../mcstas_description_examples.py | 0 .../tests/{ => mcstas}/mcstas_io_test.py | 0 .../tests/{ => mcstas}/workflow_test.py | 35 +++------ 19 files changed, 190 insertions(+), 174 deletions(-) rename packages/essnmx/src/ess/nmx/{ => mcstas}/nexus.py (99%) rename packages/essnmx/src/ess/nmx/{ => mcstas}/reduction.py (100%) rename packages/essnmx/src/ess/nmx/{ => mcstas}/streaming.py (97%) create mode 100644 packages/essnmx/src/ess/nmx/mcstas/types.py rename packages/essnmx/tests/{ => mcstas}/exporter_test.py (97%) rename packages/essnmx/tests/{ => mcstas}/loader_test.py (89%) rename packages/essnmx/tests/{ => mcstas}/mcstas_description_examples.py (100%) rename packages/essnmx/tests/{ => mcstas}/mcstas_io_test.py (100%) rename packages/essnmx/tests/{ => mcstas}/workflow_test.py (77%) diff --git a/packages/essnmx/docs/api-reference/index.md b/packages/essnmx/docs/api-reference/index.md index 4c32bbab..5da7f253 100644 --- a/packages/essnmx/docs/api-reference/index.md +++ b/packages/essnmx/docs/api-reference/index.md @@ -31,9 +31,6 @@ data mcstas - reduction - nexus - streaming types mtz_io scaling diff --git a/packages/essnmx/docs/user-guide/workflow.ipynb b/packages/essnmx/docs/user-guide/workflow.ipynb index 5c60e6d0..aec43981 100644 --- a/packages/essnmx/docs/user-guide/workflow.ipynb +++ b/packages/essnmx/docs/user-guide/workflow.ipynb @@ -19,14 +19,14 @@ "metadata": {}, "outputs": [], "source": [ - "from ess.nmx.mcstas import McStasWorkflow\n", + "from ess.nmx.mcstas import NMXMcStasWorkflow\n", "from ess.nmx.data import get_small_mcstas\n", "\n", - "from ess.nmx.types import *\n", - "from ess.nmx.reduction import merge_panels\n", - "from ess.nmx.nexus import export_as_nexus\n", + "from ess.nmx.mcstas.types import *\n", + "from ess.nmx.mcstas.reduction import merge_panels\n", + "from ess.nmx.mcstas.nexus import export_as_nexus\n", "\n", - "wf = McStasWorkflow()\n", + "wf = NMXMcStasWorkflow()\n", "# Replace with the path to your own file\n", "wf[FilePath] = get_small_mcstas()\n", "wf[MaximumCounts] = 10000\n", diff --git a/packages/essnmx/docs/user-guide/workflow_chunk.ipynb b/packages/essnmx/docs/user-guide/workflow_chunk.ipynb index f0eb7f82..04e021bc 100644 --- a/packages/essnmx/docs/user-guide/workflow_chunk.ipynb +++ b/packages/essnmx/docs/user-guide/workflow_chunk.ipynb @@ -21,11 +21,11 @@ "metadata": {}, "outputs": [], "source": [ - "from ess.nmx.mcstas import McStasWorkflow\n", + "from ess.nmx.mcstas import NMXMcStasWorkflow\n", "from ess.nmx.data import get_small_mcstas\n", - "from ess.nmx.types import *\n", + "from ess.nmx.mcstas.types import *\n", "\n", - "wf = McStasWorkflow()\n", + "wf = NMXMcStasWorkflow()\n", "# Replace with the path to your own file\n", "wf[FilePath] = get_small_mcstas()\n", "wf[MaximumCounts] = 10_000\n", @@ -92,12 +92,11 @@ "metadata": {}, "outputs": [], "source": [ - "from ess.nmx.types import DetectorName\n", "from ess.nmx.mcstas.load import (\n", " raw_event_data_chunk_generator,\n", " mcstas_weight_to_probability_scalefactor,\n", ")\n", - "from ess.nmx.streaming import calculate_number_of_chunks\n", + "from ess.nmx.mcstas.streaming import calculate_number_of_chunks\n", "from ipywidgets import IntProgress\n", "\n", "CHUNK_SIZE = 10 # Number of event rows to process at once\n", @@ -194,7 +193,7 @@ "metadata": {}, "outputs": [], "source": [ - "from ess.nmx.nexus import NXLauetofWriter\n", + "from ess.nmx.mcstas.nexus import NXLauetofWriter\n", "\n", "\n", "def temp_generator(file_path, detector_name):\n", diff --git a/packages/essnmx/src/ess/nmx/__init__.py b/packages/essnmx/src/ess/nmx/__init__.py index 7abf7bd0..b3941868 100644 --- a/packages/essnmx/src/ess/nmx/__init__.py +++ b/packages/essnmx/src/ess/nmx/__init__.py @@ -11,15 +11,6 @@ del importlib -from .reduction import NMXReducedDataGroup -from .types import MaximumCounts, NMXRawEventCountsDataGroup +from .mcstas import NMXMcStasWorkflow -default_parameters = {MaximumCounts: 10000} - -del MaximumCounts - -__all__ = [ - "NMXRawEventCountsDataGroup", - "NMXReducedDataGroup", - "default_parameters", -] +__all__ = ["NMXMcStasWorkflow"] diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 34562137..f4da09f2 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -14,14 +14,26 @@ build_logger, collect_matching_input_files, ) -from .nexus import ( + +# Temporarily keeping them until we migrate GenericWorkflow here +from .mcstas.nexus import ( _compute_positions, _export_detector_metadata_as_nxlauetof, _export_reduced_data_as_nxlauetof, _export_static_metadata_as_nxlauetof, ) -from .streaming import _validate_chunk_size -from .types import Compression, NMXDetectorMetadata, NMXExperimentMetadata + +# Temporarily keeping them until we migrate GenericWorkflow here +from .mcstas.types import NMXDetectorMetadata, NMXExperimentMetadata +from .types import Compression + + +def _validate_chunk_size(chunk_size: int) -> None: + """Validate the chunk size.""" + if not isinstance(chunk_size, int): + raise TypeError("Chunk size must be an integer.") + if chunk_size < -1: + raise ValueError("Invalid chunk size. It should be -1(for all) or > 0.") def _retrieve_source_position(file: snx.File) -> sc.Variable: diff --git a/packages/essnmx/src/ess/nmx/mcstas/__init__.py b/packages/essnmx/src/ess/nmx/mcstas/__init__.py index 3026dc20..11700114 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/__init__.py +++ b/packages/essnmx/src/ess/nmx/mcstas/__init__.py @@ -1,14 +1,15 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2024 Scipp contributors (https://github.com/scipp) -from ..types import MaximumCounts +from .types import MaximumCounts default_parameters = {MaximumCounts: 10000} -def McStasWorkflow(): +def NMXMcStasWorkflow(): import sciline as sl - from ess.nmx.reduction import ( + from .load import providers as loader_providers + from .reduction import ( calculate_maximum_toa, calculate_minimum_toa, format_nmx_reduced_data, @@ -16,8 +17,6 @@ def McStasWorkflow(): raw_event_probability_to_counts, reduce_raw_event_probability, ) - - from .load import providers as loader_providers from .xml import read_mcstas_geometry_xml return sl.Pipeline( diff --git a/packages/essnmx/src/ess/nmx/mcstas/executables.py b/packages/essnmx/src/ess/nmx/mcstas/executables.py index 14ec7f68..690f6b5f 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/executables.py +++ b/packages/essnmx/src/ess/nmx/mcstas/executables.py @@ -16,14 +16,19 @@ StreamProcessor, ) -from ..nexus import ( +from ..types import Compression +from . import NMXMcStasWorkflow +from .load import ( + mcstas_weight_to_probability_scalefactor, + raw_event_data_chunk_generator, +) +from .nexus import ( _export_detector_metadata_as_nxlauetof, _export_reduced_data_as_nxlauetof, _export_static_metadata_as_nxlauetof, ) -from ..streaming import calculate_number_of_chunks -from ..types import ( - Compression, +from .streaming import calculate_number_of_chunks +from .types import ( DetectorIndex, DetectorName, FilePath, @@ -41,11 +46,6 @@ RawEventProbability, TimeBinSteps, ) -from . import McStasWorkflow -from .load import ( - mcstas_weight_to_probability_scalefactor, - raw_event_data_chunk_generator, -) from .xml import McStasInstrument @@ -147,7 +147,7 @@ def reduction( logger: logging.Logger | None = None, toa_min_max_prob: tuple[float] | None = None, ) -> None: - wf = wf.copy() if wf is not None else McStasWorkflow() + wf = wf.copy() if wf is not None else NMXMcStasWorkflow() wf[FilePath] = input_file # Set static info wf[McStasInstrument] = wf.compute(McStasInstrument) @@ -283,8 +283,52 @@ def _add_mcstas_args(parser: argparse.ArgumentParser) -> None: ) +def build_reduction_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Command line arguments for the NMX reduction. " + "It assumes 14 Hz pulse speed." + ) + input_arg_group = parser.add_argument_group("Input Options") + input_arg_group.add_argument( + "--input_file", type=str, help="Path to the input file", required=True + ) + input_arg_group.add_argument( + "--nbins", + type=int, + default=50, + help="Number of TOF bins", + ) + input_arg_group.add_argument( + "--detector_ids", + type=int, + nargs="+", + default=[0, 1, 2], + help="Detector indices to process", + ) + + output_arg_group = parser.add_argument_group("Output Options") + output_arg_group.add_argument( + "--output_file", + type=str, + default="scipp_output.h5", + help="Path to the output file", + ) + output_arg_group.add_argument( + "--compression", + type=str, + default=Compression.BITSHUFFLE_LZ4.name, + choices=[compression_key.name for compression_key in Compression], + help="Compress option of reduced output file. Default: BITSHUFFLE_LZ4", + ) + output_arg_group.add_argument( + "--verbose", "-v", action="store_true", help="Increase output verbosity" + ) + + return parser + + def main() -> None: - from .._executable_helper import build_logger, build_reduction_arg_parser + from .._executable_helper import build_logger parser = build_reduction_arg_parser() _add_mcstas_args(parser) @@ -295,7 +339,7 @@ def main() -> None: logger = build_logger(args) - wf = McStasWorkflow() + wf = NMXMcStasWorkflow() reduction( input_file=input_file, output_file=output_file, diff --git a/packages/essnmx/src/ess/nmx/mcstas/load.py b/packages/essnmx/src/ess/nmx/mcstas/load.py index a3158407..c3830d0b 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/load.py +++ b/packages/essnmx/src/ess/nmx/mcstas/load.py @@ -6,7 +6,7 @@ import scipp as sc import scippnexus as snx -from ..types import ( +from .types import ( CrystalRotation, DetectorBankPrefix, DetectorIndex, diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/mcstas/nexus.py similarity index 99% rename from packages/essnmx/src/ess/nmx/nexus.py rename to packages/essnmx/src/ess/nmx/mcstas/nexus.py index 64f31b36..404772b3 100644 --- a/packages/essnmx/src/ess/nmx/nexus.py +++ b/packages/essnmx/src/ess/nmx/mcstas/nexus.py @@ -256,7 +256,8 @@ def export_as_nexus( """ warnings.warn( DeprecationWarning( - "Exporting to custom NeXus format will be deprecated in the near future." + "Exporting to custom NeXus format will be deprecated in the near future " + ">=26.12.0. " "Please use ``export_as_nxlauetof`` instead." ), stacklevel=2, diff --git a/packages/essnmx/src/ess/nmx/reduction.py b/packages/essnmx/src/ess/nmx/mcstas/reduction.py similarity index 100% rename from packages/essnmx/src/ess/nmx/reduction.py rename to packages/essnmx/src/ess/nmx/mcstas/reduction.py diff --git a/packages/essnmx/src/ess/nmx/streaming.py b/packages/essnmx/src/ess/nmx/mcstas/streaming.py similarity index 97% rename from packages/essnmx/src/ess/nmx/streaming.py rename to packages/essnmx/src/ess/nmx/mcstas/streaming.py index 153a01ff..2ea46257 100644 --- a/packages/essnmx/src/ess/nmx/streaming.py +++ b/packages/essnmx/src/ess/nmx/mcstas/streaming.py @@ -7,7 +7,7 @@ from ess.reduce.streaming import Accumulator -from .mcstas.load import _validate_chunk_size, load_event_data_bank_name +from .load import _validate_chunk_size, load_event_data_bank_name from .types import DetectorBankPrefix, DetectorName, FilePath diff --git a/packages/essnmx/src/ess/nmx/mcstas/types.py b/packages/essnmx/src/ess/nmx/mcstas/types.py new file mode 100644 index 00000000..0d629021 --- /dev/null +++ b/packages/essnmx/src/ess/nmx/mcstas/types.py @@ -0,0 +1,78 @@ +from dataclasses import dataclass +from typing import Any, NewType + +import scipp as sc + +FilePath = NewType("FilePath", str) +"""File name of a file containing the results of a McStas run""" + +DetectorIndex = NewType("DetectorIndex", int | sc.Variable | sc.DataArray) +"""Index of the detector to load. Index ordered by the id:s of the pixels""" + +DetectorName = NewType("DetectorName", str) +"""Name of the detector to load""" + +DetectorBankPrefix = NewType("DetectorBankPrefix", str) +"""Prefix identifying the event data array containing +the events from the selected detector""" + +MaximumCounts = NewType("MaximumCounts", int) +"""Maximum number of counts after scaling the event counts""" + +MaximumProbability = NewType("MaximumProbability", sc.Variable) +"""Maximum probability to scale the McStas event counts""" + +McStasWeight2CountScaleFactor = NewType("McStasWeight2CountScaleFactor", sc.Variable) +"""Scale factor to convert McStas weights to counts""" + +NMXExperimentMetadata = NewType("NMXExperimentMetadata", sc.DataGroup) +"""Metadata of the experiment""" + +NMXDetectorMetadata = NewType("NMXDetectorMetadata", sc.DataGroup) +"""Metadata of the detector""" + +RawEventProbability = NewType("RawEventProbability", sc.DataArray) +"""DataArray containing the event probabilities read from the McStas file, +has coordinates 'id' and 't' """ + +NMXRawEventCountsDataGroup = NewType("NMXRawEventCountsDataGroup", sc.DataGroup) +"""DataGroup containing the RawEventData, experiment metadata and detector metadata""" + +ProtonCharge = NewType("ProtonCharge", sc.Variable) +"""The proton charge signal""" + +CrystalRotation = NewType("CrystalRotation", sc.Variable) +"""Rotation of the crystal""" + +DetectorGeometry = NewType("DetectorGeometry", Any) +"""Description of the geometry of the detector banks""" + +TimeBinSteps = NewType("TimeBinSteps", int) +"""Number of bins in the binning of the time coordinate""" + +PixelIds = NewType("PixelIds", sc.Variable) +"""The pixel ids of the detector""" + +NMXReducedProbability = NewType("NMXReducedProbability", sc.DataArray) +"""Histogram of time-of-arrival and pixel-id.""" + +NMXReducedCounts = NewType("NMXReducedCounts", sc.DataArray) +"""Histogram of time-of-arrival and pixel-id.""" + +NMXReducedDataGroup = NewType("NMXReducedDataGroup", sc.DataGroup) +"""Datagroup containing Histogram(id, t), experiment metadata and detector metadata""" + +MinimumTimeOfArrival = NewType("MinimumTimeOfArrival", sc.Variable) +"""Minimum time of arrival of the raw data""" + +MaximumTimeOfArrival = NewType("MaximumTimeOfArrival", sc.Variable) +"""Maximum time of arrival of the raw data""" + + +@dataclass +class NMXRawDataMetadata: + """Metadata of the raw data, i.e. maximum weight and min/max time of arrival""" + + max_probability: MaximumProbability + min_toa: MinimumTimeOfArrival + max_toa: MaximumTimeOfArrival diff --git a/packages/essnmx/src/ess/nmx/mcstas/xml.py b/packages/essnmx/src/ess/nmx/mcstas/xml.py index 5478f6a0..36ad0ab7 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/xml.py +++ b/packages/essnmx/src/ess/nmx/mcstas/xml.py @@ -11,7 +11,7 @@ from defusedxml.ElementTree import fromstring from ..rotation import axis_angle_to_quaternion, quaternion_to_matrix -from ..types import FilePath +from .types import FilePath T = TypeVar('T') diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index baa7c3b8..ad4d3de5 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -1,82 +1,4 @@ import enum -from dataclasses import dataclass -from typing import Any, NewType - -import scipp as sc - -FilePath = NewType("FilePath", str) -"""File name of a file containing the results of a McStas run""" - -DetectorIndex = NewType("DetectorIndex", int | sc.Variable | sc.DataArray) -"""Index of the detector to load. Index ordered by the id:s of the pixels""" - -DetectorName = NewType("DetectorName", str) -"""Name of the detector to load""" - -DetectorBankPrefix = NewType("DetectorBankPrefix", str) -"""Prefix identifying the event data array containing -the events from the selected detector""" - -MaximumCounts = NewType("MaximumCounts", int) -"""Maximum number of counts after scaling the event counts""" - -MaximumProbability = NewType("MaximumProbability", sc.Variable) -"""Maximum probability to scale the McStas event counts""" - -McStasWeight2CountScaleFactor = NewType("McStasWeight2CountScaleFactor", sc.Variable) -"""Scale factor to convert McStas weights to counts""" - -NMXExperimentMetadata = NewType("NMXExperimentMetadata", sc.DataGroup) -"""Metadata of the experiment""" - -NMXDetectorMetadata = NewType("NMXDetectorMetadata", sc.DataGroup) -"""Metadata of the detector""" - -RawEventProbability = NewType("RawEventProbability", sc.DataArray) -"""DataArray containing the event probabilities read from the McStas file, -has coordinates 'id' and 't' """ - -NMXRawEventCountsDataGroup = NewType("NMXRawEventCountsDataGroup", sc.DataGroup) -"""DataGroup containing the RawEventData, experiment metadata and detector metadata""" - -ProtonCharge = NewType("ProtonCharge", sc.Variable) -"""The proton charge signal""" - -CrystalRotation = NewType("CrystalRotation", sc.Variable) -"""Rotation of the crystal""" - -DetectorGeometry = NewType("DetectorGeometry", Any) -"""Description of the geometry of the detector banks""" - -TimeBinSteps = NewType("TimeBinSteps", int) -"""Number of bins in the binning of the time coordinate""" - -PixelIds = NewType("PixelIds", sc.Variable) -"""The pixel ids of the detector""" - -NMXReducedProbability = NewType("NMXReducedProbability", sc.DataArray) -"""Histogram of time-of-arrival and pixel-id.""" - -NMXReducedCounts = NewType("NMXReducedCounts", sc.DataArray) -"""Histogram of time-of-arrival and pixel-id.""" - -NMXReducedDataGroup = NewType("NMXReducedDataGroup", sc.DataGroup) -"""Datagroup containing Histogram(id, t), experiment metadata and detector metadata""" - -MinimumTimeOfArrival = NewType("MinimumTimeOfArrival", sc.Variable) -"""Minimum time of arrival of the raw data""" - -MaximumTimeOfArrival = NewType("MaximumTimeOfArrival", sc.Variable) -"""Maximum time of arrival of the raw data""" - - -@dataclass -class NMXRawDataMetadata: - """Metadata of the raw data, i.e. maximum weight and min/max time of arrival""" - - max_probability: MaximumProbability - min_toa: MinimumTimeOfArrival - max_toa: MaximumTimeOfArrival class Compression(enum.StrEnum): diff --git a/packages/essnmx/tests/exporter_test.py b/packages/essnmx/tests/mcstas/exporter_test.py similarity index 97% rename from packages/essnmx/tests/exporter_test.py rename to packages/essnmx/tests/mcstas/exporter_test.py index 379dbe62..1d30cacc 100644 --- a/packages/essnmx/tests/exporter_test.py +++ b/packages/essnmx/tests/mcstas/exporter_test.py @@ -6,8 +6,8 @@ import pytest import scipp as sc -from ess.nmx.nexus import export_as_nexus -from ess.nmx.reduction import NMXReducedDataGroup +from ess.nmx.mcstas.nexus import export_as_nexus +from ess.nmx.mcstas.types import NMXReducedDataGroup @pytest.fixture diff --git a/packages/essnmx/tests/loader_test.py b/packages/essnmx/tests/mcstas/loader_test.py similarity index 89% rename from packages/essnmx/tests/loader_test.py rename to packages/essnmx/tests/mcstas/loader_test.py index b112257e..c9d090d3 100644 --- a/packages/essnmx/tests/loader_test.py +++ b/packages/essnmx/tests/mcstas/loader_test.py @@ -5,16 +5,14 @@ from collections.abc import Generator import pytest -import sciline as sl import scipp as sc import scippnexus as snx from scipp.testing import assert_allclose, assert_identical -from ess.nmx import default_parameters +from ess.nmx import NMXMcStasWorkflow from ess.nmx.data import get_small_mcstas from ess.nmx.mcstas.load import bank_names_to_detector_names, load_crystal_rotation -from ess.nmx.mcstas.load import providers as loader_providers -from ess.nmx.types import ( +from ess.nmx.mcstas.types import ( DetectorBankPrefix, DetectorIndex, FilePath, @@ -64,14 +62,9 @@ def check_scalar_properties_mcstas_3(dg: NMXRawEventCountsDataGroup): def test_file_reader_mcstas3(detector_index, fast_axis, slow_axis) -> None: file_path = get_small_mcstas() - pl = sl.Pipeline( - loader_providers, - params={ - FilePath: file_path, - DetectorIndex: detector_index, - **default_parameters, - }, - ) + pl = NMXMcStasWorkflow() + pl[FilePath] = file_path + pl[DetectorIndex] = detector_index dg, bank = pl.compute((NMXRawEventCountsDataGroup, DetectorBankPrefix)).values() entry_path = f"entry1/data/{bank}_dat_list_p_x_y_n_id_t" @@ -112,14 +105,9 @@ def test_file_reader_mcstas_additional_fields(tmp_mcstas_file: pathlib.Path) -> del file[entry_path] file[new_entry_path] = dataset - pl = sl.Pipeline( - loader_providers, - params={ - FilePath: str(tmp_mcstas_file), - DetectorIndex: 0, - **default_parameters, - }, - ) + pl = NMXMcStasWorkflow() + pl[FilePath] = str(tmp_mcstas_file) + pl[DetectorIndex] = 0 dg = pl.compute(NMXRawEventCountsDataGroup) assert isinstance(dg, sc.DataGroup) diff --git a/packages/essnmx/tests/mcstas_description_examples.py b/packages/essnmx/tests/mcstas/mcstas_description_examples.py similarity index 100% rename from packages/essnmx/tests/mcstas_description_examples.py rename to packages/essnmx/tests/mcstas/mcstas_description_examples.py diff --git a/packages/essnmx/tests/mcstas_io_test.py b/packages/essnmx/tests/mcstas/mcstas_io_test.py similarity index 100% rename from packages/essnmx/tests/mcstas_io_test.py rename to packages/essnmx/tests/mcstas/mcstas_io_test.py diff --git a/packages/essnmx/tests/workflow_test.py b/packages/essnmx/tests/mcstas/workflow_test.py similarity index 77% rename from packages/essnmx/tests/workflow_test.py rename to packages/essnmx/tests/mcstas/workflow_test.py index cbd2964b..6f78bb76 100644 --- a/packages/essnmx/tests/workflow_test.py +++ b/packages/essnmx/tests/mcstas/workflow_test.py @@ -7,22 +7,15 @@ import sciline as sl import scipp as sc -from ess.nmx import default_parameters +from ess.nmx import NMXMcStasWorkflow from ess.nmx.data import get_small_mcstas -from ess.nmx.mcstas.load import providers as load_providers -from ess.nmx.reduction import ( - NMXReducedDataGroup, - format_nmx_reduced_data, - merge_panels, - proton_charge_from_event_counts, - raw_event_probability_to_counts, - reduce_raw_event_probability, -) -from ess.nmx.types import ( +from ess.nmx.mcstas.reduction import merge_panels +from ess.nmx.mcstas.types import ( DetectorIndex, FilePath, MaximumCounts, NMXRawEventCountsDataGroup, + NMXReducedDataGroup, TimeBinSteps, ) @@ -34,20 +27,10 @@ def mcstas_file_path(request: pytest.FixtureRequest) -> pathlib.Path: @pytest.fixture def mcstas_workflow(mcstas_file_path: pathlib.Path) -> sl.Pipeline: - return sl.Pipeline( - [ - *load_providers, - reduce_raw_event_probability, - proton_charge_from_event_counts, - raw_event_probability_to_counts, - format_nmx_reduced_data, - ], - params={ - FilePath: mcstas_file_path, - TimeBinSteps: 50, - **default_parameters, - }, - ) + wf = NMXMcStasWorkflow() + wf[FilePath] = mcstas_file_path + wf[TimeBinSteps] = 50 + return wf @pytest.fixture @@ -79,6 +62,8 @@ def test_pipeline_mcstas_reduction(multi_bank_mcstas_workflow: sl.Pipeline) -> N """Test if the loader graph is complete.""" from scipp.testing import assert_allclose, assert_identical + from ess.nmx.mcstas import default_parameters + nmx_reduced_data = multi_bank_mcstas_workflow.compute(NMXReducedDataGroup) assert nmx_reduced_data.shape == (3, (1280, 1280)[0] * (1280, 1280)[1], 50) # Panel, Pixels, Time bins From 29ee81d89b743525137b00dc33972d44c85f8854 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 2 Dec 2025 15:30:44 +0100 Subject: [PATCH 302/403] Rename workflow notebooks. --- packages/essnmx/docs/user-guide/index.md | 4 ++-- .../docs/user-guide/{workflow.ipynb => mcstas_workflow.ipynb} | 0 .../{workflow_chunk.ipynb => mcstas_workflow_chunk.ipynb} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename packages/essnmx/docs/user-guide/{workflow.ipynb => mcstas_workflow.ipynb} (100%) rename packages/essnmx/docs/user-guide/{workflow_chunk.ipynb => mcstas_workflow_chunk.ipynb} (100%) diff --git a/packages/essnmx/docs/user-guide/index.md b/packages/essnmx/docs/user-guide/index.md index 8f7ddfd0..550cfb7f 100644 --- a/packages/essnmx/docs/user-guide/index.md +++ b/packages/essnmx/docs/user-guide/index.md @@ -5,8 +5,8 @@ maxdepth: 1 --- -workflow -workflow_chunk +mcstas_workflow +mcstas_workflow_chunk scaling_workflow installation ``` diff --git a/packages/essnmx/docs/user-guide/workflow.ipynb b/packages/essnmx/docs/user-guide/mcstas_workflow.ipynb similarity index 100% rename from packages/essnmx/docs/user-guide/workflow.ipynb rename to packages/essnmx/docs/user-guide/mcstas_workflow.ipynb diff --git a/packages/essnmx/docs/user-guide/workflow_chunk.ipynb b/packages/essnmx/docs/user-guide/mcstas_workflow_chunk.ipynb similarity index 100% rename from packages/essnmx/docs/user-guide/workflow_chunk.ipynb rename to packages/essnmx/docs/user-guide/mcstas_workflow_chunk.ipynb From b53be1f24b6fc48e3dd222aeff389dbfee310dc7 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 2 Dec 2025 15:52:37 +0100 Subject: [PATCH 303/403] Update notebook title. --- packages/essnmx/docs/user-guide/mcstas_workflow_chunk.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/docs/user-guide/mcstas_workflow_chunk.ipynb b/packages/essnmx/docs/user-guide/mcstas_workflow_chunk.ipynb index 04e021bc..7257ed80 100644 --- a/packages/essnmx/docs/user-guide/mcstas_workflow_chunk.ipynb +++ b/packages/essnmx/docs/user-guide/mcstas_workflow_chunk.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Workflow - Chunk by Chunk\n", + "# McStas Workflow - Chunk by Chunk\n", "In this example, we will process McStas events chunk by chunk, panel by panel." ] }, From 19e1f510b2cf666bbc7189ad076a0a3a7a662cae Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 2 Dec 2025 15:57:58 +0100 Subject: [PATCH 304/403] Update notebook title. --- packages/essnmx/docs/user-guide/mcstas_workflow.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/docs/user-guide/mcstas_workflow.ipynb b/packages/essnmx/docs/user-guide/mcstas_workflow.ipynb index aec43981..4924246f 100644 --- a/packages/essnmx/docs/user-guide/mcstas_workflow.ipynb +++ b/packages/essnmx/docs/user-guide/mcstas_workflow.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Workflow\n", + "# McStas Workflow\n", "In this example, we will use McStas 3 simulation file.\n", "\n", "## Build Pipeline (Collect Parameters and Providers)\n", From 0f7831f92e5d389fc1cd3294861ecf2d2b616fbd Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 4 Dec 2025 11:08:16 +0100 Subject: [PATCH 305/403] Update configurations and reduction interface function signature. --- .../essnmx/src/ess/nmx/_executable_helper.py | 203 ++++--- packages/essnmx/src/ess/nmx/executables.py | 119 ++-- packages/essnmx/src/ess/nmx/nexus.py | 555 ++++++++++++++++++ packages/essnmx/tests/executable_test.py | 30 +- 4 files changed, 737 insertions(+), 170 deletions(-) create mode 100644 packages/essnmx/src/ess/nmx/nexus.py diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index 8ce2f653..d8ca016a 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -140,21 +140,6 @@ def add_args_from_pydantic_model( return parser -T = TypeVar('T', bound=BaseModel) - - -def from_args(cls: type[T], args: argparse.Namespace) -> T: - """Create an instance of the pydantic model from the argparse namespace. - - It ignores any extra arguments in the namespace that are not part of the model. - """ - kwargs = { - field_name: _retrieve_field_value(field_name, field_info, args) - for field_name, field_info in cls.model_fields.items() - } - return cls(**kwargs) - - class InputConfig(BaseModel): # Add title of the basemodel model_config = {"title": "Input Configuration"} @@ -201,42 +186,66 @@ class InputConfig(BaseModel): ) -class TOAUnit(enum.StrEnum): +class TimeBinUnit(enum.StrEnum): ms = 'ms' us = 'us' ns = 'ns' +class TimeBinCoordinate(enum.StrEnum): + event_time_offset = 'event_time_offset' + time_of_flight = 'time_of_flight' + + class WorkflowConfig(BaseModel): # Add title of the basemodel model_config = {"title": "Workflow Configuration"} + time_bin_coordinate: TimeBinCoordinate = Field( + title="Time Bin Coordinate", + description="Coordinate to bin the time data.", + default=TimeBinCoordinate.event_time_offset, + ) nbins: int = Field( - title="Number of TOF Bins", - description="Number of TOF bins", + title="Number of Time Bins", + description="Number of Time bins", default=50, ) - min_toa: int = Field( - title="Minimum Time of Arrival", - description="Minimum time of arrival (TOA) in [toa_unit].", - default=0, + min_time_bin: int | None = Field( + title="Minimum Time Bin", + description="Minimum time edge of [time_bin_coordinate] in [time_bin_unit].", + default=None, ) - max_toa: int = Field( - title="Maximum Time of Arrival", - description="Maximum time of arrival (TOA) in [toa_unit].", - default=int((1 / 14) * 1_000), + max_time_bin: int | None = Field( + title="Maximum Time Bin", + description="Maximum time edge of [time_bin_coordinate] in [time_bin_unit].", + default=None, ) - toa_unit: TOAUnit = Field( - title="Unit of TOA", - description="Unit of TOA.", - default=TOAUnit.ms, + time_bin_unit: TimeBinUnit = Field( + title="Unit of Time Bins", + description="Unit of time bins.", + default=TimeBinUnit.ms, ) - fast_axis: Literal['x', 'y'] | None = Field( - title="Fast Axis", - description="Specify the fast axis of the detector. " - "If None, it will be determined " - "automatically based on the pixel offsets.", + tof_lookup_table_file_path: str | None = Field( + title="TOF Lookup Table File Path", + description="Path to the TOF lookup table file. " + "If None, the lookup table will be computed on-the-fly.", default=None, ) + tof_simulation_min_wavelength: float = Field( + title="TOF Simulation Minimum Wavelength", + description="Minimum wavelength for TOF simulation in Angstrom.", + default=1.8, + ) + tof_simulation_max_wavelength: float = Field( + title="TOF Simulation Maximum Wavelength", + description="Maximum wavelength for TOF simulation in Angstrom.", + default=3.6, + ) + tof_simulation_seed: int = Field( + title="TOF Simulation Seed", + description="Random seed for TOF simulation.", + default=42, # No reason. + ) class OutputConfig(BaseModel): @@ -265,64 +274,82 @@ class ReductionConfig(BaseModel): """Container for all reduction configurations.""" inputs: InputConfig - workflow: WorkflowConfig - output: OutputConfig - - @classmethod - def build_argument_parser(cls) -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - description="Command line arguments for the ESS NMX reduction. " - "It assumes 14 Hz pulse speed." - ) - parser = add_args_from_pydantic_model(model_cls=InputConfig, parser=parser) - parser = add_args_from_pydantic_model(model_cls=WorkflowConfig, parser=parser) - parser = add_args_from_pydantic_model(model_cls=OutputConfig, parser=parser) - return parser - - @classmethod - def from_args(cls, args: argparse.Namespace) -> "ReductionConfig": - return cls( - inputs=from_args(InputConfig, args), - workflow=from_args(WorkflowConfig, args), - output=from_args(OutputConfig, args), - ) + workflow: WorkflowConfig = Field(default_factory=WorkflowConfig) + output: OutputConfig = Field(default_factory=OutputConfig) @property def _children(self) -> list[BaseModel]: return [self.inputs, self.workflow, self.output] - def to_command_arguments(self, one_line: bool = True) -> list[str] | str: - """Convert the config to a list of command line arguments. - - Parameters - ---------- - one_line: - If True, return a single string with all arguments joined by spaces. - If False, return a list of argument strings. - - """ - args = {} - for instance in self._children: - args.update(instance.model_dump(mode='python')) - args = {f"--{k.replace('_', '-')}": v for k, v in args.items()} - - arg_list = [] - for k, v in args.items(): - if not isinstance(v, bool): - arg_list.append(k) - if isinstance(v, list): - arg_list.extend(str(item) for item in v) - elif isinstance(v, enum.StrEnum): - arg_list.append(v.value) - else: - arg_list.append(str(v)) - elif v is True: - arg_list.append(k) - - if one_line: - return ' '.join(arg_list) - else: - return arg_list + +T = TypeVar('T', bound=BaseModel) + + +def from_args(cls: type[T], args: argparse.Namespace) -> T: + """Create an instance of the pydantic model from the argparse namespace. + + It ignores any extra arguments in the namespace that are not part of the model. + """ + kwargs = { + field_name: _retrieve_field_value(field_name, field_info, args) + for field_name, field_info in cls.model_fields.items() + } + return cls(**kwargs) + + +def build_reduction_argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Command line arguments for the ESS NMX reduction. " + "It assumes 14 Hz pulse speed." + ) + parser = add_args_from_pydantic_model(model_cls=InputConfig, parser=parser) + parser = add_args_from_pydantic_model(model_cls=WorkflowConfig, parser=parser) + parser = add_args_from_pydantic_model(model_cls=OutputConfig, parser=parser) + return parser + + +def reduction_config_from_args(args: argparse.Namespace) -> ReductionConfig: + return ReductionConfig( + inputs=from_args(InputConfig, args), + workflow=from_args(WorkflowConfig, args), + output=from_args(OutputConfig, args), + ) + + +def to_command_arguments( + config: ReductionConfig, one_line: bool = True +) -> list[str] | str: + """Convert the config to a list of command line arguments. + + Parameters + ---------- + one_line: + If True, return a single string with all arguments joined by spaces. + If False, return a list of argument strings. + + """ + args = {} + for instance in config._children: + args.update(instance.model_dump(mode='python')) + args = {f"--{k.replace('_', '-')}": v for k, v in args.items() if v is not None} + + arg_list = [] + for k, v in args.items(): + if not isinstance(v, bool): + arg_list.append(k) + if isinstance(v, list): + arg_list.extend(str(item) for item in v) + elif isinstance(v, enum.StrEnum): + arg_list.append(v.value) + else: + arg_list.append(str(v)) + elif v is True: + arg_list.append(k) + + if one_line: + return ' '.join(arg_list) + else: + return arg_list def build_logger(args: argparse.Namespace | OutputConfig) -> logging.Logger: diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index f4da09f2..813fcf50 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -12,7 +12,9 @@ from ._executable_helper import ( ReductionConfig, build_logger, + build_reduction_argument_parser, collect_matching_input_files, + reduction_config_from_args, ) # Temporarily keeping them until we migrate GenericWorkflow here @@ -190,57 +192,26 @@ def _retrieve_input_file(input_file: list[pathlib.Path] | pathlib.Path) -> pathl def reduction( *, - input_file: list[pathlib.Path] | pathlib.Path, - output_file: pathlib.Path, - chunk_size: int = 1_000, - detector_ids: list[int | str], - compression: Compression = Compression.BITSHUFFLE_LZ4, + config: ReductionConfig, logger: logging.Logger | None = None, - min_toa: sc.Variable | int = 0, - max_toa: sc.Variable | int = int((1 / 14) * 1_000), # Default for ESS NMX - toa_bin_edges: sc.Variable | int = 250, - fast_axis: Literal['x', 'y'] | None = None, # 'x', 'y', or None to auto-detect - display: Callable | None = None, # For Jupyter notebook display + display: Callable | None = None, ) -> sc.DataGroup: """Reduce NMX data from a Nexus file and export to NXLauetof(ESS NMX specific) file. - This workflow is written as a flatten function without using sciline Pipeline. - It is because the first part of NMX reduction only requires - a few steps of processing and it is overkill to use a Pipeline or GenericWorkflow. - - We also do not apply frame unwrapping or pulse skipping here, - as it is not expected from NMX experiments. - - Frame unwrapping may be applied later on the result of this function if needed - however, then the whole range of `event_time_offset` should have been histogrammed - so that the unwrapping can be applied. - i.e. `min_toa` should be 0 and `max_toa` should be 1/14 seconds - for 14 Hz pulse frequency. - TODO: Implement tof/wavelength workflow for NMX. - Parameters ---------- - input_file: - Path to the input Nexus file containing NMX data. - output_file: - Path to the output file where reduced data will be saved. - chunk_size: - Number of pulses to process in each chunk. If <= 0, all data is processed - at once. It represents the number of event_time_zero entries to read at once. - detector_ids: - List of detector IDs (as integers or names) to process. - compression: - If True, the output data will be compressed. + config: + Reduction configuration. + + Data reduction parameters are taken from this config + instead of passing them directly as keyword arguments. + They can be either built from command-line arguments + using `ReductionConfig.from_args()` or constructed manually. + + If the reduced data is successfully written to the output file + the configuration is also saved there for future reference. logger: Logger to use for logging messages. If None, a default logger is created. - min_toa: - Minimum time of arrival (TOA) in milliseconds. Default is 0 ms. - max_toa: - Maximum time of arrival (TOA) in milliseconds. Default is 1/14 seconds, - typical for ESS NMX. - toa_bin_edges: - Number of time of arrival (TOA) bin edges or a scipp Variable defining the - edges. Default is 250 edges. display: Callable for displaying messages, useful in Jupyter notebooks. If None, defaults to logger.info. @@ -259,9 +230,13 @@ def reduction( display = logger.info toa_bin_edges = build_toa_bin_edges( - min_toa=min_toa, max_toa=max_toa, toa_bin_edges=toa_bin_edges + min_toa=config.workflow.min_time_bin or 0, + max_toa=config.workflow.max_time_bin or int((1 / 14) * 1_000), + toa_bin_edges=config.workflow.nbins, + ) + input_file_path = _retrieve_input_file( + [pathlib.Path(p) for p in config.inputs.input_file] ) - input_file_path = _retrieve_input_file(input_file) with snx.File(input_file_path) as f: intrument_group = f['entry/instrument'] dets = intrument_group[snx.NXdetector] @@ -270,11 +245,12 @@ def reduction( detector_id_map = { det_name: dets[det_name] for i, det_name in enumerate(detector_group_keys) - if i in detector_ids or det_name in detector_ids + if i in config.inputs.detector_ids or det_name in config.inputs.detector_ids } - if len(detector_id_map) != len(detector_ids): + if len(detector_id_map) != len(config.inputs.detector_ids): raise ValueError( - f"Requested detector ids {detector_ids} not found in the file.\n" + f"Requested detector ids {config.inputs.detector_ids} " + "not found in the file.\n" f"Found {detector_group_keys}\n" f"Try using integer indices instead of names." ) @@ -299,20 +275,19 @@ def reduction( _export_static_metadata_as_nxlauetof( experiment_metadata=experiment_metadata, - output_file=output_file, + output_file=pathlib.Path(config.output.output_file), ) detector_grs = {} for det_name, det_group in detector_id_map.items(): display(f"Processing {det_name}") - if chunk_size <= 0: + if config.inputs.chunk_size_events <= 0: dg = det_group[()] else: # Slice the first chunk for metadata extraction - dg = det_group['event_time_zero', 0:chunk_size] - + dg = det_group['event_time_zero', 0 : config.inputs.chunk_size_events] display("Computing detector positions...") display(dg := _compute_positions(dg, auto_fix_transformations=True)) - detector = build_detector_desc(det_name, dg, fast_axis=fast_axis) + detector = build_detector_desc(det_name, dg) detector_meta = sc.DataGroup( { 'fast_axis': detector.fast_axis, @@ -331,14 +306,15 @@ def reduction( } ) _export_detector_metadata_as_nxlauetof( - NMXDetectorMetadata(detector_meta), output_file=output_file + NMXDetectorMetadata(detector_meta), + output_file=pathlib.Path(config.output.output_file), ) da: sc.DataArray = dg['data'] event_time_offset_unit = da.bins.coords['event_time_offset'].bins.unit display("Event time offset unit: %s", event_time_offset_unit) toa_bin_edges = toa_bin_edges.to(unit=event_time_offset_unit, copy=False) - if chunk_size <= 0: + if config.inputs.chunk_size_events <= 0: counts = da.hist(event_time_offset=toa_bin_edges).rename_dims( x_pixel_offset='x', y_pixel_offset='y', event_time_offset='t' ) @@ -346,7 +322,7 @@ def reduction( else: num_chunks = calculate_number_of_chunks( - det_group, chunk_size=chunk_size + det_group, chunk_size=config.inputs.chunk_size_events ) display(f"Number of chunks: {num_chunks}") counts = da.hist(event_time_offset=toa_bin_edges).rename_dims( @@ -356,7 +332,10 @@ def reduction( for chunk_index in range(1, num_chunks): cur_chunk = det_group[ 'event_time_zero', - chunk_index * chunk_size : (chunk_index + 1) * chunk_size, + chunk_index * config.inputs.chunk_size_events : ( + chunk_index + 1 + ) + * config.inputs.chunk_size_events, ] display(f"Processing chunk {chunk_index + 1} of {num_chunks}") cur_chunk = _compute_positions( @@ -386,18 +365,21 @@ def reduction( display("Saving reduced data to Nexus file...") _export_reduced_data_as_nxlauetof( dg, - output_file=output_file, - compress_counts=(compression == Compression.BITSHUFFLE_LZ4), + output_file=pathlib.Path(config.output.output_file), + compress_counts=( + config.output.compression == Compression.BITSHUFFLE_LZ4 + ), ) detector_grs[det_name] = dg display("Reduction completed successfully.") - return sc.DataGroup(detector_grs) + histograms = {name: det_gr['counts'] for name, det_gr in detector_grs.items()} + return sc.DataGroup(histogram=sc.DataGroup(histograms)) def main() -> None: - parser = ReductionConfig.build_argument_parser() - config = ReductionConfig.from_args(parser.parse_args()) + parser = build_reduction_argument_parser() + config = reduction_config_from_args(parser.parse_args()) input_file = collect_matching_input_files(*config.inputs.input_file) output_file = pathlib.Path(config.output.output_file).resolve() @@ -406,15 +388,4 @@ def main() -> None: logger.info("Input file: %s", input_file) logger.info("Output file: %s", output_file) - reduction( - input_file=input_file, - output_file=output_file, - chunk_size=config.inputs.chunk_size_pulse, - detector_ids=config.inputs.detector_ids, - compression=config.output.compression, - toa_bin_edges=config.workflow.nbins, - min_toa=sc.scalar(config.workflow.min_toa, unit='ms'), - max_toa=sc.scalar(config.workflow.max_toa, unit='ms'), - fast_axis=config.workflow.fast_axis, - logger=logger, - ) + reduction(config=config, logger=logger) diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/nexus.py new file mode 100644 index 00000000..1fc5858d --- /dev/null +++ b/packages/essnmx/src/ess/nmx/nexus.py @@ -0,0 +1,555 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2025 Scipp contributors (https://github.com/scipp) +import io +import pathlib +import warnings +from functools import wraps +from typing import Any + +import h5py +import numpy as np +import scipp as sc + + +def _fallback_compute_positions(dg: sc.DataGroup) -> sc.DataGroup: + import warnings + + import scippnexus as snx + + warnings.warn( + "Using fallback compute_positions due to empty log entries. " + "This may lead to incorrect results. Please check the data carefully." + "The fallback will replace empty logs with a scalar value of zero.", + UserWarning, + stacklevel=2, + ) + + empty_transformations = [ + transformation + for transformation in dg['depends_on'].transformations.values() + if 'time' in transformation.value.dims + and transformation.sizes['time'] == 0 # empty log + ] + for transformation in empty_transformations: + orig_value = transformation.value + orig_value = sc.scalar(0, unit=orig_value.unit, dtype=orig_value.dtype) + transformation.value = orig_value + return snx.compute_positions(dg, store_transform='transform_matrix') + + +def _compute_positions( + dg: sc.DataGroup, auto_fix_transformations: bool = False +) -> sc.DataGroup: + """Compute positions of the data group from transformations. + + Wraps the `scippnexus.compute_positions` function + and provides a fallback for cases where the transformations + contain empty logs. + + Parameters + ---------- + dg: + Data group containing the transformations and data. + auto_fix_transformations: + If `True`, it will attempt to fix empty transformations. + It will replace them with a scalar value of zero. + It is because adding a time dimension will make it not possible + to compute positions of children due to time-dependent transformations. + + Returns + ------- + : + Data group with computed positions. + + Warnings + -------- + If `auto_fix_transformations` is `True`, it will warn about the fallback + being used due to empty logs or scalar transformations. + This is because the fallback may lead to incorrect results. + + """ + import scippnexus as snx + + try: + return snx.compute_positions(dg, store_transform='transform_matrix') + except ValueError as e: + if auto_fix_transformations: + return _fallback_compute_positions(dg) + raise e + + +def _create_dataset_from_string(*, root_entry: h5py.Group, name: str, var: str) -> None: + root_entry.create_dataset(name, dtype=h5py.string_dtype(), data=var) + + +def _create_dataset_from_var( + *, + root_entry: h5py.Group, + var: sc.Variable, + name: str, + long_name: str | None = None, + compression: str | None = None, + compression_opts: int | tuple[int, int] | None = None, + chunks: tuple[int, ...] | int | bool | None = None, + dtype: Any = None, +) -> h5py.Dataset: + compression_options = {} + if compression is not None: + compression_options["compression"] = compression + if compression_opts is not None: + compression_options["compression_opts"] = compression_opts + + dataset = root_entry.create_dataset( + name, + data=var.values if dtype is None else var.values.astype(dtype, copy=False), + chunks=chunks, + **compression_options, + ) + if var.unit is not None: + dataset.attrs["units"] = str(var.unit) + if long_name is not None: + dataset.attrs["long_name"] = long_name + return dataset + + +@wraps(_create_dataset_from_var) +def _create_compressed_dataset(*args, **kwargs): + """Create dataset with compression options. + + It will try to use ``bitshuffle`` for compression if available. + Otherwise, it will fall back to ``gzip`` compression. + + [``Bitshuffle/LZ4``](https://github.com/kiyo-masui/bitshuffle) + is used for convenience. + Since ``Dectris`` uses it for their Nexus file compression, + it is compatible with DIALS. + ``Bitshuffle/LZ4`` tends to give similar results to + GZIP and other compression algorithms with better performance. + A naive implementation of bitshuffle/LZ4 compression, + shown in [issue #124](https://github.com/scipp/essnmx/issues/124), + led to 80% file reduction (365 MB vs 1.8 GB). + + """ + try: + import bitshuffle.h5 + + compression_filter = bitshuffle.h5.H5FILTER + default_compression_opts = (0, bitshuffle.h5.H5_COMPRESS_LZ4) + except ImportError: + warnings.warn( + UserWarning( + "Could not find the bitshuffle.h5 module from bitshuffle package. " + "The bitshuffle package is not installed or only partially installed. " + "Exporting to NeXus files with bitshuffle compression is not possible." + ), + stacklevel=2, + ) + compression_filter = "gzip" + default_compression_opts = 4 + + return _create_dataset_from_var( + *args, + **kwargs, + compression=compression_filter, + compression_opts=default_compression_opts, + ) + + +def _create_root_data_entry(file_obj: h5py.File) -> h5py.Group: + nx_entry = file_obj.create_group("NMX_data") + nx_entry.attrs["NX_class"] = "NXentry" + nx_entry.attrs["default"] = "data" + nx_entry.attrs["name"] = "NMX" + nx_entry["name"] = "NMX" + nx_entry["definition"] = "TOFRAW" + return nx_entry + + +def _create_sample_group(data: sc.DataGroup, nx_entry: h5py.Group) -> h5py.Group: + nx_sample = nx_entry.create_group("NXsample") + nx_sample["name"] = data['sample_name'].value + _create_dataset_from_var( + root_entry=nx_sample, + var=data['crystal_rotation'], + name='crystal_rotation', + long_name='crystal rotation in Phi (XYZ)', + ) + return nx_sample + + +def _create_instrument_group(data: sc.DataGroup, nx_entry: h5py.Group) -> h5py.Group: + nx_instrument = nx_entry.create_group("NXinstrument") + nx_instrument.create_dataset("proton_charge", data=data['proton_charge'].values) + + nx_detector_1 = nx_instrument.create_group("detector_1") + # Detector counts + _create_compressed_dataset( + root_entry=nx_detector_1, + name="counts", + var=data['counts'], + ) + # Time of arrival bin edges + _create_dataset_from_var( + root_entry=nx_detector_1, + var=data['counts'].coords['t'], + name="t_bin", + long_name="t_bin TOF (ms)", + ) + # Pixel IDs + _create_compressed_dataset( + root_entry=nx_detector_1, + name="pixel_id", + var=data['counts'].coords['id'], + long_name="pixel ID", + ) + return nx_instrument + + +def _create_detector_group(data: sc.DataGroup, nx_entry: h5py.Group) -> h5py.Group: + nx_detector = nx_entry.create_group("NXdetector") + # Position of the first pixel (lowest ID) in the detector + _create_compressed_dataset( + root_entry=nx_detector, + name="origin", + var=data['origin_position'], + ) + # Fast axis, along where the pixel ID increases by 1 + _create_dataset_from_var( + root_entry=nx_detector, var=data['fast_axis'], name="fast_axis" + ) + # Slow axis, along where the pixel ID increases + # by the number of pixels in the fast axis + _create_dataset_from_var( + root_entry=nx_detector, var=data['slow_axis'], name="slow_axis" + ) + return nx_detector + + +def _create_source_group(data: sc.DataGroup, nx_entry: h5py.Group) -> h5py.Group: + nx_source = nx_entry.create_group("NXsource") + nx_source["name"] = "European Spallation Source" + nx_source["short_name"] = "ESS" + nx_source["type"] = "Spallation Neutron Source" + nx_source["distance"] = sc.norm(data['source_position']).value + nx_source["probe"] = "neutron" + nx_source["target_material"] = "W" + return nx_source + + +def export_as_nexus( + data: sc.DataGroup, output_file: str | pathlib.Path | io.BytesIO +) -> None: + """Export the reduced data to a NeXus file. + + Currently exporting step is not expected to be part of sciline pipelines. + """ + warnings.warn( + DeprecationWarning( + "Exporting to custom NeXus format will be deprecated in the near future " + ">=26.12.0. " + "Please use ``export_as_nxlauetof`` instead." + ), + stacklevel=2, + ) + with h5py.File(output_file, "w") as f: + f.attrs["default"] = "NMX_data" + nx_entry = _create_root_data_entry(f) + _create_sample_group(data, nx_entry) + _create_instrument_group(data, nx_entry) + _create_detector_group(data, nx_entry) + _create_source_group(data, nx_entry) + + +def _create_lauetof_data_entry(file_obj: h5py.File) -> h5py.Group: + nx_entry = file_obj.create_group("entry") + nx_entry.attrs["NX_class"] = "NXentry" + return nx_entry + + +def _add_lauetof_definition(nx_entry: h5py.Group) -> None: + _create_dataset_from_string(root_entry=nx_entry, name="definition", var="NXlauetof") + + +def _add_lauetof_instrument(nx_entry: h5py.Group) -> h5py.Group: + nx_instrument = nx_entry.create_group("instrument") + nx_instrument.attrs["NX_class"] = "NXinstrument" + _create_dataset_from_string(root_entry=nx_instrument, name="name", var="NMX") + return nx_instrument + + +def _add_lauetof_source_group(dg, nx_instrument: h5py.Group) -> None: + nx_source = nx_instrument.create_group("source") + nx_source.attrs["NX_class"] = "NXsource" + _create_dataset_from_string( + root_entry=nx_source, name="name", var="European Spallation Source" + ) + _create_dataset_from_string(root_entry=nx_source, name="short_name", var="ESS") + _create_dataset_from_string( + root_entry=nx_source, name="type", var="Spallation Neutron Source" + ) + _create_dataset_from_var( + root_entry=nx_source, name="distance", var=sc.norm(dg["source_position"]) + ) + # Legacy probe information. + _create_dataset_from_string(root_entry=nx_source, name="probe", var="neutron") + + +def _add_lauetof_detector_group(dg: sc.DataGroup, nx_instrument: h5py.Group) -> None: + nx_detector = nx_instrument.create_group(dg["detector_name"].value) # Detector name + nx_detector.attrs["NX_class"] = "NXdetector" + _create_dataset_from_var( + name="polar_angle", + root_entry=nx_detector, + var=sc.scalar(0, unit='deg'), # TODO: Add real data + ) + _create_dataset_from_var( + name="azimuthal_angle", + root_entry=nx_detector, + var=sc.scalar(0, unit='deg'), # TODO: Add real data + ) + _create_dataset_from_var( + name="x_pixel_size", root_entry=nx_detector, var=dg["x_pixel_size"] + ) + _create_dataset_from_var( + name="y_pixel_size", root_entry=nx_detector, var=dg["y_pixel_size"] + ) + _create_dataset_from_var( + name="distance", + root_entry=nx_detector, + var=sc.scalar(0, unit='m'), # TODO: Add real data + ) + # Legacy geometry information until we have a better way to store it + _create_dataset_from_var( + name="origin", root_entry=nx_detector, var=dg['origin_position'] + ) + # Fast axis, along where the pixel ID increases by 1 + _create_dataset_from_var( + root_entry=nx_detector, var=dg['fast_axis'], name="fast_axis" + ) + # Slow axis, along where the pixel ID increases + # by the number of pixels in the fast axis + _create_dataset_from_var( + root_entry=nx_detector, var=dg['slow_axis'], name="slow_axis" + ) + + +def _add_lauetof_sample_group(dg, nx_entry: h5py.Group) -> None: + nx_sample = nx_entry.create_group("sample") + nx_sample.attrs["NX_class"] = "NXsample" + _create_dataset_from_var( + root_entry=nx_sample, + var=dg['crystal_rotation'], + name='crystal_rotation', + long_name='crystal rotation in Phi (XYZ)', + ) + _create_dataset_from_string( + root_entry=nx_sample, + name='name', + var=dg['sample_name'].value, + ) + _create_dataset_from_var( + name='orientation_matrix', + root_entry=nx_sample, + var=sc.array( + dims=['i', 'j'], + values=[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], + unit="dimensionless", + ), # TODO: Add real data, the sample orientation matrix + ) + _create_dataset_from_var( + name='unit_cell', + root_entry=nx_sample, + var=sc.array( + dims=['i'], + values=[1.0, 1.0, 1.0, 90.0, 90.0, 90.0], + unit="dimensionless", # TODO: Add real data, + # a, b, c, alpha, beta, gamma + ), + ) + + +def _add_lauetof_monitor_group(data: sc.DataGroup, nx_entry: h5py.Group) -> None: + nx_monitor = nx_entry.create_group("control") + nx_monitor.attrs["NX_class"] = "NXmonitor" + _create_dataset_from_string(root_entry=nx_monitor, name='mode', var='monitor') + nx_monitor["preset"] = 0.0 # Check if this is the correct value + data_dset = _create_dataset_from_var( + name='data', + root_entry=nx_monitor, + var=sc.array( + dims=['tof'], values=[1, 1, 1], unit="counts" + ), # TODO: Add real data, bin values + ) + data_dset.attrs["signal"] = 1 + data_dset.attrs["primary"] = 1 + _create_dataset_from_var( + name='time_of_flight', + root_entry=nx_monitor, + var=sc.array( + dims=['tof'], values=[1, 1, 1], unit="s" + ), # TODO: Add real data, bin edges + ) + + +def _add_arbitrary_metadata( + nx_entry: h5py.Group, **arbitrary_metadata: sc.Variable +) -> None: + if not arbitrary_metadata: + return + + metadata_group = nx_entry.create_group("metadata") + for key, value in arbitrary_metadata.items(): + if not isinstance(value, sc.Variable): + import warnings + + msg = f"Skipping metadata key '{key}' as it is not a scipp.Variable." + warnings.warn(UserWarning(msg), stacklevel=2) + continue + else: + _create_dataset_from_var( + name=key, + root_entry=metadata_group, + var=value, + ) + + +def _export_static_metadata_as_nxlauetof( + experiment_metadata, + output_file: str | pathlib.Path | io.BytesIO, + **arbitrary_metadata: sc.Variable, +) -> None: + """Export the metadata to a NeXus file with the LAUE_TOF application definition. + + ``Metadata`` in this context refers to the information + that is not part of the reduced detector counts itself, + but is necessary for the interpretation of the reduced data. + Since NMX can have arbitrary number of detectors, + this function can take multiple detector metadata objects. + + Parameters + ---------- + experiment_metadata: + Experiment metadata object. + output_file: + Output file path. + arbitrary_metadata: + Arbitrary metadata that does not fit into the existing metadata objects. + + """ + with h5py.File(output_file, "w") as f: + f.attrs["NX_class"] = "NXlauetof" + nx_entry = _create_lauetof_data_entry(f) + _add_lauetof_definition(nx_entry) + _add_lauetof_sample_group(experiment_metadata, nx_entry) + nx_instrument = _add_lauetof_instrument(nx_entry) + _add_lauetof_source_group(experiment_metadata, nx_instrument) + # Placeholder for ``monitor`` group + _add_lauetof_monitor_group(experiment_metadata, nx_entry) + # Skipping ``NXdata``(name) field with data link + # Add arbitrary metadata + _add_arbitrary_metadata(nx_entry, **arbitrary_metadata) + + +def _export_detector_metadata_as_nxlauetof( + *detector_metadatas, + output_file: str | pathlib.Path | io.BytesIO, + append_mode: bool = True, +) -> None: + """Export the detector specific metadata to a NeXus file. + + Since NMX can have arbitrary number of detectors, + this function can take multiple detector metadata objects. + + Parameters + ---------- + detector_metadatas: + Detector metadata objects. + output_file: + Output file path. + + """ + + if not append_mode: + raise NotImplementedError("Only append mode is supported for now.") + + with h5py.File(output_file, "r+") as f: + nx_entry = f["entry"] + if "instrument" not in nx_entry: + nx_instrument = _add_lauetof_instrument(f["entry"]) + else: + nx_instrument = nx_entry["instrument"] + # Add detector group metadata + for detector_metadata in detector_metadatas: + _add_lauetof_detector_group(detector_metadata, nx_instrument) + + +def _extract_counts(dg: sc.DataGroup) -> sc.Variable: + counts: sc.DataArray = dg['counts'].data + if 'id' in counts.dims: + num_x, num_y = dg["detector_shape"].value + return sc.fold(counts, dim='id', sizes={'x': num_x, 'y': num_y}) + else: + # If there is no 'id' dimension, we assume it is already in the correct shape + return counts + + +def _export_reduced_data_as_nxlauetof( + dg, + output_file: str | pathlib.Path | io.BytesIO, + *, + append_mode: bool = True, + compress_counts: bool = True, +) -> None: + """Export the reduced data to a NeXus file with the LAUE_TOF application definition. + + Even though this function only exports + reduced data(detector counts and its coordinates), + the input should contain all the necessary metadata + for minimum sanity check. + + Parameters + ---------- + dg: + Reduced data and metadata. + output_file: + Output file path. + append_mode: + If ``True``, the file is opened in append mode. + If ``False``, the file is opened in None-append mode. + > None-append mode is not supported for now. + > Only append mode is supported for now. + compress_counts: + If ``True``, the detector counts are compressed using bitshuffle. + It is because only the detector counts are expected to be large. + + """ + if not append_mode: + raise NotImplementedError("Only append mode is supported for now.") + + with h5py.File(output_file, "r+") as f: + nx_detector: h5py.Group = f[f"entry/instrument/{dg['detector_name'].value}"] + # Data - shape: [n_x_pixels, n_y_pixels, n_tof_bins] + # The actual application definition defines it as integer, + # but we keep the original data type for now + num_x, num_y = dg["detector_shape"].value # Probably better way to do this + if compress_counts: + data_dset = _create_compressed_dataset( + name="data", + root_entry=nx_detector, + var=_extract_counts(dg), + chunks=(num_x, num_y, 1), + dtype=np.uint, + ) + else: + data_dset = _create_dataset_from_var( + name="data", + root_entry=nx_detector, + var=_extract_counts(dg), + dtype=np.uint, + ) + data_dset.attrs["signal"] = 1 + _create_dataset_from_var( + name='time_of_flight', + root_entry=nx_detector, + var=sc.midpoints(dg['counts'].coords['t'], dim='t'), + ) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index aa20c1d5..6c428792 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -15,8 +15,12 @@ InputConfig, OutputConfig, ReductionConfig, - TOAUnit, + TimeBinCoordinate, + TimeBinUnit, WorkflowConfig, + build_reduction_argument_parser, + reduction_config_from_args, + to_command_arguments, ) from ess.nmx.types import Compression @@ -25,7 +29,7 @@ def _build_arg_list_from_pydantic_instance(*instances: pydantic.BaseModel) -> li args = {} for instance in instances: args.update(instance.model_dump(mode='python')) - args = {f"--{k.replace('_', '-')}": v for k, v in args.items()} + args = {f"--{k.replace('_', '-')}": v for k, v in args.items() if v is not None} arg_list = [] for k, v in args.items(): @@ -63,6 +67,9 @@ def _check_non_default_config(testing_config: ReductionConfig) -> None: testing_model = testing_child.model_dump(mode='python') default_model = default_child.model_dump(mode='python') for key, testing_value in testing_model.items(): + if key == 'tof_lookup_table_file_path': + # This value may be None or default, so we skip the check. + continue default_value = default_model[key] assert ( testing_value != default_value @@ -81,7 +88,14 @@ def test_reduction_config() -> None: chunk_size_events=100000, ) workflow_options = WorkflowConfig( - nbins=100, min_toa=10, max_toa=100_000, toa_unit=TOAUnit.us, fast_axis='y' + nbins=100, + min_time_bin=10, + max_time_bin=100_000, + time_bin_coordinate=TimeBinCoordinate.time_of_flight, + time_bin_unit=TimeBinUnit.us, + tof_simulation_max_wavelength=5.0, + tof_simulation_min_wavelength=1.0, + tof_simulation_seed=12345, ) output_options = OutputConfig( output_file='test-output.h5', compression=Compression.NONE, verbose=True @@ -96,12 +110,12 @@ def test_reduction_config() -> None: arg_list = _build_arg_list_from_pydantic_instance( input_options, workflow_options, output_options ) - assert arg_list == expected_config.to_command_arguments(one_line=False) + assert arg_list == to_command_arguments(expected_config, one_line=False) # Parse arguments and build config from them. - parser = ReductionConfig.build_argument_parser() + parser = build_reduction_argument_parser() args = parser.parse_args(arg_list) - config = ReductionConfig.from_args(args) + config = reduction_config_from_args(args) assert expected_config == config @@ -150,9 +164,9 @@ def test_executable_runs(small_nmx_nexus_path, tmp_path: pathlib.Path): str(nbins), '--output-file', output_file.as_posix(), - '--min-toa', + '--min-time-bin', str(int(expected_toa_bins.min().value)), - '--max-toa', + '--max-time-bin', str(int(expected_toa_bins.max().value)), ) # Validate that all commands are strings and contain no unsafe characters From e749d3810caa0569929c664c6f55fc691e1fe88c Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 4 Dec 2025 11:24:22 +0100 Subject: [PATCH 306/403] Wrap display retrieving. --- packages/essnmx/src/ess/nmx/executables.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 813fcf50..42f2c184 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -190,6 +190,17 @@ def _retrieve_input_file(input_file: list[pathlib.Path] | pathlib.Path) -> pathl return input_file_path +def _retrieve_display( + logger: logging.Logger | None, display: Callable | None +) -> Callable: + if display is not None: + return display + elif logger is not None: + return logger.info + else: + return logging.getLogger(__name__).info + + def reduction( *, config: ReductionConfig, @@ -222,12 +233,7 @@ def reduction( A DataGroup containing the reduced data for each selected detector. """ - import scippnexus as snx - - if logger is None: - logger = logging.getLogger(__name__) - if display is None: - display = logger.info + display = _retrieve_display(logger, display) toa_bin_edges = build_toa_bin_edges( min_toa=config.workflow.min_time_bin or 0, From 1afdbafd53f04dd76eff0b288f27ea9bf024dc85 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 4 Dec 2025 11:40:17 +0100 Subject: [PATCH 307/403] Retrieve input/output file paths in the reduction function. --- packages/essnmx/src/ess/nmx/executables.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 42f2c184..52ded282 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -234,15 +234,19 @@ def reduction( """ display = _retrieve_display(logger, display) + input_file_path = _retrieve_input_file( + collect_matching_input_files(*config.inputs.input_file) + ).resolve() + display(f"Input file: {input_file_path}") + + output_file_path = pathlib.Path(config.output.output_file).resolve() + display(f"Output file: {output_file_path}") toa_bin_edges = build_toa_bin_edges( min_toa=config.workflow.min_time_bin or 0, max_toa=config.workflow.max_time_bin or int((1 / 14) * 1_000), toa_bin_edges=config.workflow.nbins, ) - input_file_path = _retrieve_input_file( - [pathlib.Path(p) for p in config.inputs.input_file] - ) with snx.File(input_file_path) as f: intrument_group = f['entry/instrument'] dets = intrument_group[snx.NXdetector] @@ -281,7 +285,7 @@ def reduction( _export_static_metadata_as_nxlauetof( experiment_metadata=experiment_metadata, - output_file=pathlib.Path(config.output.output_file), + output_file=output_file_path, ) detector_grs = {} for det_name, det_group in detector_id_map.items(): @@ -313,7 +317,7 @@ def reduction( ) _export_detector_metadata_as_nxlauetof( NMXDetectorMetadata(detector_meta), - output_file=pathlib.Path(config.output.output_file), + output_file=output_file_path, ) da: sc.DataArray = dg['data'] @@ -371,7 +375,7 @@ def reduction( display("Saving reduced data to Nexus file...") _export_reduced_data_as_nxlauetof( dg, - output_file=pathlib.Path(config.output.output_file), + output_file=output_file_path, compress_counts=( config.output.compression == Compression.BITSHUFFLE_LZ4 ), @@ -386,12 +390,6 @@ def reduction( def main() -> None: parser = build_reduction_argument_parser() config = reduction_config_from_args(parser.parse_args()) - - input_file = collect_matching_input_files(*config.inputs.input_file) - output_file = pathlib.Path(config.output.output_file).resolve() logger = build_logger(config.output) - logger.info("Input file: %s", input_file) - logger.info("Output file: %s", output_file) - reduction(config=config, logger=logger) From 2e9b00145d8ca0cf9aa0dd866b623eea20063157 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 4 Dec 2025 14:06:06 +0100 Subject: [PATCH 308/403] Use generic workflow. --- .../essnmx/src/ess/nmx/_executable_helper.py | 146 +---- packages/essnmx/src/ess/nmx/configurations.py | 149 ++++++ packages/essnmx/src/ess/nmx/executables.py | 420 +++++---------- packages/essnmx/src/ess/nmx/nexus.py | 503 +++++++----------- packages/essnmx/src/ess/nmx/types.py | 68 +++ packages/essnmx/src/ess/nmx/workflows.py | 308 +++++++++++ packages/essnmx/tests/executable_test.py | 82 ++- 7 files changed, 889 insertions(+), 787 deletions(-) create mode 100644 packages/essnmx/src/ess/nmx/configurations.py create mode 100644 packages/essnmx/src/ess/nmx/workflows.py diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index d8ca016a..afa18438 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -10,11 +10,11 @@ from types import UnionType from typing import Literal, TypeGuard, TypeVar, Union, get_args, get_origin -from pydantic import BaseModel, Field +from pydantic import BaseModel from pydantic.fields import FieldInfo from pydantic_core import PydanticUndefined -from .types import Compression +from .configurations import InputConfig, OutputConfig, ReductionConfig, WorkflowConfig def _validate_annotation(annotation) -> TypeGuard[type]: @@ -140,148 +140,6 @@ def add_args_from_pydantic_model( return parser -class InputConfig(BaseModel): - # Add title of the basemodel - model_config = {"title": "Input Configuration"} - # File IO - input_file: list[str] = Field( - title="Input File", - description="Path to the input file. If multiple file paths are given," - " the output(histogram) will be merged(summed) " - "and will not save individual outputs per input file. ", - ) - swmr: bool = Field( - title="SWMR Mode", - description="Open the input file in SWMR mode", - default=False, - ) - # Detector selection - detector_ids: list[int] = Field( - title="Detector IDs", - description="Detector indices to process", - default=[0, 1, 2], - ) - # Chunking options - iter_chunk: bool = Field( - title="Iterate in Chunks", - description="Whether to process the input file in chunks " - " based on the hdf5 dataset chunk size. " - "It is ignored if hdf5 dataset is not chunked. " - "If True, it overrides chunk-size-pulse and chunk-size-events options.", - default=False, - ) - chunk_size_pulse: int = Field( - title="Chunk Size Pulse", - description="Number of pulses to process in each chunk. " - "If 0 or negative, process all pulses at once.", - default=0, - ) - chunk_size_events: int = Field( - title="Chunk Size Events", - description="Number of events to process in each chunk. " - "If 0 or negative, process all events at once." - "If both chunk-size-pulse and chunk-size-events are set, " - "chunk-size-pulse is preferred.", - default=0, - ) - - -class TimeBinUnit(enum.StrEnum): - ms = 'ms' - us = 'us' - ns = 'ns' - - -class TimeBinCoordinate(enum.StrEnum): - event_time_offset = 'event_time_offset' - time_of_flight = 'time_of_flight' - - -class WorkflowConfig(BaseModel): - # Add title of the basemodel - model_config = {"title": "Workflow Configuration"} - time_bin_coordinate: TimeBinCoordinate = Field( - title="Time Bin Coordinate", - description="Coordinate to bin the time data.", - default=TimeBinCoordinate.event_time_offset, - ) - nbins: int = Field( - title="Number of Time Bins", - description="Number of Time bins", - default=50, - ) - min_time_bin: int | None = Field( - title="Minimum Time Bin", - description="Minimum time edge of [time_bin_coordinate] in [time_bin_unit].", - default=None, - ) - max_time_bin: int | None = Field( - title="Maximum Time Bin", - description="Maximum time edge of [time_bin_coordinate] in [time_bin_unit].", - default=None, - ) - time_bin_unit: TimeBinUnit = Field( - title="Unit of Time Bins", - description="Unit of time bins.", - default=TimeBinUnit.ms, - ) - tof_lookup_table_file_path: str | None = Field( - title="TOF Lookup Table File Path", - description="Path to the TOF lookup table file. " - "If None, the lookup table will be computed on-the-fly.", - default=None, - ) - tof_simulation_min_wavelength: float = Field( - title="TOF Simulation Minimum Wavelength", - description="Minimum wavelength for TOF simulation in Angstrom.", - default=1.8, - ) - tof_simulation_max_wavelength: float = Field( - title="TOF Simulation Maximum Wavelength", - description="Maximum wavelength for TOF simulation in Angstrom.", - default=3.6, - ) - tof_simulation_seed: int = Field( - title="TOF Simulation Seed", - description="Random seed for TOF simulation.", - default=42, # No reason. - ) - - -class OutputConfig(BaseModel): - # Add title of the basemodel - model_config = {"title": "Output Configuration"} - # Log verbosity - verbose: bool = Field( - title="Verbose Logging", - description="Increase output verbosity.", - default=False, - ) - # File output - output_file: str = Field( - title="Output File", - description="Path to the output file.", - default="scipp_output.h5", - ) - compression: Compression = Field( - title="Compression", - description="Compress option of reduced output file.", - default=Compression.BITSHUFFLE_LZ4, - ) - - -class ReductionConfig(BaseModel): - """Container for all reduction configurations.""" - - inputs: InputConfig - workflow: WorkflowConfig = Field(default_factory=WorkflowConfig) - output: OutputConfig = Field(default_factory=OutputConfig) - - @property - def _children(self) -> list[BaseModel]: - return [self.inputs, self.workflow, self.output] - - T = TypeVar('T', bound=BaseModel) diff --git a/packages/essnmx/src/ess/nmx/configurations.py b/packages/essnmx/src/ess/nmx/configurations.py new file mode 100644 index 00000000..87e1ce97 --- /dev/null +++ b/packages/essnmx/src/ess/nmx/configurations.py @@ -0,0 +1,149 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2025 Scipp contributors (https://github.com/scipp) +import enum + +from pydantic import BaseModel, Field + +from .types import Compression + + +class InputConfig(BaseModel): + # Add title of the basemodel + model_config = {"title": "Input Configuration"} + # File IO + input_file: list[str] = Field( + title="Input File", + description="Path to the input file. If multiple file paths are given," + " the output(histogram) will be merged(summed) " + "and will not save individual outputs per input file. ", + ) + swmr: bool = Field( + title="SWMR Mode", + description="Open the input file in SWMR mode", + default=False, + ) + # Detector selection + detector_ids: list[int] = Field( + title="Detector IDs", + description="Detector indices to process", + default=[0, 1, 2], + ) + # Chunking options + iter_chunk: bool = Field( + title="Iterate in Chunks", + description="Whether to process the input file in chunks " + " based on the hdf5 dataset chunk size. " + "It is ignored if hdf5 dataset is not chunked. " + "If True, it overrides chunk-size-pulse and chunk-size-events options.", + default=False, + ) + chunk_size_pulse: int = Field( + title="Chunk Size Pulse", + description="Number of pulses to process in each chunk. " + "If 0 or negative, process all pulses at once.", + default=0, + ) + chunk_size_events: int = Field( + title="Chunk Size Events", + description="Number of events to process in each chunk. " + "If 0 or negative, process all events at once." + "If both chunk-size-pulse and chunk-size-events are set, " + "chunk-size-pulse is preferred.", + default=0, + ) + + +class TimeBinUnit(enum.StrEnum): + ms = 'ms' + us = 'us' + ns = 'ns' + + +class TimeBinCoordinate(enum.StrEnum): + event_time_offset = 'event_time_offset' + time_of_flight = 'time_of_flight' + + +class WorkflowConfig(BaseModel): + # Add title of the basemodel + model_config = {"title": "Workflow Configuration"} + time_bin_coordinate: TimeBinCoordinate = Field( + title="Time Bin Coordinate", + description="Coordinate to bin the time data.", + default=TimeBinCoordinate.event_time_offset, + ) + nbins: int = Field( + title="Number of Time Bins", + description="Number of Time bins", + default=50, + ) + min_time_bin: int | None = Field( + title="Minimum Time Bin", + description="Minimum time edge of [time_bin_coordinate] in [time_bin_unit].", + default=None, + ) + max_time_bin: int | None = Field( + title="Maximum Time Bin", + description="Maximum time edge of [time_bin_coordinate] in [time_bin_unit].", + default=None, + ) + time_bin_unit: TimeBinUnit = Field( + title="Unit of Time Bins", + description="Unit of time bins.", + default=TimeBinUnit.ms, + ) + tof_lookup_table_file_path: str | None = Field( + title="TOF Lookup Table File Path", + description="Path to the TOF lookup table file. " + "If None, the lookup table will be computed on-the-fly.", + default=None, + ) + tof_simulation_min_wavelength: float = Field( + title="TOF Simulation Minimum Wavelength", + description="Minimum wavelength for TOF simulation in Angstrom.", + default=1.8, + ) + tof_simulation_max_wavelength: float = Field( + title="TOF Simulation Maximum Wavelength", + description="Maximum wavelength for TOF simulation in Angstrom.", + default=3.6, + ) + tof_simulation_seed: int = Field( + title="TOF Simulation Seed", + description="Random seed for TOF simulation.", + default=42, # No reason. + ) + + +class OutputConfig(BaseModel): + # Add title of the basemodel + model_config = {"title": "Output Configuration"} + # Log verbosity + verbose: bool = Field( + title="Verbose Logging", + description="Increase output verbosity.", + default=False, + ) + # File output + output_file: str = Field( + title="Output File", + description="Path to the output file.", + default="scipp_output.h5", + ) + compression: Compression = Field( + title="Compression", + description="Compress option of reduced output file.", + default=Compression.BITSHUFFLE_LZ4, + ) + + +class ReductionConfig(BaseModel): + """Container for all reduction configurations.""" + + inputs: InputConfig + workflow: WorkflowConfig = Field(default_factory=WorkflowConfig) + output: OutputConfig = Field(default_factory=OutputConfig) + + @property + def _children(self) -> list[BaseModel]: + return [self.inputs, self.workflow, self.output] diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 52ded282..2d83cb42 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -2,176 +2,35 @@ # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) import logging import pathlib -from collections.abc import Callable -from dataclasses import dataclass -from typing import Literal +from collections.abc import Callable, Iterable +import sciline as sl import scipp as sc import scippnexus as snx +from ess.reduce.nexus.types import Filename, NeXusName, SampleRun +from ess.reduce.time_of_flight.types import TimeOfFlightLookupTable, TofDetector + from ._executable_helper import ( - ReductionConfig, build_logger, build_reduction_argument_parser, collect_matching_input_files, reduction_config_from_args, ) - -# Temporarily keeping them until we migrate GenericWorkflow here -from .mcstas.nexus import ( - _compute_positions, - _export_detector_metadata_as_nxlauetof, - _export_reduced_data_as_nxlauetof, - _export_static_metadata_as_nxlauetof, +from .configurations import ReductionConfig, WorkflowConfig +from .nexus import ( + export_detector_metadata_as_nxlauetof, + export_monitor_metadata_as_nxlauetof, + export_reduced_data_as_nxlauetof, + export_static_metadata_as_nxlauetof, ) - -# Temporarily keeping them until we migrate GenericWorkflow here -from .mcstas.types import NMXDetectorMetadata, NMXExperimentMetadata -from .types import Compression - - -def _validate_chunk_size(chunk_size: int) -> None: - """Validate the chunk size.""" - if not isinstance(chunk_size, int): - raise TypeError("Chunk size must be an integer.") - if chunk_size < -1: - raise ValueError("Invalid chunk size. It should be -1(for all) or > 0.") - - -def _retrieve_source_position(file: snx.File) -> sc.Variable: - da = file['entry/instrument/source'][()] - return _compute_positions(da, auto_fix_transformations=True)['position'] - - -def _retrieve_sample_position(file: snx.File) -> sc.Variable: - da = file['entry/sample'][()] - return _compute_positions(da, auto_fix_transformations=True)['position'] - - -def _retrieve_crystal_rotation(file: snx.File) -> sc.Variable: - if 'crystal_rotation' not in file['entry/sample']: - import warnings - - warnings.warn( - "No crystal rotation found in the Nexus file under " - "'entry/sample/crystal_rotation'. Returning zero rotation.", - RuntimeWarning, - stacklevel=2, - ) - return sc.vector([0, 0, 0], unit='deg') - - # Temporary way of storing crystal rotation. - # streaming-sample-mcstas module writes crystal rotation under - # 'entry/sample/crystal_rotation' as an array of three values. - return file['entry/sample/crystal_rotation'][()] - - -def _decide_fast_axis(da: sc.DataArray) -> str: - x_slice = da['x_pixel_offset', 0].coords['detector_number'] - y_slice = da['y_pixel_offset', 0].coords['detector_number'] - - if (x_slice.max() < y_slice.max()).value: - return 'y' - elif (x_slice.max() > y_slice.max()).value: - return 'x' - else: - raise ValueError( - "Cannot decide fast axis based on pixel offsets. " - "Please specify the fast axis explicitly." - ) - - -def _decide_step(offsets: sc.Variable) -> sc.Variable: - """Decide the step size based on the offsets assuming at least 2 values.""" - sorted_offsets = sc.sort(offsets, key=offsets.dim, order='ascending') - return sorted_offsets[1] - sorted_offsets[0] - - -@dataclass -class DetectorDesc: - """Detector information extracted from McStas instrument xml description.""" - - name: str - id_start: int # 'idstart' - num_x: int # 'xpixels' - num_y: int # 'ypixels' - step_x: sc.Variable # 'xstep' - step_y: sc.Variable # 'ystep' - start_x: float # 'xstart' - start_y: float # 'ystart' - position: sc.Variable # 'x', 'y', 'z' - # Calculated fields - rotation_matrix: sc.Variable - fast_axis_name: str - slow_axis_name: str - fast_axis: sc.Variable - slow_axis: sc.Variable - - -def build_detector_desc( - name: str, dg: sc.DataGroup, *, fast_axis: Literal['x', 'y'] | None = None -) -> DetectorDesc: - da: sc.DataArray = dg['data'] - _fast_axis = fast_axis if fast_axis is not None else _decide_fast_axis(da) - transformation_matrix = dg['transform_matrix'] - t_unit = transformation_matrix.unit - fast_axis_vector = ( - sc.vector([1, 0, 0], unit=t_unit) - if _fast_axis == 'x' - else sc.vector([0, 1, 0], unit=t_unit) - ) - slow_axis_vector = ( - sc.vector([0, 1, 0], unit=t_unit) - if _fast_axis == 'x' - else sc.vector([1, 0, 0], unit=t_unit) - ) - return DetectorDesc( - name=name, - id_start=da.coords['detector_number'].min().value, - num_x=da.sizes['x_pixel_offset'], - num_y=da.sizes['y_pixel_offset'], - start_x=da.coords['x_pixel_offset'].min().value, - start_y=da.coords['y_pixel_offset'].min().value, - position=dg['position'], - rotation_matrix=dg['transform_matrix'], - fast_axis_name=_fast_axis, - slow_axis_name='x' if _fast_axis == 'y' else 'y', - fast_axis=fast_axis_vector, - slow_axis=slow_axis_vector, - step_x=_decide_step(da.coords['x_pixel_offset']), - step_y=_decide_step(da.coords['y_pixel_offset']), - ) - - -def calculate_number_of_chunks(detector_gr: snx.Group, *, chunk_size: int = 0) -> int: - _validate_chunk_size(chunk_size) - event_time_zero_size = detector_gr.sizes['event_time_zero'] - if chunk_size == -1: - return 1 # Read all at once - else: - return event_time_zero_size // chunk_size + int( - event_time_zero_size % chunk_size != 0 - ) - - -def build_toa_bin_edges( - *, - min_toa: sc.Variable | int = 0, - max_toa: sc.Variable | int = int((1 / 14) * 1_000), # Default for ESS NMX - toa_bin_edges: sc.Variable | int = 250, -) -> sc.Variable: - if isinstance(toa_bin_edges, sc.Variable): - return toa_bin_edges - elif isinstance(toa_bin_edges, int): - min_toa = sc.scalar(min_toa, unit='ms') if isinstance(min_toa, int) else min_toa - max_toa = sc.scalar(max_toa, unit='ms') if isinstance(max_toa, int) else max_toa - return sc.linspace( - dim='event_time_offset', - start=min_toa.value, - stop=max_toa.to(unit=min_toa.unit).value, - unit=min_toa.unit, - num=toa_bin_edges + 1, - ) +from .types import ( + NMXDetectorMetadata, + NMXMonitorMetadata, + NMXSampleMetadata, + NMXSourceMetadata, +) +from .workflows import NMXWorkflow, compute_lookup_table, select_detector_names def _retrieve_input_file(input_file: list[pathlib.Path] | pathlib.Path) -> pathlib.Path: @@ -201,6 +60,45 @@ def _retrieve_display( return logging.getLogger(__name__).info +def compute_and_cache_lookup_table( + *, + wf: sl.Pipeline, + workflow_config: WorkflowConfig, + detector_names: Iterable[str], + display: Callable, +) -> sl.Pipeline: + """Compute and cache the TOF lookup table in the workflow. + + **Note**: ``base_wf`` is modified in-place and also returned. + """ + # We compute one lookup table that covers all range + # to avoid multiple tof simulations. + if workflow_config.tof_lookup_table_file_path is None: + display("Computing TOF lookup table from simulation...") + else: + display("Loading TOF lookup table from file...") + + lookup_table = compute_lookup_table( + base_wf=wf, workflow_config=workflow_config, detector_names=detector_names + ) + wf[TimeOfFlightLookupTable] = lookup_table + return wf + + +def _finalize_tof_bin_edges( + *, tof_das: sc.DataGroup, config: WorkflowConfig +) -> sc.Variable: + tof_bin_edges = sc.concat( + tuple(tof_da.coords['tof'] for tof_da in tof_das.values()), dim='tof' + ) + return sc.linspace( + dim='tof', + start=sc.min(tof_bin_edges), + stop=sc.max(tof_bin_edges), + num=config.nbins + 1, + ) + + def reduction( *, config: ReductionConfig, @@ -242,149 +140,73 @@ def reduction( output_file_path = pathlib.Path(config.output.output_file).resolve() display(f"Output file: {output_file_path}") - toa_bin_edges = build_toa_bin_edges( - min_toa=config.workflow.min_time_bin or 0, - max_toa=config.workflow.max_time_bin or int((1 / 14) * 1_000), - toa_bin_edges=config.workflow.nbins, + detector_names = select_detector_names( + input_files=[input_file_path], detector_ids=config.inputs.detector_ids + ) + + base_wf = NMXWorkflow() + # Insert input file path into the workflow for later use + base_wf[Filename] = input_file_path + + base_wf = compute_and_cache_lookup_table( + wf=base_wf, + workflow_config=config.workflow, + detector_names=detector_names, + display=display, ) - with snx.File(input_file_path) as f: - intrument_group = f['entry/instrument'] - dets = intrument_group[snx.NXdetector] - detector_group_keys = list(dets.keys()) - display(f"Found NXdetectors: {detector_group_keys}") - detector_id_map = { - det_name: dets[det_name] - for i, det_name in enumerate(detector_group_keys) - if i in config.inputs.detector_ids or det_name in config.inputs.detector_ids - } - if len(detector_id_map) != len(config.inputs.detector_ids): - raise ValueError( - f"Requested detector ids {config.inputs.detector_ids} " - "not found in the file.\n" - f"Found {detector_group_keys}\n" - f"Try using integer indices instead of names." - ) - display(f"Selected detectors: {list(detector_id_map.keys())}") - source_position = _retrieve_source_position(f) - sample_position = _retrieve_sample_position(f) - crystal_rotation = _retrieve_crystal_rotation(f) - experiment_metadata = NMXExperimentMetadata( - sc.DataGroup( - { - 'crystal_rotation': crystal_rotation, - 'sample_position': sample_position, - 'source_position': source_position, - 'sample_name': sc.scalar(f['entry/sample/name'][()]), - } - ) + metadatas = base_wf.compute((NMXSampleMetadata, NMXSourceMetadata)) + export_static_metadata_as_nxlauetof( + sample_metadata=metadatas[NMXSampleMetadata], + source_metadata=metadatas[NMXSourceMetadata], + output_file=config.output.output_file, + ) + tof_das = sc.DataGroup() + detector_metas = sc.DataGroup() + for detector_name in detector_names: + cur_wf = base_wf.copy() + cur_wf[NeXusName[snx.NXdetector]] = detector_name + results = cur_wf.compute((TofDetector[SampleRun], NMXDetectorMetadata)) + detector_meta: NMXDetectorMetadata = results[NMXDetectorMetadata] + export_detector_metadata_as_nxlauetof( + detector_metadata=detector_meta, output_file=config.output.output_file ) - display(experiment_metadata) - display("Experiment metadata component:") - for name, component in experiment_metadata.items(): - display(f"{name}: {component}") - - _export_static_metadata_as_nxlauetof( - experiment_metadata=experiment_metadata, - output_file=output_file_path, + detector_metas[detector_name] = detector_meta + # Binning into 1 bin and getting final tof bin edges later. + tof_das[detector_name] = results[TofDetector[SampleRun]].bin(tof=1) + + tof_bin_edges = _finalize_tof_bin_edges(tof_das=tof_das, config=config.workflow) + + monitor_metadata = NMXMonitorMetadata( + tof_bin_coord='tof', + # TODO: Use real monitor data + # Currently NMX simulations or experiments do not have monitors + monitor_histogram=sc.DataArray( + coords={'tof': tof_bin_edges}, + data=sc.ones_like(tof_bin_edges[:-1]), + ), + ) + export_monitor_metadata_as_nxlauetof( + monitor_metadata=monitor_metadata, output_file=config.output.output_file + ) + + # Histogram detector counts + tof_histograms = sc.DataGroup() + for detector_name, tof_da in tof_das.items(): + det_meta: NMXDetectorMetadata = detector_metas[detector_name] + histogram = tof_da.hist(tof=tof_bin_edges) + tof_histograms[detector_name] = histogram + export_reduced_data_as_nxlauetof( + detector_name=det_meta.detector_name, + da=histogram, + output_file=config.output.output_file, + compress_mode=config.output.compression, ) - detector_grs = {} - for det_name, det_group in detector_id_map.items(): - display(f"Processing {det_name}") - if config.inputs.chunk_size_events <= 0: - dg = det_group[()] - else: - # Slice the first chunk for metadata extraction - dg = det_group['event_time_zero', 0 : config.inputs.chunk_size_events] - display("Computing detector positions...") - display(dg := _compute_positions(dg, auto_fix_transformations=True)) - detector = build_detector_desc(det_name, dg) - detector_meta = sc.DataGroup( - { - 'fast_axis': detector.fast_axis, - 'slow_axis': detector.slow_axis, - 'origin_position': sc.vector([0, 0, 0], unit='m'), - 'position': detector.position, - 'detector_shape': sc.scalar( - ( - dg['data'].sizes['x_pixel_offset'], - dg['data'].sizes['y_pixel_offset'], - ) - ), - 'x_pixel_size': detector.step_x, - 'y_pixel_size': detector.step_y, - 'detector_name': sc.scalar(detector.name), - } - ) - _export_detector_metadata_as_nxlauetof( - NMXDetectorMetadata(detector_meta), - output_file=output_file_path, - ) - - da: sc.DataArray = dg['data'] - event_time_offset_unit = da.bins.coords['event_time_offset'].bins.unit - display("Event time offset unit: %s", event_time_offset_unit) - toa_bin_edges = toa_bin_edges.to(unit=event_time_offset_unit, copy=False) - if config.inputs.chunk_size_events <= 0: - counts = da.hist(event_time_offset=toa_bin_edges).rename_dims( - x_pixel_offset='x', y_pixel_offset='y', event_time_offset='t' - ) - counts.coords['t'] = counts.coords['event_time_offset'] - - else: - num_chunks = calculate_number_of_chunks( - det_group, chunk_size=config.inputs.chunk_size_events - ) - display(f"Number of chunks: {num_chunks}") - counts = da.hist(event_time_offset=toa_bin_edges).rename_dims( - x_pixel_offset='x', y_pixel_offset='y', event_time_offset='t' - ) - counts.coords['t'] = counts.coords['event_time_offset'] - for chunk_index in range(1, num_chunks): - cur_chunk = det_group[ - 'event_time_zero', - chunk_index * config.inputs.chunk_size_events : ( - chunk_index + 1 - ) - * config.inputs.chunk_size_events, - ] - display(f"Processing chunk {chunk_index + 1} of {num_chunks}") - cur_chunk = _compute_positions( - cur_chunk, auto_fix_transformations=True - ) - cur_counts = ( - cur_chunk['data'] - .hist(event_time_offset=toa_bin_edges) - .rename_dims( - x_pixel_offset='x', - y_pixel_offset='y', - event_time_offset='t', - ) - ) - cur_counts.coords['t'] = cur_counts.coords['event_time_offset'] - counts += cur_counts - display("Accumulated counts:") - display(counts.sum().data) - - dg = sc.DataGroup( - counts=counts, - detector_shape=detector_meta['detector_shape'], - detector_name=detector_meta['detector_name'], - ) - display("Final data group:") - display(dg) - display("Saving reduced data to Nexus file...") - _export_reduced_data_as_nxlauetof( - dg, - output_file=output_file_path, - compress_counts=( - config.output.compression == Compression.BITSHUFFLE_LZ4 - ), - ) - detector_grs[det_name] = dg - - display("Reduction completed successfully.") - histograms = {name: det_gr['counts'] for name, det_gr in detector_grs.items()} - return sc.DataGroup(histogram=sc.DataGroup(histograms)) + + return sc.DataGroup( + metadata=detector_metas, + histogram=tof_histograms, + lookup_table=base_wf.compute(TimeOfFlightLookupTable), + ) def main() -> None: diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/nexus.py index 1fc5858d..eb233b2d 100644 --- a/packages/essnmx/src/ess/nmx/nexus.py +++ b/packages/essnmx/src/ess/nmx/nexus.py @@ -3,79 +3,31 @@ import io import pathlib import warnings -from functools import wraps from typing import Any import h5py import numpy as np import scipp as sc - -def _fallback_compute_positions(dg: sc.DataGroup) -> sc.DataGroup: - import warnings - - import scippnexus as snx - - warnings.warn( - "Using fallback compute_positions due to empty log entries. " - "This may lead to incorrect results. Please check the data carefully." - "The fallback will replace empty logs with a scalar value of zero.", - UserWarning, - stacklevel=2, - ) - - empty_transformations = [ - transformation - for transformation in dg['depends_on'].transformations.values() - if 'time' in transformation.value.dims - and transformation.sizes['time'] == 0 # empty log - ] - for transformation in empty_transformations: - orig_value = transformation.value - orig_value = sc.scalar(0, unit=orig_value.unit, dtype=orig_value.dtype) - transformation.value = orig_value - return snx.compute_positions(dg, store_transform='transform_matrix') - - -def _compute_positions( - dg: sc.DataGroup, auto_fix_transformations: bool = False -) -> sc.DataGroup: - """Compute positions of the data group from transformations. - - Wraps the `scippnexus.compute_positions` function - and provides a fallback for cases where the transformations - contain empty logs. - - Parameters - ---------- - dg: - Data group containing the transformations and data. - auto_fix_transformations: - If `True`, it will attempt to fix empty transformations. - It will replace them with a scalar value of zero. - It is because adding a time dimension will make it not possible - to compute positions of children due to time-dependent transformations. - - Returns - ------- - : - Data group with computed positions. - - Warnings - -------- - If `auto_fix_transformations` is `True`, it will warn about the fallback - being used due to empty logs or scalar transformations. - This is because the fallback may lead to incorrect results. - - """ - import scippnexus as snx - - try: - return snx.compute_positions(dg, store_transform='transform_matrix') - except ValueError as e: - if auto_fix_transformations: - return _fallback_compute_positions(dg) - raise e +from .configurations import Compression +from .types import ( + NMXDetectorMetadata, + NMXMonitorMetadata, + NMXSampleMetadata, + NMXSourceMetadata, +) + + +def _check_file( + filename: str | pathlib.Path | io.BytesIO, overwrite: bool +) -> pathlib.Path | io.BytesIO: + if isinstance(filename, str | pathlib.Path): + filename = pathlib.Path(filename) + if filename.exists() and not overwrite: + raise FileExistsError( + f"File '{filename}' already exists. Use `overwrite=True` to overwrite." + ) + return filename def _create_dataset_from_string(*, root_entry: h5py.Group, name: str, var: str) -> None: @@ -112,152 +64,29 @@ def _create_dataset_from_var( return dataset -@wraps(_create_dataset_from_var) -def _create_compressed_dataset(*args, **kwargs): - """Create dataset with compression options. - - It will try to use ``bitshuffle`` for compression if available. - Otherwise, it will fall back to ``gzip`` compression. - - [``Bitshuffle/LZ4``](https://github.com/kiyo-masui/bitshuffle) - is used for convenience. - Since ``Dectris`` uses it for their Nexus file compression, - it is compatible with DIALS. - ``Bitshuffle/LZ4`` tends to give similar results to - GZIP and other compression algorithms with better performance. - A naive implementation of bitshuffle/LZ4 compression, - shown in [issue #124](https://github.com/scipp/essnmx/issues/124), - led to 80% file reduction (365 MB vs 1.8 GB). - - """ - try: - import bitshuffle.h5 - - compression_filter = bitshuffle.h5.H5FILTER - default_compression_opts = (0, bitshuffle.h5.H5_COMPRESS_LZ4) - except ImportError: - warnings.warn( - UserWarning( - "Could not find the bitshuffle.h5 module from bitshuffle package. " - "The bitshuffle package is not installed or only partially installed. " - "Exporting to NeXus files with bitshuffle compression is not possible." - ), - stacklevel=2, - ) - compression_filter = "gzip" - default_compression_opts = 4 - - return _create_dataset_from_var( - *args, - **kwargs, - compression=compression_filter, - compression_opts=default_compression_opts, - ) - - -def _create_root_data_entry(file_obj: h5py.File) -> h5py.Group: - nx_entry = file_obj.create_group("NMX_data") - nx_entry.attrs["NX_class"] = "NXentry" - nx_entry.attrs["default"] = "data" - nx_entry.attrs["name"] = "NMX" - nx_entry["name"] = "NMX" - nx_entry["definition"] = "TOFRAW" - return nx_entry - - -def _create_sample_group(data: sc.DataGroup, nx_entry: h5py.Group) -> h5py.Group: - nx_sample = nx_entry.create_group("NXsample") - nx_sample["name"] = data['sample_name'].value - _create_dataset_from_var( - root_entry=nx_sample, - var=data['crystal_rotation'], - name='crystal_rotation', - long_name='crystal rotation in Phi (XYZ)', - ) - return nx_sample - - -def _create_instrument_group(data: sc.DataGroup, nx_entry: h5py.Group) -> h5py.Group: - nx_instrument = nx_entry.create_group("NXinstrument") - nx_instrument.create_dataset("proton_charge", data=data['proton_charge'].values) - - nx_detector_1 = nx_instrument.create_group("detector_1") - # Detector counts - _create_compressed_dataset( - root_entry=nx_detector_1, - name="counts", - var=data['counts'], - ) - # Time of arrival bin edges - _create_dataset_from_var( - root_entry=nx_detector_1, - var=data['counts'].coords['t'], - name="t_bin", - long_name="t_bin TOF (ms)", - ) - # Pixel IDs - _create_compressed_dataset( - root_entry=nx_detector_1, - name="pixel_id", - var=data['counts'].coords['id'], - long_name="pixel ID", - ) - return nx_instrument - - -def _create_detector_group(data: sc.DataGroup, nx_entry: h5py.Group) -> h5py.Group: - nx_detector = nx_entry.create_group("NXdetector") - # Position of the first pixel (lowest ID) in the detector - _create_compressed_dataset( - root_entry=nx_detector, - name="origin", - var=data['origin_position'], - ) - # Fast axis, along where the pixel ID increases by 1 - _create_dataset_from_var( - root_entry=nx_detector, var=data['fast_axis'], name="fast_axis" - ) - # Slow axis, along where the pixel ID increases - # by the number of pixels in the fast axis - _create_dataset_from_var( - root_entry=nx_detector, var=data['slow_axis'], name="slow_axis" - ) - return nx_detector - - -def _create_source_group(data: sc.DataGroup, nx_entry: h5py.Group) -> h5py.Group: - nx_source = nx_entry.create_group("NXsource") - nx_source["name"] = "European Spallation Source" - nx_source["short_name"] = "ESS" - nx_source["type"] = "Spallation Neutron Source" - nx_source["distance"] = sc.norm(data['source_position']).value - nx_source["probe"] = "neutron" - nx_source["target_material"] = "W" - return nx_source - - -def export_as_nexus( - data: sc.DataGroup, output_file: str | pathlib.Path | io.BytesIO -) -> None: - """Export the reduced data to a NeXus file. +def _retrieve_compression_arguments(compress_mode: Compression) -> dict: + if compress_mode == Compression.BITSHUFFLE_LZ4: + try: + import bitshuffle.h5 + + compression_filter = bitshuffle.h5.H5FILTER + compression_opts = (0, bitshuffle.h5.H5_COMPRESS_LZ4) + except ImportError: + warnings.warn( + UserWarning( + "Could not find the bitshuffle.h5 module from bitshuffle package. " + "The bitshuffle package is not installed properly. " + "Trying with gzip compression instead..." + ), + stacklevel=2, + ) + compression_filter = "gzip" + compression_opts = 4 + else: + compression_filter = None + compression_opts = None - Currently exporting step is not expected to be part of sciline pipelines. - """ - warnings.warn( - DeprecationWarning( - "Exporting to custom NeXus format will be deprecated in the near future " - ">=26.12.0. " - "Please use ``export_as_nxlauetof`` instead." - ), - stacklevel=2, - ) - with h5py.File(output_file, "w") as f: - f.attrs["default"] = "NMX_data" - nx_entry = _create_root_data_entry(f) - _create_sample_group(data, nx_entry) - _create_instrument_group(data, nx_entry) - _create_detector_group(data, nx_entry) - _create_source_group(data, nx_entry) + return {"compression": compression_filter, "compression_opts": compression_opts} def _create_lauetof_data_entry(file_obj: h5py.File) -> h5py.Group: @@ -277,7 +106,9 @@ def _add_lauetof_instrument(nx_entry: h5py.Group) -> h5py.Group: return nx_instrument -def _add_lauetof_source_group(dg, nx_instrument: h5py.Group) -> None: +def _add_lauetof_source_group( + source_position: sc.Variable, nx_instrument: h5py.Group +) -> None: nx_source = nx_instrument.create_group("source") nx_source.attrs["NX_class"] = "NXsource" _create_dataset_from_string( @@ -288,106 +119,71 @@ def _add_lauetof_source_group(dg, nx_instrument: h5py.Group) -> None: root_entry=nx_source, name="type", var="Spallation Neutron Source" ) _create_dataset_from_var( - root_entry=nx_source, name="distance", var=sc.norm(dg["source_position"]) + root_entry=nx_source, name="distance", var=sc.norm(source_position) ) # Legacy probe information. _create_dataset_from_string(root_entry=nx_source, name="probe", var="neutron") -def _add_lauetof_detector_group(dg: sc.DataGroup, nx_instrument: h5py.Group) -> None: - nx_detector = nx_instrument.create_group(dg["detector_name"].value) # Detector name - nx_detector.attrs["NX_class"] = "NXdetector" - _create_dataset_from_var( - name="polar_angle", - root_entry=nx_detector, - var=sc.scalar(0, unit='deg'), # TODO: Add real data - ) - _create_dataset_from_var( - name="azimuthal_angle", - root_entry=nx_detector, - var=sc.scalar(0, unit='deg'), # TODO: Add real data - ) - _create_dataset_from_var( - name="x_pixel_size", root_entry=nx_detector, var=dg["x_pixel_size"] - ) - _create_dataset_from_var( - name="y_pixel_size", root_entry=nx_detector, var=dg["y_pixel_size"] - ) +def _add_lauetof_detector_group( + *, + detector_name: str, + x_pixel_size: sc.Variable, + y_pixel_size: sc.Variable, + origin_position: sc.Variable, + fast_axis: sc.Variable, + slow_axis: sc.Variable, + distance: sc.Variable, + polar_angle: sc.Variable, + azimuthal_angle: sc.Variable, + nx_instrument: h5py.Group, +) -> None: + nx_det = nx_instrument.create_group(detector_name) # Detector name + nx_det.attrs["NX_class"] = "NXdetector" + _create_dataset_from_var(name="polar_angle", root_entry=nx_det, var=polar_angle) _create_dataset_from_var( - name="distance", - root_entry=nx_detector, - var=sc.scalar(0, unit='m'), # TODO: Add real data + name="azimuthal_angle", root_entry=nx_det, var=azimuthal_angle ) + _create_dataset_from_var(name="x_pixel_size", root_entry=nx_det, var=x_pixel_size) + _create_dataset_from_var(name="y_pixel_size", root_entry=nx_det, var=y_pixel_size) + _create_dataset_from_var(name="distance", root_entry=nx_det, var=distance) # Legacy geometry information until we have a better way to store it - _create_dataset_from_var( - name="origin", root_entry=nx_detector, var=dg['origin_position'] - ) + _create_dataset_from_var(name="origin", root_entry=nx_det, var=origin_position) # Fast axis, along where the pixel ID increases by 1 - _create_dataset_from_var( - root_entry=nx_detector, var=dg['fast_axis'], name="fast_axis" - ) + _create_dataset_from_var(root_entry=nx_det, name="fast_axis", var=fast_axis) # Slow axis, along where the pixel ID increases # by the number of pixels in the fast axis - _create_dataset_from_var( - root_entry=nx_detector, var=dg['slow_axis'], name="slow_axis" - ) + _create_dataset_from_var(root_entry=nx_det, name="slow_axis", var=slow_axis) -def _add_lauetof_sample_group(dg, nx_entry: h5py.Group) -> None: +def _add_lauetof_sample_group( + *, + crystal_rotation: sc.Variable, + sample_name: str | sc.Variable, + sample_orientation_matrix: sc.Variable, + sample_unit_cell: sc.Variable, + nx_entry: h5py.Group, +) -> None: nx_sample = nx_entry.create_group("sample") nx_sample.attrs["NX_class"] = "NXsample" _create_dataset_from_var( root_entry=nx_sample, - var=dg['crystal_rotation'], + var=crystal_rotation, name='crystal_rotation', long_name='crystal rotation in Phi (XYZ)', ) _create_dataset_from_string( root_entry=nx_sample, name='name', - var=dg['sample_name'].value, + var=sample_name if isinstance(sample_name, str) else sample_name.value, ) _create_dataset_from_var( - name='orientation_matrix', - root_entry=nx_sample, - var=sc.array( - dims=['i', 'j'], - values=[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], - unit="dimensionless", - ), # TODO: Add real data, the sample orientation matrix + name='orientation_matrix', root_entry=nx_sample, var=sample_orientation_matrix ) _create_dataset_from_var( name='unit_cell', root_entry=nx_sample, - var=sc.array( - dims=['i'], - values=[1.0, 1.0, 1.0, 90.0, 90.0, 90.0], - unit="dimensionless", # TODO: Add real data, - # a, b, c, alpha, beta, gamma - ), - ) - - -def _add_lauetof_monitor_group(data: sc.DataGroup, nx_entry: h5py.Group) -> None: - nx_monitor = nx_entry.create_group("control") - nx_monitor.attrs["NX_class"] = "NXmonitor" - _create_dataset_from_string(root_entry=nx_monitor, name='mode', var='monitor') - nx_monitor["preset"] = 0.0 # Check if this is the correct value - data_dset = _create_dataset_from_var( - name='data', - root_entry=nx_monitor, - var=sc.array( - dims=['tof'], values=[1, 1, 1], unit="counts" - ), # TODO: Add real data, bin values - ) - data_dset.attrs["signal"] = 1 - data_dset.attrs["primary"] = 1 - _create_dataset_from_var( - name='time_of_flight', - root_entry=nx_monitor, - var=sc.array( - dims=['tof'], values=[1, 1, 1], unit="s" - ), # TODO: Add real data, bin edges + var=sample_unit_cell, ) @@ -413,8 +209,9 @@ def _add_arbitrary_metadata( ) -def _export_static_metadata_as_nxlauetof( - experiment_metadata, +def export_static_metadata_as_nxlauetof( + sample_metadata: NMXSampleMetadata, + source_metadata: NMXSourceMetadata, output_file: str | pathlib.Path | io.BytesIO, **arbitrary_metadata: sc.Variable, ) -> None: @@ -428,30 +225,70 @@ def _export_static_metadata_as_nxlauetof( Parameters ---------- - experiment_metadata: - Experiment metadata object. + sample_metadata: + Sample metadata object. + source_metadata: + Source metadata object. + monitor_metadata: + Monitor metadata object. output_file: Output file path. arbitrary_metadata: Arbitrary metadata that does not fit into the existing metadata objects. """ + _check_file(output_file, overwrite=True) with h5py.File(output_file, "w") as f: f.attrs["NX_class"] = "NXlauetof" nx_entry = _create_lauetof_data_entry(f) _add_lauetof_definition(nx_entry) - _add_lauetof_sample_group(experiment_metadata, nx_entry) + _add_lauetof_sample_group( + crystal_rotation=sample_metadata.crystal_rotation, + sample_name=sample_metadata.sample_name, + sample_orientation_matrix=sample_metadata.sample_orientation_matrix, + sample_unit_cell=sample_metadata.sample_unit_cell, + nx_entry=nx_entry, + ) nx_instrument = _add_lauetof_instrument(nx_entry) - _add_lauetof_source_group(experiment_metadata, nx_instrument) - # Placeholder for ``monitor`` group - _add_lauetof_monitor_group(experiment_metadata, nx_entry) + _add_lauetof_source_group(source_metadata.source_position, nx_instrument) # Skipping ``NXdata``(name) field with data link # Add arbitrary metadata _add_arbitrary_metadata(nx_entry, **arbitrary_metadata) -def _export_detector_metadata_as_nxlauetof( - *detector_metadatas, +def export_monitor_metadata_as_nxlauetof( + monitor_metadata: NMXMonitorMetadata, + output_file: str | pathlib.Path | io.BytesIO, + append_mode: bool = True, +) -> None: + """Export the detector specific metadata to a NeXus file. + + Since NMX can have arbitrary number of detectors, + this function can take multiple detector metadata objects. + + Parameters + ---------- + monitor_metadata: + Monitor metadata object. + output_file: + Output file path. + + """ + if not append_mode: + raise NotImplementedError("Only append mode is supported for now.") + + with h5py.File(output_file, "r+") as f: + nx_entry = f["entry"] + # Placeholder for ``monitor`` group + _add_lauetof_monitor_group( + tof_bin_coord=monitor_metadata.tof_bin_coord, + monitor_histogram=monitor_metadata.monitor_histogram, + nx_entry=nx_entry, + ) + + +def export_detector_metadata_as_nxlauetof( + detector_metadata: NMXDetectorMetadata, output_file: str | pathlib.Path | io.BytesIO, append_mode: bool = True, ) -> None: @@ -478,27 +315,54 @@ def _export_detector_metadata_as_nxlauetof( nx_instrument = _add_lauetof_instrument(f["entry"]) else: nx_instrument = nx_entry["instrument"] + # Add detector group metadata - for detector_metadata in detector_metadatas: - _add_lauetof_detector_group(detector_metadata, nx_instrument) + _add_lauetof_detector_group( + detector_name=detector_metadata.detector_name, + x_pixel_size=detector_metadata.x_pixel_size, + y_pixel_size=detector_metadata.y_pixel_size, + origin_position=detector_metadata.origin_position, + fast_axis=detector_metadata.fast_axis, + slow_axis=detector_metadata.slow_axis, + distance=detector_metadata.distance, + polar_angle=detector_metadata.polar_angle, + azimuthal_angle=detector_metadata.azimuthal_angle, + nx_instrument=nx_instrument, + ) -def _extract_counts(dg: sc.DataGroup) -> sc.Variable: - counts: sc.DataArray = dg['counts'].data - if 'id' in counts.dims: - num_x, num_y = dg["detector_shape"].value - return sc.fold(counts, dim='id', sizes={'x': num_x, 'y': num_y}) - else: - # If there is no 'id' dimension, we assume it is already in the correct shape - return counts +def _add_lauetof_monitor_group( + *, + tof_bin_coord: str, + monitor_histogram: sc.DataArray, + nx_entry: h5py.Group, +) -> None: + nx_monitor = nx_entry.create_group("control") + nx_monitor.attrs["NX_class"] = "NXmonitor" + _create_dataset_from_string(root_entry=nx_monitor, name='mode', var='monitor') + nx_monitor["preset"] = 0.0 # Check if this is the correct value + data_dset = _create_dataset_from_var( + name='data', + root_entry=nx_monitor, + var=monitor_histogram.data, + ) + data_dset.attrs["signal"] = 1 + data_dset.attrs["primary"] = 1 + + _create_dataset_from_var( + name='time_of_flight', + root_entry=nx_monitor, + var=monitor_histogram.coords[tof_bin_coord], + ) -def _export_reduced_data_as_nxlauetof( - dg, +def export_reduced_data_as_nxlauetof( + detector_name: str, + da: sc.DataArray, output_file: str | pathlib.Path | io.BytesIO, *, append_mode: bool = True, - compress_counts: bool = True, + compress_mode: Compression = Compression.BITSHUFFLE_LZ4, ) -> None: """Export the reduced data to a NeXus file with the LAUE_TOF application definition. @@ -527,29 +391,30 @@ def _export_reduced_data_as_nxlauetof( raise NotImplementedError("Only append mode is supported for now.") with h5py.File(output_file, "r+") as f: - nx_detector: h5py.Group = f[f"entry/instrument/{dg['detector_name'].value}"] + nx_detector: h5py.Group = f[f"entry/instrument/{detector_name}"] # Data - shape: [n_x_pixels, n_y_pixels, n_tof_bins] # The actual application definition defines it as integer, - # but we keep the original data type for now - num_x, num_y = dg["detector_shape"].value # Probably better way to do this - if compress_counts: - data_dset = _create_compressed_dataset( + # so we overwrite the dtype here. + num_x, num_y = da.sizes['x_pixel_offset'], da.sizes['y_pixel_offset'] + + if compress_mode != Compression.NONE: + compression_args = _retrieve_compression_arguments(compress_mode) + data_dset = _create_dataset_from_var( name="data", root_entry=nx_detector, - var=_extract_counts(dg), - chunks=(num_x, num_y, 1), + var=da.data, + chunks=(num_x, num_y, 1), # Chunk along tof axis dtype=np.uint, + **compression_args, ) else: data_dset = _create_dataset_from_var( - name="data", - root_entry=nx_detector, - var=_extract_counts(dg), - dtype=np.uint, + name="data", root_entry=nx_detector, var=da.data, dtype=np.uint ) + data_dset.attrs["signal"] = 1 _create_dataset_from_var( name='time_of_flight', root_entry=nx_detector, - var=sc.midpoints(dg['counts'].coords['t'], dim='t'), + var=sc.midpoints(da.coords['tof'], dim='tof'), ) diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index ad4d3de5..0336ec7c 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -1,4 +1,8 @@ import enum +from dataclasses import dataclass, field +from typing import NewType + +import scipp as sc class Compression(enum.StrEnum): @@ -9,3 +13,67 @@ class Compression(enum.StrEnum): NONE = 'NONE' BITSHUFFLE_LZ4 = 'BITSHUFFLE_LZ4' + + +TofSimulationMinWavelength = NewType("TofSimulationMinWavelength", sc.Variable) +"""Minimum wavelength for tof simulation to calculate look up table.""" + +TofSimulationMaxWavelength = NewType("TofSimulationMaxWavelength", sc.Variable) +"""Maximum wavelength for tof simulation to calculate look up table.""" + + +@dataclass(kw_only=True) +class NMXSampleMetadata: + crystal_rotation: sc.Variable + sample_position: sc.Variable + sample_name: sc.Variable | str + # Temporarily hardcoding some values + # TODO: Remove hardcoded values + sample_orientation_matrix: sc.Variable = field( + default_factory=lambda: sc.array( + dims=['i', 'j'], + values=[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], + unit="dimensionless", + ) + ) + sample_unit_cell: sc.Variable = field( + default_factory=lambda: sc.array( + dims=['i'], + values=[1.0, 1.0, 1.0, 90.0, 90.0, 90.0], + unit="dimensionless", # TODO: Add real data, + # a, b, c, alpha, beta, gamma + ) + ) + + +@dataclass(kw_only=True) +class NMXSourceMetadata: + source_position: sc.Variable + + +@dataclass(kw_only=True) +class NMXMonitorMetadata: + monitor_histogram: sc.DataArray + tof_bin_coord: str = field( + default='tof', + metadata={ + "description": "Name of the time-of-flight coordinate " + "in the monitor histogram." + }, + ) + + +@dataclass(kw_only=True) +class NMXDetectorMetadata: + detector_name: str + x_pixel_size: sc.Variable + y_pixel_size: sc.Variable + origin_position: sc.Variable + fast_axis: sc.Variable + slow_axis: sc.Variable + distance: sc.Variable + # TODO: Remove hardcoded values + polar_angle: sc.Variable = field(default_factory=lambda: sc.scalar(0, unit='deg')) + azimuthal_angle: sc.Variable = field( + default_factory=lambda: sc.scalar(0, unit='deg') + ) diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py new file mode 100644 index 00000000..d37517f3 --- /dev/null +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -0,0 +1,308 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2025 Scipp contributors (https://github.com/scipp) +import pathlib +from collections.abc import Callable, Iterable + +import pandas as pd +import sciline +import scipp as sc +import scippnexus as snx +import tof + +from ess.reduce.nexus.types import ( + EmptyDetector, + Filename, + NeXusComponent, + NeXusName, + NeXusTransformation, + Position, + SampleRun, +) +from ess.reduce.time_of_flight import ( + DetectorLtotal, + GenericTofWorkflow, + LtotalRange, + NumberOfSimulatedNeutrons, + SimulationResults, + SimulationSeed, + TofLookupTableWorkflow, +) +from ess.reduce.time_of_flight.types import ( + TimeOfFlightLookupTable, + TimeOfFlightLookupTableFilename, +) +from ess.reduce.workflow import register_workflow + +from .configurations import WorkflowConfig +from .types import ( + NMXDetectorMetadata, + NMXSampleMetadata, + NMXSourceMetadata, + TofSimulationMaxWavelength, + TofSimulationMinWavelength, +) + +default_parameters = { + TofSimulationMaxWavelength: sc.scalar(3.6, unit='angstrom'), + TofSimulationMinWavelength: sc.scalar(1.8, unit='angstrom'), +} + + +def _validate_mergable_workflow(wf: sciline.Pipeline): + if wf.indices: + raise NotImplementedError("Only flat workflow can be merged.") + + +def _merge_workflows( + base_wf: sciline.Pipeline, merged_wf: sciline.Pipeline +) -> sciline.Pipeline: + _validate_mergable_workflow(base_wf) + _validate_mergable_workflow(merged_wf) + + for key, spec in merged_wf.underlying_graph.nodes.items(): + if 'value' in spec: + base_wf[key] = spec['value'] + elif (provider_spec := spec.get('provider')) is not None: + base_wf.insert(provider_spec.func) + + return base_wf + + +def _simulate_fixed_wavelength_tof( + wmin: TofSimulationMinWavelength, + wmax: TofSimulationMaxWavelength, + ltotal_range: LtotalRange, + neutrons: NumberOfSimulatedNeutrons, + seed: SimulationSeed, +) -> SimulationResults: + """ + Simulate a pulse of neutrons propagating through a chopper cascade using the + ``tof`` package (https://tof.readthedocs.io). + + Parameters + ---------- + """ + source = tof.Source( + facility="ess", neutrons=neutrons, pulses=2, seed=seed, wmax=wmax, wmin=wmin + ) + nmx_det = tof.Detector(distance=max(ltotal_range), name="detector") + model = tof.Model(source=source, choppers=[], detectors=[nmx_det]) + results = model.run() + events = results["detector"].data.squeeze().flatten(to="event") + return SimulationResults( + time_of_arrival=events.coords["toa"], + speed=events.coords["speed"], + wavelength=events.coords["wavelength"], + weight=events.data, + distance=results["detector"].distance, + ) + + +def _ltotal_range(detector_ltotal: DetectorLtotal[SampleRun]) -> LtotalRange: + margin = sc.scalar(0.5, unit='m').to( + unit=detector_ltotal.unit + ) # Hardcoded margin of 50 cm. It's because the detector width is ~50 cm. + ltotal_min = sc.min(detector_ltotal) - margin + ltotal_max = sc.max(detector_ltotal) + margin + return LtotalRange((ltotal_min, ltotal_max)) + + +def patch_workflow_lookup_table_steps(*, wf: sciline.Pipeline) -> sciline.Pipeline: + patched_wf = wf.copy() + + # Use TofLookupTableWorkflow + patched_wf = _merge_workflows(patched_wf, TofLookupTableWorkflow()) + patched_wf.insert(_simulate_fixed_wavelength_tof) + patched_wf.insert(_ltotal_range) + return patched_wf + + +def _merge_panels(*da: sc.DataArray) -> sc.DataArray: + """Merge multiple DataArrays representing different panels into one.""" + merged = sc.concat(da, dim='panel') + return merged + + +def select_detector_names( + *, + input_files: list[pathlib.Path] | None = None, + detector_ids: Iterable[int] = (0, 1, 2), +): + if input_files is not None: + detector_names = [] + # Collect all detector names from input files + for input_file in input_files: + with snx.File(input_file) as nexus_file: + detector_names.extend( + nexus_file['entry/instrument'][snx.NXdetector].keys() + ) + detector_names = sorted(set(detector_names)) + return [detector_names[i_d] for i_d in detector_ids] + else: + return ['detector_panel_0', 'detector_panel_1', 'detector_panel_2'] + + +def map_detector_names( + *, + wf: sciline.Pipeline, + detector_names: Iterable[str], + mapped_type: type, + reduce_func: Callable = _merge_panels, +) -> sciline.Pipeline: + """Map detector indices(`panel`) to detector names in the workflow.""" + detector_name_map = pd.DataFrame({NeXusName[snx.NXdetector]: detector_names}) + detector_name_map.rename_axis(index='panel', inplace=True) + wf[mapped_type] = wf[mapped_type].map(detector_name_map).reduce(func=reduce_func) + return wf + + +def assemble_sample_metadata( + crystal_rotation: Position[snx.NXcrystal, SampleRun], + sample_position: Position[snx.NXsample, SampleRun], + sample_component: NeXusComponent[snx.NXsample, SampleRun], +) -> NMXSampleMetadata: + """Assemble sample metadata for NMX reduction workflow.""" + return NMXSampleMetadata( + sample_name=sample_component['name'], + crystal_rotation=crystal_rotation, + sample_position=sample_position, + ) + + +def assemble_source_metadata( + source_position: Position[snx.NXsource, SampleRun], +) -> NMXSourceMetadata: + """Assemble source metadata for NMX reduction workflow.""" + return NMXSourceMetadata(source_position=source_position) + + +def _decide_fast_axis(da: sc.DataArray) -> str: + x_slice = da['x_pixel_offset', 0].coords['detector_number'] + y_slice = da['y_pixel_offset', 0].coords['detector_number'] + + if (x_slice.max() < y_slice.max()).value: + return 'y' + elif (x_slice.max() > y_slice.max()).value: + return 'x' + else: + raise ValueError( + "Cannot decide fast axis based on pixel offsets. " + "Please specify the fast axis explicitly." + ) + + +def _decide_step(offsets: sc.Variable) -> sc.Variable: + """Decide the step size based on the offsets assuming at least 2 values.""" + sorted_offsets = sc.sort(offsets, key=offsets.dim, order='ascending') + return sorted_offsets[1] - sorted_offsets[0] + + +def _normalize_vector(vec: sc.Variable) -> sc.Variable: + return vec / sc.norm(vec) + + +def _retrieve_crystal_rotation( + file_path: Filename[SampleRun], +) -> Position[snx.NXcrystal, SampleRun]: + """Temporary provider to retrieve crystal rotation from Nexus file.""" + with snx.File(file_path) as file: + if 'crystal_rotation' not in file['entry/sample']: + import warnings + + warnings.warn( + "No crystal rotation found in the Nexus file under " + "'entry/sample/crystal_rotation'. Returning zero rotation.", + RuntimeWarning, + stacklevel=1, + ) + return Position[snx.NXcrystal, SampleRun](sc.vector([0, 0, 0], unit='deg')) + + # Temporary way of storing crystal rotation. + # streaming-sample-mcstas module writes crystal rotation under + # 'entry/sample/crystal_rotation' as an array of three values. + return Position[snx.NXcrystal, SampleRun]( + file['entry/sample/crystal_rotation'][()] + ) + + +def assemble_detector_metadata( + detector_component: NeXusComponent[snx.NXdetector, SampleRun], + transformation: NeXusTransformation[snx.NXdetector, SampleRun], + source_position: Position[snx.NXsource, SampleRun], + empty_detector: EmptyDetector[SampleRun], +) -> NMXDetectorMetadata: + """Assemble detector metadata for NMX reduction workflow.""" + first_id = empty_detector.coords['detector_number'].min() + # Assuming `empty_detector` has (`x_pixel_offset`, `y_pixel_offset`) dims + origin = empty_detector.flatten(dims=empty_detector.dims, to='detector_number')[ + 'detector_number', first_id + ].coords['position'] + _fast_axis = _decide_fast_axis(empty_detector) + t_unit = transformation.value.unit + + fast_axis_vector = transformation.value * ( + sc.vector([1.0, 0, 0], unit=t_unit) + if _fast_axis == 'x' + else sc.vector([0.0, 1, 0], unit=t_unit) + ) + slow_axis_vector = transformation.value * ( + sc.vector([0.0, 1, 0], unit=t_unit) + if _fast_axis == 'x' + else sc.vector([1.0, 0, 0], unit=t_unit) + ) + x_pixel_size = _decide_step(empty_detector.coords['x_pixel_offset']) + y_pixel_size = _decide_step(empty_detector.coords['y_pixel_offset']) + distance = sc.norm(origin - source_position.to(unit=origin.unit)) + + return NMXDetectorMetadata( + detector_name=detector_component['nexus_component_name'], + x_pixel_size=x_pixel_size, + y_pixel_size=y_pixel_size, + origin_position=origin, + fast_axis=_normalize_vector(fast_axis_vector), + slow_axis=_normalize_vector(slow_axis_vector), + distance=distance, + ) + + +@register_workflow +def NMXWorkflow() -> sciline.Pipeline: + generic_wf = GenericTofWorkflow(run_types=[SampleRun], monitor_types=[]) + + generic_wf.insert(_retrieve_crystal_rotation) + generic_wf.insert(assemble_sample_metadata) + generic_wf.insert(assemble_source_metadata) + generic_wf.insert(assemble_detector_metadata) + for key, value in default_parameters.items(): + generic_wf[key] = value + + return generic_wf + + +def compute_lookup_table( + *, + base_wf: sciline.Pipeline, + workflow_config: WorkflowConfig, + detector_names: Iterable[str], +) -> sc.DataArray: + wf = base_wf.copy() + if workflow_config.tof_lookup_table_file_path is not None: + wf[TimeOfFlightLookupTableFilename] = workflow_config.tof_lookup_table_file_path + else: + wf = patch_workflow_lookup_table_steps(wf=wf) + wmax = sc.scalar(workflow_config.tof_simulation_max_wavelength, unit='angstrom') + wmin = sc.scalar(workflow_config.tof_simulation_min_wavelength, unit='angstrom') + wf[TofSimulationMaxWavelength] = wmax + wf[TofSimulationMinWavelength] = wmin + wf[SimulationSeed] = workflow_config.tof_simulation_seed + wf = map_detector_names( + wf=wf, + detector_names=detector_names, + mapped_type=DetectorLtotal[SampleRun], + reduce_func=_merge_panels, + ) + + return wf.compute(TimeOfFlightLookupTable) + + +__all__ = ['NMXWorkflow'] diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index 6c428792..e3560ac8 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -3,25 +3,25 @@ import pathlib import subprocess +from contextlib import contextmanager from enum import Enum import pydantic import pytest import scipp as sc import scippnexus as snx -from scipp.testing import assert_allclose from ess.nmx._executable_helper import ( InputConfig, OutputConfig, ReductionConfig, - TimeBinCoordinate, - TimeBinUnit, WorkflowConfig, build_reduction_argument_parser, reduction_config_from_args, to_command_arguments, ) +from ess.nmx.configurations import TimeBinCoordinate, TimeBinUnit +from ess.nmx.executables import reduction from ess.nmx.types import Compression @@ -127,9 +127,7 @@ def small_nmx_nexus_path(): return get_small_nmx_nexus() -def _check_output_file( - output_file_path: pathlib.Path, expected_toa_output: sc.Variable -): +def _check_output_file(output_file_path: pathlib.Path, nbins: int): detector_names = [f'detector_panel_{i}' for i in range(3)] with snx.File(output_file_path, 'r') as f: # Test @@ -137,7 +135,7 @@ def _check_output_file( det_gr = f[f'entry/instrument/{name}'] assert det_gr is not None toa_edges = det_gr['time_of_flight'][()] - assert_allclose(toa_edges, expected_toa_output) + assert len(toa_edges) == nbins def test_executable_runs(small_nmx_nexus_path, tmp_path: pathlib.Path): @@ -147,15 +145,6 @@ def test_executable_runs(small_nmx_nexus_path, tmp_path: pathlib.Path): nbins = 20 # Small number of bins for testing. # The output has 1280x1280 pixels per detector per time bin. - expected_toa_bins = sc.linspace( - dim='dim_0', - start=2, # Unrealistic number for testing - stop=int((1 / 15) * 1_000), # Unrealistic number for testing - num=nbins + 1, - unit='ms', - ) - expected_toa_output = sc.midpoints(expected_toa_bins, dim='dim_0').to(unit='ns') - commands = ( 'essnmx-reduce', '--input-file', @@ -164,18 +153,61 @@ def test_executable_runs(small_nmx_nexus_path, tmp_path: pathlib.Path): str(nbins), '--output-file', output_file.as_posix(), - '--min-time-bin', - str(int(expected_toa_bins.min().value)), - '--max-time-bin', - str(int(expected_toa_bins.max().value)), ) # Validate that all commands are strings and contain no unsafe characters result = subprocess.run( # noqa: S603 - We are not accepting arbitrary input here. - commands, - text=True, - capture_output=True, - check=False, + commands, text=True, capture_output=True, check=False ) assert result.returncode == 0 assert output_file.exists() - _check_output_file(output_file, expected_toa_output=expected_toa_output) + _check_output_file(output_file, nbins=nbins) + + +@contextmanager +def known_warnings(): + with pytest.warns(RuntimeWarning, match="No crystal rotation*"): + yield + + +@pytest.fixture +def temp_output_file(tmp_path: pathlib.Path): + output_file_path = tmp_path / "scipp_output.h5" + yield output_file_path + if output_file_path.exists(): + output_file_path.unlink() + + +@pytest.fixture +def reduction_config( + small_nmx_nexus_path: pathlib.Path, temp_output_file: pathlib.Path +) -> ReductionConfig: + input_config = InputConfig(input_file=[small_nmx_nexus_path.as_posix()]) + # Compression option is not default (NONE) but + # the actual default compression option, BITSHUFFLE_LZ4, + # only properly works in linux so we set it to NONE here + # for convenience of testing on all platforms. + output_config = OutputConfig( + output_file=temp_output_file.as_posix(), compression=Compression.NONE + ) + return ReductionConfig(inputs=input_config, output=output_config) + + +def _retrieve_one_hist(results: sc.DataGroup) -> sc.DataArray: + """Helper to retrieve the first DataArray from the results dictionary.""" + return results['histogram']['detector_panel_0'] + + +def test_reduction_default_settings(reduction_config: ReductionConfig) -> None: + # Only check if reduction runs without errors with default settings. + with known_warnings(): + reduction(config=reduction_config) + + +def test_reduction_only_number_of_time_bins(reduction_config: ReductionConfig) -> None: + reduction_config.workflow.nbins = 20 + reduction_config.workflow.time_bin_coordinate = TimeBinCoordinate.time_of_flight + with known_warnings(): + hist = _retrieve_one_hist(reduction(config=reduction_config)) + + # Check that the number of time bins is as expected. + assert len(hist.coords['tof']) == 21 # nbins + 1 edges From 41d12cf8d4e2f8b0429c33e2b005b28781007238 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 4 Dec 2025 14:21:24 +0100 Subject: [PATCH 309/403] Update input file pattern retrieval routine. --- packages/essnmx/src/ess/nmx/executables.py | 25 +++++++++++++--------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 2d83cb42..4cd16d7e 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -33,16 +33,23 @@ from .workflows import NMXWorkflow, compute_lookup_table, select_detector_names -def _retrieve_input_file(input_file: list[pathlib.Path] | pathlib.Path) -> pathlib.Path: +def _retrieve_input_file(input_file: list[str]) -> pathlib.Path: """Temporary helper to retrieve a single input file from the list Until multiple input file support is implemented. """ - if isinstance(input_file, list) and len(input_file) != 1: - raise NotImplementedError( - "Currently, only a single input file is supported for reduction." - ) - elif isinstance(input_file, list): - input_file_path = input_file[0] + if isinstance(input_file, list): + input_files = collect_matching_input_files(*input_file) + if len(input_files) == 0: + raise ValueError( + "No input files found for reduction." + "Check if the file paths are correct.", + input_file, + ) + elif len(input_files) > 1: + raise NotImplementedError( + "Currently, only a single input file is supported for reduction." + ) + input_file_path = input_files[0] else: input_file_path = input_file @@ -132,9 +139,7 @@ def reduction( """ display = _retrieve_display(logger, display) - input_file_path = _retrieve_input_file( - collect_matching_input_files(*config.inputs.input_file) - ).resolve() + input_file_path = _retrieve_input_file(config.inputs.input_file).resolve() display(f"Input file: {input_file_path}") output_file_path = pathlib.Path(config.output.output_file).resolve() From edae5031550f50cfa71ceaa53592d3104e2c1bc0 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:58:50 +0100 Subject: [PATCH 310/403] Fix origin of detector. --- packages/essnmx/src/ess/nmx/workflows.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index d37517f3..a7946238 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -232,11 +232,8 @@ def assemble_detector_metadata( empty_detector: EmptyDetector[SampleRun], ) -> NMXDetectorMetadata: """Assemble detector metadata for NMX reduction workflow.""" - first_id = empty_detector.coords['detector_number'].min() - # Assuming `empty_detector` has (`x_pixel_offset`, `y_pixel_offset`) dims - origin = empty_detector.flatten(dims=empty_detector.dims, to='detector_number')[ - 'detector_number', first_id - ].coords['position'] + # Origin should be the center of the detector. + origin = empty_detector.coords['position'].mean() _fast_axis = _decide_fast_axis(empty_detector) t_unit = transformation.value.unit From 9c12ccfab77e672dfa94c435af3446ee67acc942 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:12:23 +0100 Subject: [PATCH 311/403] Calculate fast/slow axis. --- packages/essnmx/src/ess/nmx/workflows.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index a7946238..d0af71c2 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -232,21 +232,20 @@ def assemble_detector_metadata( empty_detector: EmptyDetector[SampleRun], ) -> NMXDetectorMetadata: """Assemble detector metadata for NMX reduction workflow.""" + positions = empty_detector.coords['position'] # Origin should be the center of the detector. - origin = empty_detector.coords['position'].mean() + origin = positions.mean() _fast_axis = _decide_fast_axis(empty_detector) + _slow_axis = 'y' if _fast_axis == 'x' else 'x' t_unit = transformation.value.unit - fast_axis_vector = transformation.value * ( - sc.vector([1.0, 0, 0], unit=t_unit) - if _fast_axis == 'x' - else sc.vector([0.0, 1, 0], unit=t_unit) - ) - slow_axis_vector = transformation.value * ( - sc.vector([0.0, 1, 0], unit=t_unit) - if _fast_axis == 'x' - else sc.vector([1.0, 0, 0], unit=t_unit) - ) + axis_vectors = { + 'x': positions['x_pixel_offset', -1] - positions['x_pixel_offset', 0], + 'y': positions['y_pixel_offset', -1] - positions['y_pixel_offset', 0], + } + + fast_axis_vector = axis_vectors[_fast_axis].to(unit=t_unit) + slow_axis_vector = axis_vectors[_slow_axis].to(unit=t_unit) x_pixel_size = _decide_step(empty_detector.coords['x_pixel_offset']) y_pixel_size = _decide_step(empty_detector.coords['y_pixel_offset']) distance = sc.norm(origin - source_position.to(unit=origin.unit)) From 274433bb7d622b60cec5863ec0e44592313f6263 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Fri, 5 Dec 2025 15:14:24 +0100 Subject: [PATCH 312/403] Calculate fast/slow axis. --- packages/essnmx/src/ess/nmx/workflows.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index d0af71c2..4e0ec129 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -240,8 +240,10 @@ def assemble_detector_metadata( t_unit = transformation.value.unit axis_vectors = { - 'x': positions['x_pixel_offset', -1] - positions['x_pixel_offset', 0], - 'y': positions['y_pixel_offset', -1] - positions['y_pixel_offset', 0], + 'x': positions['x_pixel_offset', 1]['y_pixel_offset', 0] + - positions['x_pixel_offset', 0]['y_pixel_offset', 0], + 'y': positions['y_pixel_offset', 1]['x_pixel_offset', 0] + - positions['y_pixel_offset', 0]['x_pixel_offset', 0], } fast_axis_vector = axis_vectors[_fast_axis].to(unit=t_unit) From 57dc49bcfda321701b77a733c988a917dd32d294 Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 9 Dec 2025 16:38:38 +0100 Subject: [PATCH 313/403] Update configuration option titles. Co-authored-by: jokasimr --- packages/essnmx/src/ess/nmx/configurations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/configurations.py b/packages/essnmx/src/ess/nmx/configurations.py index 87e1ce97..3ce97737 100644 --- a/packages/essnmx/src/ess/nmx/configurations.py +++ b/packages/essnmx/src/ess/nmx/configurations.py @@ -78,12 +78,12 @@ class WorkflowConfig(BaseModel): default=50, ) min_time_bin: int | None = Field( - title="Minimum Time Bin", + title="Minimum Time", description="Minimum time edge of [time_bin_coordinate] in [time_bin_unit].", default=None, ) max_time_bin: int | None = Field( - title="Maximum Time Bin", + title="Maximum Time", description="Maximum time edge of [time_bin_coordinate] in [time_bin_unit].", default=None, ) From c2a9201385adb0ed4dd49a490e994f612ae55c67 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 9 Dec 2025 17:02:44 +0100 Subject: [PATCH 314/403] Use location spec to retrieve transformation vector of crystal. --- packages/essnmx/src/ess/nmx/workflows.py | 39 +++++++++++++----------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index 4e0ec129..169c183c 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -205,24 +205,29 @@ def _retrieve_crystal_rotation( file_path: Filename[SampleRun], ) -> Position[snx.NXcrystal, SampleRun]: """Temporary provider to retrieve crystal rotation from Nexus file.""" - with snx.File(file_path) as file: - if 'crystal_rotation' not in file['entry/sample']: - import warnings - - warnings.warn( - "No crystal rotation found in the Nexus file under " - "'entry/sample/crystal_rotation'. Returning zero rotation.", - RuntimeWarning, - stacklevel=1, - ) - return Position[snx.NXcrystal, SampleRun](sc.vector([0, 0, 0], unit='deg')) - - # Temporary way of storing crystal rotation. - # streaming-sample-mcstas module writes crystal rotation under - # 'entry/sample/crystal_rotation' as an array of three values. - return Position[snx.NXcrystal, SampleRun]( - file['entry/sample/crystal_rotation'][()] + from ess.reduce.nexus._nexus_loader import load_from_path + from ess.reduce.nexus.types import NeXusLocationSpec + + spec = NeXusLocationSpec( + filename=file_path, + component_name='sample/crystal_rotation', + ) + try: + rotation: snx.nxtransformations.Transform = load_from_path(location=spec) + except KeyError: + import warnings + + warnings.warn( + "No crystal rotation found in the Nexus file under " + f"'entry/{spec.component_name}'. Returning zero rotation.", + RuntimeWarning, + stacklevel=1, ) + zero_rotation = sc.vector([0, 0, 0], unit='deg') + return Position[snx.NXcrystal, SampleRun](zero_rotation) + else: + # TODO: Make sure if retrieving rotation vector is enough here. + return Position[snx.NXcrystal, SampleRun](rotation.vector) def assemble_detector_metadata( From 26606ce541894fd34a90d82d454c4c1427d83f00 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 9 Dec 2025 17:18:15 +0100 Subject: [PATCH 315/403] Ltotal configurable instead of reading them from files. --- packages/essnmx/src/ess/nmx/configurations.py | 10 +++ packages/essnmx/src/ess/nmx/executables.py | 28 +++----- packages/essnmx/src/ess/nmx/workflows.py | 64 ++++--------------- packages/essnmx/tests/executable_test.py | 2 + 4 files changed, 33 insertions(+), 71 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/configurations.py b/packages/essnmx/src/ess/nmx/configurations.py index 3ce97737..dbfa65a5 100644 --- a/packages/essnmx/src/ess/nmx/configurations.py +++ b/packages/essnmx/src/ess/nmx/configurations.py @@ -108,6 +108,16 @@ class WorkflowConfig(BaseModel): description="Maximum wavelength for TOF simulation in Angstrom.", default=3.6, ) + tof_simulation_min_ltotal: float = Field( + title="TOF Simulation Minimum Ltotal", + description="Minimum total flight path for TOF simulation in meters.", + default=150.0, + ) + tof_simulation_max_ltotal: float = Field( + title="TOF Simulation Maximum Ltotal", + description="Maximum total flight path for TOF simulation in meters.", + default=170.0, + ) tof_simulation_seed: int = Field( title="TOF Simulation Seed", description="Random seed for TOF simulation.", diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 4cd16d7e..a10a2363 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -2,7 +2,7 @@ # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) import logging import pathlib -from collections.abc import Callable, Iterable +from collections.abc import Callable import sciline as sl import scipp as sc @@ -71,25 +71,19 @@ def compute_and_cache_lookup_table( *, wf: sl.Pipeline, workflow_config: WorkflowConfig, - detector_names: Iterable[str], display: Callable, ) -> sl.Pipeline: - """Compute and cache the TOF lookup table in the workflow. - - **Note**: ``base_wf`` is modified in-place and also returned. - """ - # We compute one lookup table that covers all range - # to avoid multiple tof simulations. + """Compute and cache the TOF lookup table in the workflow.""" if workflow_config.tof_lookup_table_file_path is None: display("Computing TOF lookup table from simulation...") else: display("Loading TOF lookup table from file...") - lookup_table = compute_lookup_table( - base_wf=wf, workflow_config=workflow_config, detector_names=detector_names + lookup_table_cached_wf = wf.copy() + lookup_table_cached_wf[TimeOfFlightLookupTable] = compute_lookup_table( + workflow_config=workflow_config ) - wf[TimeOfFlightLookupTable] = lookup_table - return wf + return lookup_table_cached_wf def _finalize_tof_bin_edges( @@ -150,15 +144,13 @@ def reduction( ) base_wf = NMXWorkflow() - # Insert input file path into the workflow for later use - base_wf[Filename] = input_file_path + # Insert parameters and cache intermediate results + base_wf[Filename] = input_file_path base_wf = compute_and_cache_lookup_table( - wf=base_wf, - workflow_config=config.workflow, - detector_names=detector_names, - display=display, + wf=base_wf, workflow_config=config.workflow, display=display ) + metadatas = base_wf.compute((NMXSampleMetadata, NMXSourceMetadata)) export_static_metadata_as_nxlauetof( sample_metadata=metadatas[NMXSampleMetadata], diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index 169c183c..be23d3b8 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -19,7 +19,6 @@ SampleRun, ) from ess.reduce.time_of_flight import ( - DetectorLtotal, GenericTofWorkflow, LtotalRange, NumberOfSimulatedNeutrons, @@ -48,26 +47,6 @@ } -def _validate_mergable_workflow(wf: sciline.Pipeline): - if wf.indices: - raise NotImplementedError("Only flat workflow can be merged.") - - -def _merge_workflows( - base_wf: sciline.Pipeline, merged_wf: sciline.Pipeline -) -> sciline.Pipeline: - _validate_mergable_workflow(base_wf) - _validate_mergable_workflow(merged_wf) - - for key, spec in merged_wf.underlying_graph.nodes.items(): - if 'value' in spec: - base_wf[key] = spec['value'] - elif (provider_spec := spec.get('provider')) is not None: - base_wf.insert(provider_spec.func) - - return base_wf - - def _simulate_fixed_wavelength_tof( wmin: TofSimulationMinWavelength, wmax: TofSimulationMaxWavelength, @@ -98,25 +77,6 @@ def _simulate_fixed_wavelength_tof( ) -def _ltotal_range(detector_ltotal: DetectorLtotal[SampleRun]) -> LtotalRange: - margin = sc.scalar(0.5, unit='m').to( - unit=detector_ltotal.unit - ) # Hardcoded margin of 50 cm. It's because the detector width is ~50 cm. - ltotal_min = sc.min(detector_ltotal) - margin - ltotal_max = sc.max(detector_ltotal) + margin - return LtotalRange((ltotal_min, ltotal_max)) - - -def patch_workflow_lookup_table_steps(*, wf: sciline.Pipeline) -> sciline.Pipeline: - patched_wf = wf.copy() - - # Use TofLookupTableWorkflow - patched_wf = _merge_workflows(patched_wf, TofLookupTableWorkflow()) - patched_wf.insert(_simulate_fixed_wavelength_tof) - patched_wf.insert(_ltotal_range) - return patched_wf - - def _merge_panels(*da: sc.DataArray) -> sc.DataArray: """Merge multiple DataArrays representing different panels into one.""" merged = sc.concat(da, dim='panel') @@ -282,28 +242,26 @@ def NMXWorkflow() -> sciline.Pipeline: return generic_wf -def compute_lookup_table( - *, - base_wf: sciline.Pipeline, - workflow_config: WorkflowConfig, - detector_names: Iterable[str], -) -> sc.DataArray: - wf = base_wf.copy() +def compute_lookup_table(*, workflow_config: WorkflowConfig) -> sc.DataArray: if workflow_config.tof_lookup_table_file_path is not None: + wf = NMXWorkflow() wf[TimeOfFlightLookupTableFilename] = workflow_config.tof_lookup_table_file_path else: - wf = patch_workflow_lookup_table_steps(wf=wf) + wf = TofLookupTableWorkflow() + wf.insert(_simulate_fixed_wavelength_tof) + wmax = sc.scalar(workflow_config.tof_simulation_max_wavelength, unit='angstrom') wmin = sc.scalar(workflow_config.tof_simulation_min_wavelength, unit='angstrom') wf[TofSimulationMaxWavelength] = wmax wf[TofSimulationMinWavelength] = wmin wf[SimulationSeed] = workflow_config.tof_simulation_seed - wf = map_detector_names( - wf=wf, - detector_names=detector_names, - mapped_type=DetectorLtotal[SampleRun], - reduce_func=_merge_panels, + ltotal_min = sc.scalar( + value=workflow_config.tof_simulation_min_ltotal, unit='m' + ) + ltotal_max = sc.scalar( + value=workflow_config.tof_simulation_max_ltotal, unit='m' ) + wf[LtotalRange] = LtotalRange((ltotal_min, ltotal_max)) return wf.compute(TimeOfFlightLookupTable) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index e3560ac8..d385d13b 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -95,6 +95,8 @@ def test_reduction_config() -> None: time_bin_unit=TimeBinUnit.us, tof_simulation_max_wavelength=5.0, tof_simulation_min_wavelength=1.0, + tof_simulation_min_ltotal=140.0, + tof_simulation_max_ltotal=200.0, tof_simulation_seed=12345, ) output_options = OutputConfig( From 90c8339bc139a37ff950e7b006581e5461b326b3 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 10 Dec 2025 15:18:50 +0100 Subject: [PATCH 316/403] Hardcode detector names in the workflow. --- packages/essnmx/src/ess/nmx/executables.py | 4 +-- packages/essnmx/src/ess/nmx/workflows.py | 33 +++++++++++----------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index a10a2363..45cf1577 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -139,9 +139,7 @@ def reduction( output_file_path = pathlib.Path(config.output.output_file).resolve() display(f"Output file: {output_file_path}") - detector_names = select_detector_names( - input_files=[input_file_path], detector_ids=config.inputs.detector_ids - ) + detector_names = select_detector_names(detector_ids=config.inputs.detector_ids) base_wf = NMXWorkflow() diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index be23d3b8..127b13d1 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) -import pathlib from collections.abc import Callable, Iterable import pandas as pd @@ -83,23 +82,23 @@ def _merge_panels(*da: sc.DataArray) -> sc.DataArray: return merged -def select_detector_names( - *, - input_files: list[pathlib.Path] | None = None, - detector_ids: Iterable[int] = (0, 1, 2), -): - if input_files is not None: - detector_names = [] - # Collect all detector names from input files - for input_file in input_files: - with snx.File(input_file) as nexus_file: - detector_names.extend( - nexus_file['entry/instrument'][snx.NXdetector].keys() - ) - detector_names = sorted(set(detector_names)) - return [detector_names[i_d] for i_d in detector_ids] +def select_detector_names(*, detector_ids: Iterable[int] = (0, 1, 2)): + import os + + # Users can override detector names via environment variable + # It is a comma-separated list of detector names + # e.g., NMX_DETECTOR_NAMES=detector_panel_0,detector_panel_1,detector_panel_2 + # The detector names are not expected to be changed from the default ones, + # but this option is provided for minimum flexibility. + DETECTOR_NAME_VAR = os.environ.get("NMX_DETECTOR_NAMES", None) + if DETECTOR_NAME_VAR is not None: + return tuple( + name + for i_name, name in enumerate(DETECTOR_NAME_VAR.split(',')) + if i_name in detector_ids + ) else: - return ['detector_panel_0', 'detector_panel_1', 'detector_panel_2'] + return tuple(f'detector_panel_{i}' for i in detector_ids) def map_detector_names( From fac8dde24139481e4810ddf7e74cddbace8fe269 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 10 Dec 2025 16:26:10 +0100 Subject: [PATCH 317/403] Initialize workflow using workflow config object in the earlier step. --- packages/essnmx/src/ess/nmx/executables.py | 32 ++-------- packages/essnmx/src/ess/nmx/workflows.py | 69 +++++++++++++++------- 2 files changed, 54 insertions(+), 47 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 45cf1577..94b7f550 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -4,7 +4,6 @@ import pathlib from collections.abc import Callable -import sciline as sl import scipp as sc import scippnexus as snx @@ -30,7 +29,7 @@ NMXSampleMetadata, NMXSourceMetadata, ) -from .workflows import NMXWorkflow, compute_lookup_table, select_detector_names +from .workflows import initialize_nmx_workflow, select_detector_names def _retrieve_input_file(input_file: list[str]) -> pathlib.Path: @@ -67,25 +66,6 @@ def _retrieve_display( return logging.getLogger(__name__).info -def compute_and_cache_lookup_table( - *, - wf: sl.Pipeline, - workflow_config: WorkflowConfig, - display: Callable, -) -> sl.Pipeline: - """Compute and cache the TOF lookup table in the workflow.""" - if workflow_config.tof_lookup_table_file_path is None: - display("Computing TOF lookup table from simulation...") - else: - display("Loading TOF lookup table from file...") - - lookup_table_cached_wf = wf.copy() - lookup_table_cached_wf[TimeOfFlightLookupTable] = compute_lookup_table( - workflow_config=workflow_config - ) - return lookup_table_cached_wf - - def _finalize_tof_bin_edges( *, tof_das: sc.DataGroup, config: WorkflowConfig ) -> sc.Variable: @@ -141,13 +121,11 @@ def reduction( detector_names = select_detector_names(detector_ids=config.inputs.detector_ids) - base_wf = NMXWorkflow() - + # Initialize workflow + base_wf = initialize_nmx_workflow(config=config.workflow) # Insert parameters and cache intermediate results - base_wf[Filename] = input_file_path - base_wf = compute_and_cache_lookup_table( - wf=base_wf, workflow_config=config.workflow, display=display - ) + base_wf[Filename[SampleRun]] = input_file_path + base_wf[TimeOfFlightLookupTable] = base_wf.compute(TimeOfFlightLookupTable) metadatas = base_wf.compute((NMXSampleMetadata, NMXSourceMetadata)) export_static_metadata_as_nxlauetof( diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index 127b13d1..b56a5df9 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -25,10 +25,7 @@ SimulationSeed, TofLookupTableWorkflow, ) -from ess.reduce.time_of_flight.types import ( - TimeOfFlightLookupTable, - TimeOfFlightLookupTableFilename, -) +from ess.reduce.time_of_flight.types import TimeOfFlightLookupTableFilename from ess.reduce.workflow import register_workflow from .configurations import WorkflowConfig @@ -241,28 +238,60 @@ def NMXWorkflow() -> sciline.Pipeline: return generic_wf -def compute_lookup_table(*, workflow_config: WorkflowConfig) -> sc.DataArray: - if workflow_config.tof_lookup_table_file_path is not None: - wf = NMXWorkflow() - wf[TimeOfFlightLookupTableFilename] = workflow_config.tof_lookup_table_file_path +def _validate_mergable_workflow(wf: sciline.Pipeline): + if wf.indices: + raise NotImplementedError("Only flat workflow can be merged.") + + +def _merge_workflows( + base_wf: sciline.Pipeline, merged_wf: sciline.Pipeline +) -> sciline.Pipeline: + _validate_mergable_workflow(base_wf) + _validate_mergable_workflow(merged_wf) + + for key, spec in merged_wf.underlying_graph.nodes.items(): + if 'value' in spec: + base_wf[key] = spec['value'] + elif (provider_spec := spec.get('provider')) is not None: + base_wf.insert(provider_spec.func) + + return base_wf + + +def initialize_nmx_workflow(*, config: WorkflowConfig) -> sciline.Pipeline: + """Initialize NMX workflow according to the workflow configuration. + + If a TOF lookup table file path is provided in the configuration, + it is used directly. Otherwise, a TOF simulation workflow is added to + the NMX workflow to compute the lookup table on-the-fly. + + All other parameters required for TOF simulation are also set + as parameters in the workflow. + + Parameters + ---------- + config: + Workflow configuration for NMX reduction. + params: + Additional parameters to set in the workflow. + + """ + wf = NMXWorkflow() + if config.tof_lookup_table_file_path is not None: + wf[TimeOfFlightLookupTableFilename] = config.tof_lookup_table_file_path else: - wf = TofLookupTableWorkflow() + wf = _merge_workflows(base_wf=wf, merged_wf=TofLookupTableWorkflow()) wf.insert(_simulate_fixed_wavelength_tof) - - wmax = sc.scalar(workflow_config.tof_simulation_max_wavelength, unit='angstrom') - wmin = sc.scalar(workflow_config.tof_simulation_min_wavelength, unit='angstrom') + wmax = sc.scalar(config.tof_simulation_max_wavelength, unit='angstrom') + wmin = sc.scalar(config.tof_simulation_min_wavelength, unit='angstrom') wf[TofSimulationMaxWavelength] = wmax wf[TofSimulationMinWavelength] = wmin - wf[SimulationSeed] = workflow_config.tof_simulation_seed - ltotal_min = sc.scalar( - value=workflow_config.tof_simulation_min_ltotal, unit='m' - ) - ltotal_max = sc.scalar( - value=workflow_config.tof_simulation_max_ltotal, unit='m' - ) + wf[SimulationSeed] = config.tof_simulation_seed + ltotal_min = sc.scalar(value=config.tof_simulation_min_ltotal, unit='m') + ltotal_max = sc.scalar(value=config.tof_simulation_max_ltotal, unit='m') wf[LtotalRange] = LtotalRange((ltotal_min, ltotal_max)) - return wf.compute(TimeOfFlightLookupTable) + return wf __all__ = ['NMXWorkflow'] From cc6605275678598b0eefbebe2a1b606097fbbeea Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 10 Dec 2025 16:56:16 +0100 Subject: [PATCH 318/403] Remove unused helper function. --- packages/essnmx/src/ess/nmx/workflows.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index b56a5df9..d8aaac43 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) -from collections.abc import Callable, Iterable +from collections.abc import Iterable -import pandas as pd import sciline import scipp as sc import scippnexus as snx @@ -12,7 +11,6 @@ EmptyDetector, Filename, NeXusComponent, - NeXusName, NeXusTransformation, Position, SampleRun, @@ -98,20 +96,6 @@ def select_detector_names(*, detector_ids: Iterable[int] = (0, 1, 2)): return tuple(f'detector_panel_{i}' for i in detector_ids) -def map_detector_names( - *, - wf: sciline.Pipeline, - detector_names: Iterable[str], - mapped_type: type, - reduce_func: Callable = _merge_panels, -) -> sciline.Pipeline: - """Map detector indices(`panel`) to detector names in the workflow.""" - detector_name_map = pd.DataFrame({NeXusName[snx.NXdetector]: detector_names}) - detector_name_map.rename_axis(index='panel', inplace=True) - wf[mapped_type] = wf[mapped_type].map(detector_name_map).reduce(func=reduce_func) - return wf - - def assemble_sample_metadata( crystal_rotation: Position[snx.NXcrystal, SampleRun], sample_position: Position[snx.NXsample, SampleRun], From 6b29d7ebee50ead340771b131763a1f3b2629126 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 10 Dec 2025 17:04:26 +0100 Subject: [PATCH 319/403] Use min/max to find the tof bin edges boundaries. --- packages/essnmx/src/ess/nmx/executables.py | 25 ++++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 94b7f550..c5ef4f61 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -31,6 +31,9 @@ ) from .workflows import initialize_nmx_workflow, select_detector_names +_TOF_COORD_NAME = 'tof' +"""Name of the TOF coordinate used in DataArrays.""" + def _retrieve_input_file(input_file: list[str]) -> pathlib.Path: """Temporary helper to retrieve a single input file from the list @@ -70,10 +73,11 @@ def _finalize_tof_bin_edges( *, tof_das: sc.DataGroup, config: WorkflowConfig ) -> sc.Variable: tof_bin_edges = sc.concat( - tuple(tof_da.coords['tof'] for tof_da in tof_das.values()), dim='tof' + tuple(tof_da.coords[_TOF_COORD_NAME] for tof_da in tof_das.values()), + dim=_TOF_COORD_NAME, ) return sc.linspace( - dim='tof', + dim=_TOF_COORD_NAME, start=sc.min(tof_bin_edges), stop=sc.max(tof_bin_edges), num=config.nbins + 1, @@ -145,16 +149,23 @@ def reduction( ) detector_metas[detector_name] = detector_meta # Binning into 1 bin and getting final tof bin edges later. - tof_das[detector_name] = results[TofDetector[SampleRun]].bin(tof=1) - - tof_bin_edges = _finalize_tof_bin_edges(tof_das=tof_das, config=config.workflow) + tof_das[detector_name] = results[TofDetector[SampleRun]] + + # Make tof bin edges covering all detectors + # TODO: Allow user to specify tof binning parameters from config + min_tof = min(da.bins.coords[_TOF_COORD_NAME].min() for da in tof_das.values()) + max_tof = max(da.bins.coords[_TOF_COORD_NAME].max() for da in tof_das.values()) + n_edges = config.workflow.nbins + 1 + tof_bin_edges = sc.linspace( + dim=_TOF_COORD_NAME, start=min_tof, stop=max_tof, num=n_edges + ) monitor_metadata = NMXMonitorMetadata( - tof_bin_coord='tof', + tof_bin_coord=_TOF_COORD_NAME, # TODO: Use real monitor data # Currently NMX simulations or experiments do not have monitors monitor_histogram=sc.DataArray( - coords={'tof': tof_bin_edges}, + coords={_TOF_COORD_NAME: tof_bin_edges}, data=sc.ones_like(tof_bin_edges[:-1]), ), ) From efbad89d4a538299542c5cf64357dc273edd8f74 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 16 Dec 2025 09:30:56 +0100 Subject: [PATCH 320/403] Remove unused helper function. --- packages/essnmx/src/ess/nmx/executables.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index c5ef4f61..0aed085d 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -16,7 +16,7 @@ collect_matching_input_files, reduction_config_from_args, ) -from .configurations import ReductionConfig, WorkflowConfig +from .configurations import ReductionConfig from .nexus import ( export_detector_metadata_as_nxlauetof, export_monitor_metadata_as_nxlauetof, @@ -69,21 +69,6 @@ def _retrieve_display( return logging.getLogger(__name__).info -def _finalize_tof_bin_edges( - *, tof_das: sc.DataGroup, config: WorkflowConfig -) -> sc.Variable: - tof_bin_edges = sc.concat( - tuple(tof_da.coords[_TOF_COORD_NAME] for tof_da in tof_das.values()), - dim=_TOF_COORD_NAME, - ) - return sc.linspace( - dim=_TOF_COORD_NAME, - start=sc.min(tof_bin_edges), - stop=sc.max(tof_bin_edges), - num=config.nbins + 1, - ) - - def reduction( *, config: ReductionConfig, From a045fdb800f6397ae07c7262fc995271b3c6a454 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:46:43 +0100 Subject: [PATCH 321/403] Expose tof simulation number of neutron configuration. --- packages/essnmx/src/ess/nmx/configurations.py | 5 +++++ packages/essnmx/src/ess/nmx/workflows.py | 7 ++++++- packages/essnmx/tests/executable_test.py | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/configurations.py b/packages/essnmx/src/ess/nmx/configurations.py index dbfa65a5..2e664f96 100644 --- a/packages/essnmx/src/ess/nmx/configurations.py +++ b/packages/essnmx/src/ess/nmx/configurations.py @@ -98,6 +98,11 @@ class WorkflowConfig(BaseModel): "If None, the lookup table will be computed on-the-fly.", default=None, ) + tof_simulation_num_neutrons: int = Field( + title="Number of Neutrons for TOF Simulation", + description="Number of neutrons to simulate for TOF lookup table calculation.", + default=1_000_000, + ) tof_simulation_min_wavelength: float = Field( title="TOF Simulation Minimum Wavelength", description="Minimum wavelength for TOF simulation in Angstrom.", diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index d8aaac43..4bff057d 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -56,7 +56,12 @@ def _simulate_fixed_wavelength_tof( ---------- """ source = tof.Source( - facility="ess", neutrons=neutrons, pulses=2, seed=seed, wmax=wmax, wmin=wmin + facility="ess", + neutrons=neutrons, + pulses=1, # NMX does not use pulse-skipping. + seed=seed, + wmax=wmax, + wmin=wmin, ) nmx_det = tof.Detector(distance=max(ltotal_range), name="detector") model = tof.Model(source=source, choppers=[], detectors=[nmx_det]) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index d385d13b..202b2f8b 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -93,6 +93,7 @@ def test_reduction_config() -> None: max_time_bin=100_000, time_bin_coordinate=TimeBinCoordinate.time_of_flight, time_bin_unit=TimeBinUnit.us, + tof_simulation_num_neutrons=700_000, tof_simulation_max_wavelength=5.0, tof_simulation_min_wavelength=1.0, tof_simulation_min_ltotal=140.0, From f5d7177cac5047e6e66125d3513b5af9f93f553e Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 9 Dec 2025 17:21:02 +0100 Subject: [PATCH 322/403] Use 2 pulses to simulate overlapping pulses. --- packages/essnmx/src/ess/nmx/workflows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index 4bff057d..0a697a01 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -58,7 +58,7 @@ def _simulate_fixed_wavelength_tof( source = tof.Source( facility="ess", neutrons=neutrons, - pulses=1, # NMX does not use pulse-skipping. + pulses=2, seed=seed, wmax=wmax, wmin=wmin, From 21e9123cc236d674c6c3bf52605bd133d31dd033 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:46:43 +0100 Subject: [PATCH 323/403] Expose tof simulation number of neutron configuration. --- packages/essnmx/src/ess/nmx/workflows.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index 0a697a01..db55c1ce 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -58,7 +58,11 @@ def _simulate_fixed_wavelength_tof( source = tof.Source( facility="ess", neutrons=neutrons, +<<<<<<< HEAD pulses=2, +======= + pulses=1, # NMX does not use pulse-skipping. +>>>>>>> 39aeba2 (Expose tof simulation number of neutron configuration.) seed=seed, wmax=wmax, wmin=wmin, From a0f7bd29f4c82e404f1ef6ab9da2b61708fce73d Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:07:35 +0100 Subject: [PATCH 324/403] Add documentation of reduction interface. --- packages/essnmx/docs/user-guide/index.md | 1 + .../essnmx/docs/user-guide/workflow.ipynb | 171 ++++++++++++++++++ .../essnmx/src/ess/nmx/_executable_helper.py | 36 ---- packages/essnmx/src/ess/nmx/configurations.py | 55 +++++- packages/essnmx/src/ess/nmx/executables.py | 61 ++++--- packages/essnmx/src/ess/nmx/nexus.py | 4 +- packages/essnmx/tests/executable_test.py | 15 +- 7 files changed, 278 insertions(+), 65 deletions(-) create mode 100644 packages/essnmx/docs/user-guide/workflow.ipynb diff --git a/packages/essnmx/docs/user-guide/index.md b/packages/essnmx/docs/user-guide/index.md index 550cfb7f..83cdd88e 100644 --- a/packages/essnmx/docs/user-guide/index.md +++ b/packages/essnmx/docs/user-guide/index.md @@ -5,6 +5,7 @@ maxdepth: 1 --- +workflow mcstas_workflow mcstas_workflow_chunk scaling_workflow diff --git a/packages/essnmx/docs/user-guide/workflow.ipynb b/packages/essnmx/docs/user-guide/workflow.ipynb new file mode 100644 index 00000000..bd28d204 --- /dev/null +++ b/packages/essnmx/docs/user-guide/workflow.ipynb @@ -0,0 +1,171 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# NMX Reduction Workflow\n", + "\n", + "> NMX does not expect users to use python interface directly.
\n", + "This documentation is mostly for istrument data scientists or instrument scientists.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## TL;DR" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ess.nmx.executables import reduction\n", + "from ess.nmx.data import get_small_nmx_nexus\n", + "from ess.nmx.configurations import (\n", + " ReductionConfig,\n", + " OutputConfig,\n", + " InputConfig,\n", + " WorkflowConfig,\n", + " TimeBinCoordinate,\n", + ")\n", + "\n", + "# Build Configuration\n", + "config = ReductionConfig(\n", + " inputs=InputConfig(\n", + " input_file=[get_small_nmx_nexus().as_posix()],\n", + " detector_ids=[0, 1, 2], # Detector index to be reduced in alphabetical order.\n", + " ),\n", + " output=OutputConfig(\n", + " output_file=\"scipp_output.hdf\", skip_file_output=False, overwrite=True\n", + " ),\n", + " workflow=WorkflowConfig(\n", + " time_bin_coordinate=TimeBinCoordinate.time_of_flight,\n", + " nbins=10,\n", + " tof_simulation_num_neutrons=1_000_000,\n", + " tof_simulation_min_wavelength=1.8,\n", + " tof_simulation_max_wavelength=3.6,\n", + " tof_simulation_seed=42,\n", + " ),\n", + ")\n", + "\n", + "# Run Reduction\n", + "reduction(config=config, display=display)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Configuration\n", + "\n", + "`essnmx` provide command line data reduction tool for the reduction between `nexus` and `dials`.
\n", + "The `essnmx-reduce` interface will reduce `nexus` file
\n", + "and save the results into `NXlauetof`(not exactly but very close) format for `dials`.
\n", + "\n", + "Argument options could be exhaustive therefore we wrapped them into a nested pydantic model.
\n", + "Here is a python API you can use to build the configuration and turn it into a command line arguments.\n", + "\n", + "**Configuration object is pydantic model so it strictly check the type of the arguments.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ess.nmx.configurations import (\n", + " ReductionConfig,\n", + " OutputConfig,\n", + " InputConfig,\n", + " WorkflowConfig,\n", + " TimeBinCoordinate,\n", + " to_command_arguments,\n", + ")\n", + "\n", + "config = ReductionConfig(\n", + " inputs=InputConfig(\n", + " input_file=[\"PATH_TO_THE_NEXUS_FILE.hdf\"],\n", + " detector_ids=[0, 1, 2], # Detector index to be reduced in alphabetical order.\n", + " ),\n", + " output=OutputConfig(output_file=\"scipp_output.hdf\", skip_file_output=True),\n", + " workflow=WorkflowConfig(\n", + " time_bin_coordinate=TimeBinCoordinate.time_of_flight,\n", + " nbins=10,\n", + " tof_simulation_num_neutrons=1_000_000,\n", + " tof_simulation_min_wavelength=1.8,\n", + " tof_simulation_max_wavelength=3.6,\n", + " tof_simulation_seed=42,\n", + " ),\n", + ")\n", + "\n", + "display(config)\n", + "print(to_command_arguments(config=config, one_line=True))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Reduce Nexus File(s)\n", + "\n", + "`OutputConfig` has an option called `skip_file_output` if you want to reduce the file and use it only on the memory.
\n", + "Then you can use `save_results` function to explicitly save the results." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ess.nmx.executables import reduction\n", + "from ess.nmx.data import get_small_nmx_nexus\n", + "\n", + "config = ReductionConfig(\n", + " inputs=InputConfig(input_file=[get_small_nmx_nexus().as_posix()]),\n", + " output=OutputConfig(skip_file_output=True),\n", + ")\n", + "results = reduction(config=config, display=display)\n", + "results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ess.nmx.executables import save_results\n", + "\n", + "output_config = OutputConfig(output_file=\"scipp_output.hdf\", overwrite=True)\n", + "save_results(results=results, output_config=output_config)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nmx-dev-313", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/packages/essnmx/src/ess/nmx/_executable_helper.py b/packages/essnmx/src/ess/nmx/_executable_helper.py index afa18438..425c494e 100644 --- a/packages/essnmx/src/ess/nmx/_executable_helper.py +++ b/packages/essnmx/src/ess/nmx/_executable_helper.py @@ -174,42 +174,6 @@ def reduction_config_from_args(args: argparse.Namespace) -> ReductionConfig: ) -def to_command_arguments( - config: ReductionConfig, one_line: bool = True -) -> list[str] | str: - """Convert the config to a list of command line arguments. - - Parameters - ---------- - one_line: - If True, return a single string with all arguments joined by spaces. - If False, return a list of argument strings. - - """ - args = {} - for instance in config._children: - args.update(instance.model_dump(mode='python')) - args = {f"--{k.replace('_', '-')}": v for k, v in args.items() if v is not None} - - arg_list = [] - for k, v in args.items(): - if not isinstance(v, bool): - arg_list.append(k) - if isinstance(v, list): - arg_list.extend(str(item) for item in v) - elif isinstance(v, enum.StrEnum): - arg_list.append(v.value) - else: - arg_list.append(str(v)) - elif v is True: - arg_list.append(k) - - if one_line: - return ' '.join(arg_list) - else: - return arg_list - - def build_logger(args: argparse.Namespace | OutputConfig) -> logging.Logger: logger = logging.getLogger(__name__) if args.verbose: diff --git a/packages/essnmx/src/ess/nmx/configurations.py b/packages/essnmx/src/ess/nmx/configurations.py index 2e664f96..2470f6b0 100644 --- a/packages/essnmx/src/ess/nmx/configurations.py +++ b/packages/essnmx/src/ess/nmx/configurations.py @@ -140,11 +140,22 @@ class OutputConfig(BaseModel): default=False, ) # File output + skip_file_output: bool = Field( + title="Skip File Output", + description="If True, the output file will not be written.", + default=False, + ) output_file: str = Field( title="Output File", - description="Path to the output file.", + description="Path to the output file. " + "It will be overwritten if ``overwrite`` is True.", default="scipp_output.h5", ) + overwrite: bool = Field( + title="Overwrite Output File", + description="If True, overwrite the output file if ``output_file`` exists.", + default=False, + ) compression: Compression = Field( title="Compression", description="Compress option of reduced output file.", @@ -162,3 +173,45 @@ class ReductionConfig(BaseModel): @property def _children(self) -> list[BaseModel]: return [self.inputs, self.workflow, self.output] + + +def to_command_arguments( + *, config: ReductionConfig, one_line: bool = True, separator: str = '\\\n' +) -> list[str] | str: + """Convert the config to a list of command line arguments. + + Parameters + ---------- + one_line: + If True, return a single string with all arguments joined by spaces. + If False, return a list of argument strings. + + """ + args = {} + for instance in config._children: + args.update(instance.model_dump(mode='python')) + args = {f"--{k.replace('_', '-')}": v for k, v in args.items() if v is not None} + + arg_list = [] + for k, v in args.items(): + if not isinstance(v, bool): + arg_list.append(k) + if isinstance(v, list): + arg_list.extend(str(item) for item in v) + elif isinstance(v, enum.StrEnum): + arg_list.append(v.value) + else: + arg_list.append(str(v)) + elif v is True: + arg_list.append(k) + + if one_line: + # Default separator is backslash + newline for better readability + # Users can directly copy-paste the output in a terminal or a script. + return ( + (separator + '--') + .join(" ".join(arg_list).split('--')) + .removeprefix(separator) + ) + else: + return arg_list diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 0aed085d..83ac976e 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -117,22 +117,13 @@ def reduction( base_wf[TimeOfFlightLookupTable] = base_wf.compute(TimeOfFlightLookupTable) metadatas = base_wf.compute((NMXSampleMetadata, NMXSourceMetadata)) - export_static_metadata_as_nxlauetof( - sample_metadata=metadatas[NMXSampleMetadata], - source_metadata=metadatas[NMXSourceMetadata], - output_file=config.output.output_file, - ) tof_das = sc.DataGroup() detector_metas = sc.DataGroup() for detector_name in detector_names: cur_wf = base_wf.copy() cur_wf[NeXusName[snx.NXdetector]] = detector_name results = cur_wf.compute((TofDetector[SampleRun], NMXDetectorMetadata)) - detector_meta: NMXDetectorMetadata = results[NMXDetectorMetadata] - export_detector_metadata_as_nxlauetof( - detector_metadata=detector_meta, output_file=config.output.output_file - ) - detector_metas[detector_name] = detector_meta + detector_metas[detector_name] = results[NMXDetectorMetadata] # Binning into 1 bin and getting final tof bin edges later. tof_das[detector_name] = results[TofDetector[SampleRun]] @@ -154,28 +145,54 @@ def reduction( data=sc.ones_like(tof_bin_edges[:-1]), ), ) - export_monitor_metadata_as_nxlauetof( - monitor_metadata=monitor_metadata, output_file=config.output.output_file - ) # Histogram detector counts tof_histograms = sc.DataGroup() for detector_name, tof_da in tof_das.items(): - det_meta: NMXDetectorMetadata = detector_metas[detector_name] histogram = tof_da.hist(tof=tof_bin_edges) tof_histograms[detector_name] = histogram - export_reduced_data_as_nxlauetof( - detector_name=det_meta.detector_name, - da=histogram, - output_file=config.output.output_file, - compress_mode=config.output.compression, - ) - return sc.DataGroup( - metadata=detector_metas, + results = sc.DataGroup( histogram=tof_histograms, + detector=detector_metas, + sample=metadatas[NMXSampleMetadata], + source=metadatas[NMXSourceMetadata], + monitor=monitor_metadata, lookup_table=base_wf.compute(TimeOfFlightLookupTable), ) + if not config.output.skip_file_output: + save_results(results=results, output_config=config.output) + + return results + + +def save_results(*, results: sc.DataGroup, output_config: OutputConfig) -> None: + # Validate if results have expected fields + for mandatory_key in ['histogram', 'detector', 'sample', 'source', 'monitor']: + if mandatory_key not in results: + raise ValueError(f"Missing '{mandatory_key}' in results to save.") + + export_static_metadata_as_nxlauetof( + sample_metadata=results['sample'], + source_metadata=results['source'], + output_file=output_config.output_file, + overwrite=output_config.overwrite, + ) + export_monitor_metadata_as_nxlauetof( + monitor_metadata=results['monitor'], + output_file=output_config.output_file, + ) + for detector_name, detector_meta in results['detector'].items(): + export_detector_metadata_as_nxlauetof( + detector_metadata=detector_meta, + output_file=output_config.output_file, + ) + export_reduced_data_as_nxlauetof( + detector_name=detector_name, + da=results['histogram'][detector_name], + output_file=output_config.output_file, + compress_mode=output_config.compression, + ) def main() -> None: diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/nexus.py index eb233b2d..8de79d4d 100644 --- a/packages/essnmx/src/ess/nmx/nexus.py +++ b/packages/essnmx/src/ess/nmx/nexus.py @@ -210,9 +210,11 @@ def _add_arbitrary_metadata( def export_static_metadata_as_nxlauetof( + *, sample_metadata: NMXSampleMetadata, source_metadata: NMXSourceMetadata, output_file: str | pathlib.Path | io.BytesIO, + overwrite: bool = False, **arbitrary_metadata: sc.Variable, ) -> None: """Export the metadata to a NeXus file with the LAUE_TOF application definition. @@ -237,7 +239,7 @@ def export_static_metadata_as_nxlauetof( Arbitrary metadata that does not fit into the existing metadata objects. """ - _check_file(output_file, overwrite=True) + _check_file(output_file, overwrite=overwrite) with h5py.File(output_file, "w") as f: f.attrs["NX_class"] = "NXlauetof" nx_entry = _create_lauetof_data_entry(f) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index 202b2f8b..041961d5 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -18,9 +18,8 @@ WorkflowConfig, build_reduction_argument_parser, reduction_config_from_args, - to_command_arguments, ) -from ess.nmx.configurations import TimeBinCoordinate, TimeBinUnit +from ess.nmx.configurations import TimeBinCoordinate, TimeBinUnit, to_command_arguments from ess.nmx.executables import reduction from ess.nmx.types import Compression @@ -101,7 +100,11 @@ def test_reduction_config() -> None: tof_simulation_seed=12345, ) output_options = OutputConfig( - output_file='test-output.h5', compression=Compression.NONE, verbose=True + output_file='test-output.h5', + compression=Compression.NONE, + verbose=True, + skip_file_output=True, + overwrite=True, ) expected_config = ReductionConfig( inputs=input_options, workflow=workflow_options, output=output_options @@ -113,7 +116,7 @@ def test_reduction_config() -> None: arg_list = _build_arg_list_from_pydantic_instance( input_options, workflow_options, output_options ) - assert arg_list == to_command_arguments(expected_config, one_line=False) + assert arg_list == to_command_arguments(config=expected_config, one_line=False) # Parse arguments and build config from them. parser = build_reduction_argument_parser() @@ -190,7 +193,9 @@ def reduction_config( # only properly works in linux so we set it to NONE here # for convenience of testing on all platforms. output_config = OutputConfig( - output_file=temp_output_file.as_posix(), compression=Compression.NONE + output_file=temp_output_file.as_posix(), + compression=Compression.NONE, + skip_file_output=True, # No need to write output file for most tests. ) return ReductionConfig(inputs=input_config, output=output_config) From fea3a4605ab426532701deb9742055cbca17e59b Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Fri, 5 Dec 2025 14:15:39 +0100 Subject: [PATCH 325/403] Add configurations module in the API reference page. --- packages/essnmx/docs/api-reference/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/essnmx/docs/api-reference/index.md b/packages/essnmx/docs/api-reference/index.md index 5da7f253..40f2176b 100644 --- a/packages/essnmx/docs/api-reference/index.md +++ b/packages/essnmx/docs/api-reference/index.md @@ -34,5 +34,6 @@ types mtz_io scaling + configurations ``` From 5cc930971c0a18083e7db7dafd3e591f454a4950 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 16 Dec 2025 11:00:10 +0100 Subject: [PATCH 326/403] Fix unresolved merge conflict. --- packages/essnmx/src/ess/nmx/workflows.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index db55c1ce..0a697a01 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -58,11 +58,7 @@ def _simulate_fixed_wavelength_tof( source = tof.Source( facility="ess", neutrons=neutrons, -<<<<<<< HEAD pulses=2, -======= - pulses=1, # NMX does not use pulse-skipping. ->>>>>>> 39aeba2 (Expose tof simulation number of neutron configuration.) seed=seed, wmax=wmax, wmin=wmin, From 2a69e891481e9922e809ba9a01805c392bd53c67 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 16 Dec 2025 11:01:54 +0100 Subject: [PATCH 327/403] Restore the accidentally removed imported module. --- packages/essnmx/src/ess/nmx/executables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 83ac976e..2d68928d 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -16,7 +16,7 @@ collect_matching_input_files, reduction_config_from_args, ) -from .configurations import ReductionConfig +from .configurations import OutputConfig, ReductionConfig from .nexus import ( export_detector_metadata_as_nxlauetof, export_monitor_metadata_as_nxlauetof, From 68146b408bbca5c423b2eb1f7ff0a59949533a7d Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 16 Dec 2025 11:25:22 +0100 Subject: [PATCH 328/403] Add tof lut file test. --- packages/essnmx/tests/executable_test.py | 47 ++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index 041961d5..a758b2b3 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -10,6 +10,7 @@ import pytest import scipp as sc import scippnexus as snx +from scipp.testing import assert_identical from ess.nmx._executable_helper import ( InputConfig, @@ -219,3 +220,49 @@ def test_reduction_only_number_of_time_bins(reduction_config: ReductionConfig) - # Check that the number of time bins is as expected. assert len(hist.coords['tof']) == 21 # nbins + 1 edges + + +@pytest.fixture +def tof_lut_file_path(tmp_path: pathlib.Path): + """Fixture to provide the path to the small NMX NeXus file.""" + from ess.nmx.workflows import initialize_nmx_workflow + from ess.reduce.time_of_flight import TimeOfFlightLookupTable + + # Simply use the default workflow for testing. + workflow = initialize_nmx_workflow(config=WorkflowConfig()) + tof_lut: sc.DataArray = workflow.compute(TimeOfFlightLookupTable) + + # Change the tof range a bit for testing. + tof_lut *= sc.scalar(2.0, unit='dimensionless') + + lut_file_path = tmp_path / "nmx_tof_lookup_table.h5" + tof_lut.save_hdf5(lut_file_path.as_posix()) + yield lut_file_path + if lut_file_path.exists(): + lut_file_path.unlink() + + +def test_reduction_with_tof_lut_file( + reduction_config: ReductionConfig, tof_lut_file_path: pathlib.Path +) -> None: + # Make sure the config uses no TOF lookup table file initially. + assert reduction_config.workflow.tof_lookup_table_file_path is None + with known_warnings(): + default_results = reduction(config=reduction_config) + + # Update config to use the TOF lookup table file. + reduction_config.workflow.tof_lookup_table_file_path = tof_lut_file_path.as_posix() + with known_warnings(): + results = reduction(config=reduction_config) + + for default_hist, hist in zip( + default_results['histogram'].values(), + results['histogram'].values(), + strict=True, + ): + tof_edges_default = default_hist.coords['tof'] + tof_edges = hist.coords['tof'] + assert_identical(default_hist.data, hist.data) + assert_identical( + tof_edges_default * sc.scalar(2.0, unit='dimensionless'), tof_edges + ) From deee52486aa2e43b77211c1786a7c48e403ed173 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 16 Dec 2025 11:35:11 +0100 Subject: [PATCH 329/403] Filter out masked events. --- packages/essnmx/src/ess/nmx/workflows.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index d8aaac43..709276e0 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -62,6 +62,11 @@ def _simulate_fixed_wavelength_tof( model = tof.Model(source=source, choppers=[], detectors=[nmx_det]) results = model.run() events = results["detector"].data.squeeze().flatten(to="event") + # If there are any blocked neutrons, remove them + # it is not expected to have any in this simulation + # since it is not using any choppers + # but just in case we ever add any in the future + events = events[~events.masks["blocked_by_others"]] return SimulationResults( time_of_arrival=events.coords["toa"], speed=events.coords["speed"], From 45e0bf9223055a17611dddc576cfe0941f92bc22 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 9 Dec 2025 10:10:06 +0100 Subject: [PATCH 330/403] Set minimum bound of tof package. Issue with TOF LUTs with limited range of wavelengths was fixed, see https://github.com/scipp/tof/pull/109 --- packages/essnmx/pyproject.toml | 2 +- packages/essnmx/requirements/base.in | 2 +- packages/essnmx/requirements/base.txt | 24 +++++++---------- packages/essnmx/requirements/basetest.txt | 2 +- packages/essnmx/requirements/ci.txt | 4 +-- packages/essnmx/requirements/dev.txt | 6 +---- packages/essnmx/requirements/docs.txt | 14 +++++----- packages/essnmx/requirements/mypy.txt | 4 ++- packages/essnmx/requirements/nightly.in | 2 +- packages/essnmx/requirements/nightly.txt | 32 +++++++++-------------- packages/essnmx/requirements/static.txt | 2 +- 11 files changed, 42 insertions(+), 52 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 6d67ab07..676d9567 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -43,7 +43,7 @@ dependencies = [ "defusedxml>=0.7.1", "bitshuffle>=0.5.2", "msgpack>=1.0.8", - "tof>=25.12.0", + "tof>=25.12.1", ] dynamic = ["version"] diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index 7f14a721..017d4379 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -15,4 +15,4 @@ gemmi>=0.6.6 defusedxml>=0.7.1 bitshuffle>=0.5.2 msgpack>=1.0.8 -tof>=25.12.0 +tof>=25.12.1 diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 12864a3e..6ab65ed6 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:60fc64eb056a0e8e93d0a6ebf0875a2e341193b5 +# SHA1:f587b0729a7479dd1077a91433877224e995175b # # This file was generated by pip-compile-multi. # To update, run: @@ -23,7 +23,7 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -cython==3.2.1 +cython==3.2.2 # via bitshuffle dask==2025.11.0 # via -r base.in @@ -33,11 +33,11 @@ dnspython==2.8.0 # via email-validator email-validator==2.3.0 # via scippneutron -essreduce==25.11.5 +essreduce==25.12.1 # via -r base.in -fonttools==4.60.1 +fonttools==4.61.0 # via matplotlib -fsspec==2025.10.0 +fsspec==2025.12.0 # via dask gemmi==0.7.4 # via -r base.in @@ -52,8 +52,6 @@ idna==3.11 # via # email-validator # requests -importlib-metadata==8.7.0 - # via dask kiwisolver==1.4.9 # via matplotlib lazy-loader==0.4 @@ -71,7 +69,7 @@ mpltoolbox==25.10.0 # via scippneutron msgpack==1.1.2 # via -r base.in -networkx==3.6 +networkx==3.6.1 # via cyclebane numpy==2.3.5 # via @@ -95,7 +93,7 @@ partd==1.4.2 # via dask pillow==12.0.0 # via matplotlib -platformdirs==4.5.0 +platformdirs==4.5.1 # via pooch plopp==25.11.0 # via @@ -134,7 +132,7 @@ scipp==25.11.0 # scippneutron # scippnexus # tof -scippneutron==25.11.0 +scippneutron==25.11.2 # via essreduce scippnexus==25.11.0 # via @@ -147,7 +145,7 @@ scipy==1.16.3 # scippnexus six==1.17.0 # via python-dateutil -tof==25.12.0 +tof==25.12.1 # via -r base.in toolz==1.1.0 # via @@ -162,10 +160,8 @@ typing-inspection==0.4.2 # via pydantic tzdata==2025.2 # via pandas -urllib3==2.5.0 +urllib3==2.6.1 # via requests -zipp==3.23.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt index acbe0b32..e9fde5ff 100644 --- a/packages/essnmx/requirements/basetest.txt +++ b/packages/essnmx/requirements/basetest.txt @@ -13,5 +13,5 @@ pluggy==1.6.0 # via pytest pygments==2.19.2 # via pytest -pytest==9.0.1 +pytest==9.0.2 # via -r basetest.in diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index 4b49511f..5e597a7b 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -32,7 +32,7 @@ packaging==25.0 # -r ci.in # pyproject-api # tox -platformdirs==4.5.0 +platformdirs==4.5.1 # via # tox # virtualenv @@ -46,7 +46,7 @@ smmap==5.0.2 # via gitdb tox==4.32.0 # via -r ci.in -urllib3==2.5.0 +urllib3==2.6.1 # via requests virtualenv==20.35.4 # via tox diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index 6c15809f..b3ae8b91 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -12,7 +12,7 @@ -r static.txt -r test.txt -r wheels.txt -anyio==4.11.0 +anyio==4.12.0 # via # httpx # jupyter-server @@ -73,8 +73,6 @@ lark==1.3.1 # via rfc3987-syntax notebook-shim==0.2.4 # via jupyterlab -overrides==7.7.0 - # via jupyter-server pip-compile-multi==3.2.2 # via -r dev.in pip-tools==7.5.2 @@ -101,8 +99,6 @@ rfc3987-syntax==1.1.0 # via jsonschema send2trash==1.8.3 # via jupyter-server -sniffio==1.3.1 - # via anyio terminado==0.18.1 # via # jupyter-server diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 4971a08f..9eef163b 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -24,7 +24,7 @@ babel==2.17.0 # via # pydata-sphinx-theme # sphinx -beautifulsoup4==4.14.2 +beautifulsoup4==4.14.3 # via # nbconvert # pydata-sphinx-theme @@ -54,7 +54,7 @@ ipydatawidgets==4.3.5 # via pythreejs ipykernel==7.1.0 # via -r docs.in -ipython==9.7.0 +ipython==9.8.0 # via # -r docs.in # ipykernel @@ -121,7 +121,7 @@ nbformat==5.10.4 # nbclient # nbconvert # nbsphinx -nbsphinx==0.9.7 +nbsphinx==0.9.8 # via -r docs.in nest-asyncio==1.6.0 # via ipykernel @@ -163,7 +163,9 @@ referencing==0.37.0 # via # jsonschema # jsonschema-specifications -rpds-py==0.29.0 +roman-numerals-py==3.1.0 + # via sphinx +rpds-py==0.30.0 # via # jsonschema # referencing @@ -171,7 +173,7 @@ snowballstemmer==3.0.1 # via sphinx soupsieve==2.8 # via beautifulsoup4 -sphinx==8.1.3 +sphinx==8.2.3 # via # -r docs.in # autodoc-pydantic @@ -181,7 +183,7 @@ sphinx==8.1.3 # sphinx-autodoc-typehints # sphinx-copybutton # sphinx-design -sphinx-autodoc-typehints==3.0.1 +sphinx-autodoc-typehints==3.5.2 # via -r docs.in sphinx-copybutton==0.5.2 # via -r docs.in diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index 74e2fd20..1884b2a5 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -6,7 +6,9 @@ # requirements upgrade # -r test.txt -mypy==1.18.2 +librt==0.7.3 + # via mypy +mypy==1.19.0 # via -r mypy.in mypy-extensions==1.1.0 # via mypy diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index 29b77c0f..b579c0a7 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -10,7 +10,7 @@ gemmi>=0.6.6 defusedxml>=0.7.1 bitshuffle>=0.5.2 msgpack>=1.0.8 -tof>=25.12.0 +tof>=25.12.1 pytest>=7.0 scipp --index-url=https://pypi.anaconda.org/scipp-nightly-wheels/simple/ diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 625fda38..c17a115d 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:82815c7a739545007523e0e1a48d57f0f7e427db +# SHA1:ffed931b80e632af913de357166a7bc05bb84e8c # # This file was generated by pip-compile-multi. # To update, run: @@ -26,7 +26,7 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -cython==3.2.1 +cython==3.2.2 # via bitshuffle dask==2025.11.0 # via -r nightly.in @@ -36,11 +36,11 @@ dnspython==2.8.0 # via email-validator email-validator==2.3.0 # via scippneutron -essreduce==25.11.5 +essreduce==25.12.1 # via -r nightly.in -fonttools==4.60.1 +fonttools==4.61.0 # via matplotlib -fsspec==2025.10.0 +fsspec==2025.12.0 # via dask gemmi==0.7.4 # via -r nightly.in @@ -55,8 +55,6 @@ idna==3.11 # via # email-validator # requests -importlib-metadata==8.7.0 - # via dask iniconfig==2.3.0 # via pytest kiwisolver==1.4.10rc0 @@ -76,9 +74,9 @@ mpltoolbox==25.10.0 # via scippneutron msgpack==1.1.2 # via -r nightly.in -networkx==3.6 +networkx==3.6.1 # via cyclebane -numpy==2.3.5 +numpy==2.4.0rc1 # via # bitshuffle # contourpy @@ -95,13 +93,13 @@ packaging==25.0 # matplotlib # pooch # pytest -pandas==2.3.3 +pandas==3.0.0rc0 # via -r nightly.in partd==1.4.2 # via dask pillow==12.0.0 # via matplotlib -platformdirs==4.5.0 +platformdirs==4.5.1 # via pooch plopp @ git+https://github.com/scipp/plopp@main # via @@ -122,15 +120,13 @@ pygments==2.19.2 # via pytest pyparsing==3.3.0b1 # via matplotlib -pytest==9.0.1 +pytest==9.0.2 # via -r nightly.in python-dateutil==2.9.0.post0 # via # matplotlib # pandas # scippneutron -pytz==2025.2 - # via pandas pyyaml==6.0.3 # via dask requests==2.32.5 @@ -146,7 +142,7 @@ scipp==100.0.0.dev0 # scippneutron # scippnexus # tof -scippneutron==25.11.0 +scippneutron==25.11.2 # via essreduce scippnexus @ git+https://github.com/scipp/scippnexus@main # via @@ -159,7 +155,7 @@ scipy==1.16.3 # scippnexus six==1.17.0 # via python-dateutil -tof==25.12.0 +tof==25.12.1 # via -r nightly.in toolz==1.1.0 # via @@ -174,10 +170,8 @@ typing-inspection==0.4.2 # via pydantic tzdata==2025.2 # via pandas -urllib3==2.5.0 +urllib3==2.6.1 # via requests -zipp==3.23.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index 4cc95f11..2a110d83 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -15,7 +15,7 @@ identify==2.6.15 # via pre-commit nodeenv==1.9.1 # via pre-commit -platformdirs==4.5.0 +platformdirs==4.5.1 # via virtualenv pre-commit==4.5.0 # via -r static.in From 62c600783c87d54f570b198f2f11d88a55f9284a Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 16 Dec 2025 13:17:04 +0100 Subject: [PATCH 331/403] Update number of pulse to simulate to 1 --- packages/essnmx/src/ess/nmx/workflows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index 0a697a01..f4e9bd3e 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -58,7 +58,7 @@ def _simulate_fixed_wavelength_tof( source = tof.Source( facility="ess", neutrons=neutrons, - pulses=2, + pulses=1, seed=seed, wmax=wmax, wmin=wmin, From c485b769bae18c727e31b546bde1f3d01f600a98 Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 16 Dec 2025 13:58:41 +0100 Subject: [PATCH 332/403] Fix typos and grammar Co-authored-by: Neil Vaytet <39047984+nvaytet@users.noreply.github.com> --- packages/essnmx/docs/user-guide/workflow.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/essnmx/docs/user-guide/workflow.ipynb b/packages/essnmx/docs/user-guide/workflow.ipynb index bd28d204..7b403093 100644 --- a/packages/essnmx/docs/user-guide/workflow.ipynb +++ b/packages/essnmx/docs/user-guide/workflow.ipynb @@ -7,7 +7,7 @@ "# NMX Reduction Workflow\n", "\n", "> NMX does not expect users to use python interface directly.
\n", - "This documentation is mostly for istrument data scientists or instrument scientists.
" + "This documentation is mostly for instrument data scientists or instrument scientists.
" ] }, { @@ -37,7 +37,7 @@ "config = ReductionConfig(\n", " inputs=InputConfig(\n", " input_file=[get_small_nmx_nexus().as_posix()],\n", - " detector_ids=[0, 1, 2], # Detector index to be reduced in alphabetical order.\n", + " detector_ids=[0, 1, 2],\n", " ),\n", " output=OutputConfig(\n", " output_file=\"scipp_output.hdf\", skip_file_output=False, overwrite=True\n", @@ -62,14 +62,14 @@ "source": [ "## Configuration\n", "\n", - "`essnmx` provide command line data reduction tool for the reduction between `nexus` and `dials`.
\n", + "`essnmx` provides a command line data reduction tool.
\n", "The `essnmx-reduce` interface will reduce `nexus` file
\n", "and save the results into `NXlauetof`(not exactly but very close) format for `dials`.
\n", "\n", "Argument options could be exhaustive therefore we wrapped them into a nested pydantic model.
\n", - "Here is a python API you can use to build the configuration and turn it into a command line arguments.\n", + "Here is a python API you can use to build the configuration and turn it into command line arguments.\n", "\n", - "**Configuration object is pydantic model so it strictly check the type of the arguments.**" + "**The configuration object is a pydantic model, and it thus enforces strict checks on the types of the arguments.**" ] }, { From 3b23a07121d5c3621ad139b2bb9d15c939314bb4 Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 16 Dec 2025 14:03:00 +0100 Subject: [PATCH 333/403] Apply suggestions from code review Co-authored-by: Neil Vaytet <39047984+nvaytet@users.noreply.github.com> --- packages/essnmx/tests/executable_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index a758b2b3..f918a3dc 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -233,7 +233,7 @@ def tof_lut_file_path(tmp_path: pathlib.Path): tof_lut: sc.DataArray = workflow.compute(TimeOfFlightLookupTable) # Change the tof range a bit for testing. - tof_lut *= sc.scalar(2.0, unit='dimensionless') + tof_lut *= 2 lut_file_path = tmp_path / "nmx_tof_lookup_table.h5" tof_lut.save_hdf5(lut_file_path.as_posix()) @@ -264,5 +264,5 @@ def test_reduction_with_tof_lut_file( tof_edges = hist.coords['tof'] assert_identical(default_hist.data, hist.data) assert_identical( - tof_edges_default * sc.scalar(2.0, unit='dimensionless'), tof_edges + tof_edges_default * 2, tof_edges ) From d2687f7c8fce11d8fe825ac1a2194a6960fba8d6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Tue, 16 Dec 2025 13:03:33 +0000 Subject: [PATCH 334/403] Apply automatic formatting --- packages/essnmx/tests/executable_test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index f918a3dc..4fba9c62 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -263,6 +263,4 @@ def test_reduction_with_tof_lut_file( tof_edges_default = default_hist.coords['tof'] tof_edges = hist.coords['tof'] assert_identical(default_hist.data, hist.data) - assert_identical( - tof_edges_default * 2, tof_edges - ) + assert_identical(tof_edges_default * 2, tof_edges) From 54cdd4d987590fb5a62716ff1ac35dbf2bbcd21e Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 16 Dec 2025 15:49:20 +0100 Subject: [PATCH 335/403] Rephrase reason to use pydantic model. --- packages/essnmx/docs/user-guide/workflow.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/docs/user-guide/workflow.ipynb b/packages/essnmx/docs/user-guide/workflow.ipynb index 7b403093..1655dd96 100644 --- a/packages/essnmx/docs/user-guide/workflow.ipynb +++ b/packages/essnmx/docs/user-guide/workflow.ipynb @@ -66,7 +66,7 @@ "The `essnmx-reduce` interface will reduce `nexus` file
\n", "and save the results into `NXlauetof`(not exactly but very close) format for `dials`.
\n", "\n", - "Argument options could be exhaustive therefore we wrapped them into a nested pydantic model.
\n", + "For conveniences and safety, all configuration options are warpped in a nested pydantic model.
\n", "Here is a python API you can use to build the configuration and turn it into command line arguments.\n", "\n", "**The configuration object is a pydantic model, and it thus enforces strict checks on the types of the arguments.**" From c3e609fab5ce249600974b190bf503a149499b5e Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 16 Dec 2025 16:02:49 +0100 Subject: [PATCH 336/403] Fix typo! --- packages/essnmx/docs/user-guide/workflow.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/docs/user-guide/workflow.ipynb b/packages/essnmx/docs/user-guide/workflow.ipynb index 1655dd96..af3d2fc2 100644 --- a/packages/essnmx/docs/user-guide/workflow.ipynb +++ b/packages/essnmx/docs/user-guide/workflow.ipynb @@ -66,7 +66,7 @@ "The `essnmx-reduce` interface will reduce `nexus` file
\n", "and save the results into `NXlauetof`(not exactly but very close) format for `dials`.
\n", "\n", - "For conveniences and safety, all configuration options are warpped in a nested pydantic model.
\n", + "For conveniences and safety, all configuration options are wrapped in a nested pydantic model.
\n", "Here is a python API you can use to build the configuration and turn it into command line arguments.\n", "\n", "**The configuration object is a pydantic model, and it thus enforces strict checks on the types of the arguments.**" From 91759f549fcbe1998c71f1a7dc2c7cb1be79c242 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 16 Dec 2025 16:29:11 +0100 Subject: [PATCH 337/403] New interface of the toflookup table. --- packages/essnmx/tests/executable_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index 4fba9c62..9ac2acb4 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -230,10 +230,10 @@ def tof_lut_file_path(tmp_path: pathlib.Path): # Simply use the default workflow for testing. workflow = initialize_nmx_workflow(config=WorkflowConfig()) - tof_lut: sc.DataArray = workflow.compute(TimeOfFlightLookupTable) + tof_lut: TimeOfFlightLookupTable = workflow.compute(TimeOfFlightLookupTable) # Change the tof range a bit for testing. - tof_lut *= 2 + tof_lut.array *= 2 lut_file_path = tmp_path / "nmx_tof_lookup_table.h5" tof_lut.save_hdf5(lut_file_path.as_posix()) From 2e749aeb40e1698a58ef59c93b93904df6d5f7c9 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 16 Dec 2025 16:59:17 +0100 Subject: [PATCH 338/403] Fix linkcheck. --- packages/essnmx/docs/about/data_workflow_overview.md | 2 +- packages/essnmx/docs/conf.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/essnmx/docs/about/data_workflow_overview.md b/packages/essnmx/docs/about/data_workflow_overview.md index 159574d6..998db451 100644 --- a/packages/essnmx/docs/about/data_workflow_overview.md +++ b/packages/essnmx/docs/about/data_workflow_overview.md @@ -92,7 +92,7 @@ dev.dials.simple_tof_integrate refined.expt refined.refl ``` ### Scaling (LSCALE/pyscale) -Currently [LSCALE](https://scripts.iucr.org/cgi-bin/paper?S0021889898015350) can be used in a docker container which makes it indented from the OS.
+Currently [LSCALE](https://doi.org/10.1107/S0021889898015350) can be used in a docker container which makes it indented from the OS.
LSCALE is a program for scaling and normalisation of Laue intensity data.
The source code is available on [Zenodo](https://zenodo.org/records/4381992).
Since LSCALE is not maintained anymore we are currently developing a Python-based alternative to LSCALE called pyscale[^4]. diff --git a/packages/essnmx/docs/conf.py b/packages/essnmx/docs/conf.py index 67f6fc3f..b427d470 100644 --- a/packages/essnmx/docs/conf.py +++ b/packages/essnmx/docs/conf.py @@ -269,4 +269,6 @@ def do_not_plot(*args, **kwargs): # Since DOIs are supposed to be permanent, we don't need to check them.' r'https?://doi\.org/', r'https?://dx\.doi\.org/', + r'https://www\.ccp4\.ac\.uk/*', # Seems to be denied by the server. + # Manually checked and working ] From bc2271434915f2c2d1623b7480a5198f7f663dfa Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 17 Dec 2025 16:42:07 +0100 Subject: [PATCH 339/403] Handle different TimeOfFlightLookUpTable type in a test. (#180) * Handle different timeofflighttable type. * Remove DataGroup support. --- packages/essnmx/tests/executable_test.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index 9ac2acb4..f12a14db 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -225,6 +225,8 @@ def test_reduction_only_number_of_time_bins(reduction_config: ReductionConfig) - @pytest.fixture def tof_lut_file_path(tmp_path: pathlib.Path): """Fixture to provide the path to the small NMX NeXus file.""" + from dataclasses import is_dataclass + from ess.nmx.workflows import initialize_nmx_workflow from ess.reduce.time_of_flight import TimeOfFlightLookupTable @@ -233,7 +235,12 @@ def tof_lut_file_path(tmp_path: pathlib.Path): tof_lut: TimeOfFlightLookupTable = workflow.compute(TimeOfFlightLookupTable) # Change the tof range a bit for testing. - tof_lut.array *= 2 + if isinstance(tof_lut, sc.DataArray): + tof_lut *= 2 + elif is_dataclass(tof_lut): + tof_lut.array *= 2 + else: + raise TypeError("Unexpected type for TOF lookup table.") lut_file_path = tmp_path / "nmx_tof_lookup_table.h5" tof_lut.save_hdf5(lut_file_path.as_posix()) From aaf863d46dd45209b31cab0419ece83f96a34d8c Mon Sep 17 00:00:00 2001 From: Mridul Seth Date: Fri, 19 Dec 2025 11:49:13 +0100 Subject: [PATCH 340/403] Skip bitshuffle as a dep on windows --- packages/essnmx/pyproject.toml | 2 +- packages/essnmx/requirements/base.in | 2 +- packages/essnmx/requirements/base.txt | 22 +++++++++++++--------- packages/essnmx/requirements/ci.txt | 6 +++--- packages/essnmx/requirements/dev.txt | 4 +++- packages/essnmx/requirements/docs.txt | 12 +++++++----- packages/essnmx/requirements/mypy.txt | 4 ++-- packages/essnmx/requirements/nightly.in | 2 +- packages/essnmx/requirements/nightly.txt | 22 +++++++++++++--------- packages/essnmx/requirements/static.txt | 4 ++-- 10 files changed, 46 insertions(+), 34 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 676d9567..25279351 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -41,9 +41,9 @@ dependencies = [ "pandas>=2.1.2", "gemmi>=0.6.6", "defusedxml>=0.7.1", - "bitshuffle>=0.5.2", "msgpack>=1.0.8", "tof>=25.12.1", + "bitshuffle>=0.5.2;os_name == 'posix'" ] dynamic = ["version"] diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index 017d4379..e873e726 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -13,6 +13,6 @@ pooch>=1.5 pandas>=2.1.2 gemmi>=0.6.6 defusedxml>=0.7.1 -bitshuffle>=0.5.2 msgpack>=1.0.8 tof>=25.12.1 +bitshuffle>=0.5.2;os_name == 'posix' diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 6ab65ed6..748853ba 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:f587b0729a7479dd1077a91433877224e995175b +# SHA1:4a430e06cfd011c7ff94e1985071c7b73750560a # # This file was generated by pip-compile-multi. # To update, run: @@ -7,7 +7,7 @@ # annotated-types==0.7.0 # via pydantic -bitshuffle==0.5.2 +bitshuffle==0.5.2 ; os_name == "posix" # via -r base.in certifi==2025.11.12 # via requests @@ -23,9 +23,9 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -cython==3.2.2 +cython==3.2.3 # via bitshuffle -dask==2025.11.0 +dask==2025.12.0 # via -r base.in defusedxml==0.7.1 # via -r base.in @@ -35,7 +35,7 @@ email-validator==2.3.0 # via scippneutron essreduce==25.12.1 # via -r base.in -fonttools==4.61.0 +fonttools==4.61.1 # via matplotlib fsspec==2025.12.0 # via dask @@ -52,6 +52,8 @@ idna==3.11 # via # email-validator # requests +importlib-metadata==8.7.0 + # via dask kiwisolver==1.4.9 # via matplotlib lazy-loader==0.4 @@ -61,7 +63,7 @@ lazy-loader==0.4 # tof locket==1.0.0 # via partd -matplotlib==3.10.7 +matplotlib==3.10.8 # via # mpltoolbox # plopp @@ -125,7 +127,7 @@ sciline==25.11.1 # via # -r base.in # essreduce -scipp==25.11.0 +scipp==25.12.0 # via # -r base.in # essreduce @@ -158,10 +160,12 @@ typing-extensions==4.15.0 # typing-inspection typing-inspection==0.4.2 # via pydantic -tzdata==2025.2 +tzdata==2025.3 # via pandas -urllib3==2.6.1 +urllib3==2.6.2 # via requests +zipp==3.23.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index 5e597a7b..685f1892 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -5,7 +5,7 @@ # # requirements upgrade # -cachetools==6.2.2 +cachetools==6.2.4 # via tox certifi==2025.11.12 # via requests @@ -17,7 +17,7 @@ colorama==0.4.6 # via tox distlib==0.4.0 # via virtualenv -filelock==3.20.0 +filelock==3.20.1 # via # tox # virtualenv @@ -46,7 +46,7 @@ smmap==5.0.2 # via gitdb tox==4.32.0 # via -r ci.in -urllib3==2.6.1 +urllib3==2.6.2 # via requests virtualenv==20.35.4 # via tox diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index b3ae8b91..fe6d0b48 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -65,7 +65,7 @@ jupyter-server==2.17.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.5.0 +jupyterlab==4.5.1 # via -r dev.in jupyterlab-server==2.28.0 # via jupyterlab @@ -73,6 +73,8 @@ lark==1.3.1 # via rfc3987-syntax notebook-shim==0.2.4 # via jupyterlab +overrides==7.7.0 + # via jupyter-server pip-compile-multi==3.2.2 # via -r dev.in pip-tools==7.5.2 diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 9eef163b..511343b1 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -34,7 +34,7 @@ comm==0.2.3 # via # ipykernel # ipywidgets -debugpy==1.8.17 +debugpy==1.8.19 # via ipykernel decorator==5.2.1 # via ipython @@ -77,7 +77,7 @@ jsonschema==4.25.1 # via nbformat jsonschema-specifications==2025.9.1 # via jsonschema -jupyter-client==8.6.3 +jupyter-client==8.7.0 # via # ipykernel # nbclient @@ -163,7 +163,9 @@ referencing==0.37.0 # via # jsonschema # jsonschema-specifications -roman-numerals-py==3.1.0 +roman-numerals==4.1.0 + # via roman-numerals-py +roman-numerals-py==4.1.0 # via sphinx rpds-py==0.30.0 # via @@ -171,7 +173,7 @@ rpds-py==0.30.0 # referencing snowballstemmer==3.0.1 # via sphinx -soupsieve==2.8 +soupsieve==2.8.1 # via beautifulsoup4 sphinx==8.2.3 # via @@ -205,7 +207,7 @@ stack-data==0.6.3 # via ipython tinycss2==1.4.0 # via bleach -tornado==6.5.2 +tornado==6.5.4 # via # ipykernel # jupyter-client diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index 1884b2a5..7c71bd8e 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -6,9 +6,9 @@ # requirements upgrade # -r test.txt -librt==0.7.3 +librt==0.7.4 # via mypy -mypy==1.19.0 +mypy==1.19.1 # via -r mypy.in mypy-extensions==1.1.0 # via mypy diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index b579c0a7..ac3d7c95 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -8,9 +8,9 @@ pooch>=1.5 pandas>=2.1.2 gemmi>=0.6.6 defusedxml>=0.7.1 -bitshuffle>=0.5.2 msgpack>=1.0.8 tof>=25.12.1 +bitshuffle>=0.5.2;os_name == 'posix' pytest>=7.0 scipp --index-url=https://pypi.anaconda.org/scipp-nightly-wheels/simple/ diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index c17a115d..0421653e 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:ffed931b80e632af913de357166a7bc05bb84e8c +# SHA1:b96d14f543449e35d860f235d9259b0b1504fb14 # # This file was generated by pip-compile-multi. # To update, run: @@ -10,7 +10,7 @@ annotated-types==0.7.0 # via pydantic -bitshuffle==0.5.2 +bitshuffle==0.5.2 ; os_name == "posix" # via -r nightly.in certifi==2025.11.12 # via requests @@ -26,9 +26,9 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -cython==3.2.2 +cython==3.2.3 # via bitshuffle -dask==2025.11.0 +dask==2025.12.0 # via -r nightly.in defusedxml==0.8.0rc2 # via -r nightly.in @@ -38,7 +38,7 @@ email-validator==2.3.0 # via scippneutron essreduce==25.12.1 # via -r nightly.in -fonttools==4.61.0 +fonttools==4.61.1 # via matplotlib fsspec==2025.12.0 # via dask @@ -55,6 +55,8 @@ idna==3.11 # via # email-validator # requests +importlib-metadata==8.7.0 + # via dask iniconfig==2.3.0 # via pytest kiwisolver==1.4.10rc0 @@ -66,7 +68,7 @@ lazy-loader==0.4 # tof locket==1.0.0 # via partd -matplotlib==3.10.7 +matplotlib==3.10.8 # via # mpltoolbox # plopp @@ -149,7 +151,7 @@ scippnexus @ git+https://github.com/scipp/scippnexus@main # -r nightly.in # essreduce # scippneutron -scipy==1.16.3 +scipy==1.17.0rc1 # via # scippneutron # scippnexus @@ -168,10 +170,12 @@ typing-extensions==4.15.0 # typing-inspection typing-inspection==0.4.2 # via pydantic -tzdata==2025.2 +tzdata==2025.3 # via pandas -urllib3==2.6.1 +urllib3==2.6.2 # via requests +zipp==3.23.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index 2a110d83..c71d195d 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -9,7 +9,7 @@ cfgv==3.5.0 # via pre-commit distlib==0.4.0 # via virtualenv -filelock==3.20.0 +filelock==3.20.1 # via virtualenv identify==2.6.15 # via pre-commit @@ -17,7 +17,7 @@ nodeenv==1.9.1 # via pre-commit platformdirs==4.5.1 # via virtualenv -pre-commit==4.5.0 +pre-commit==4.5.1 # via -r static.in pyyaml==6.0.3 # via pre-commit From f840a74439cbdf7e80ed9b072992ae209c4916e1 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 14 Jan 2026 13:12:40 +0100 Subject: [PATCH 341/403] Copier update --- packages/essnmx/.copier-answers.yml | 2 +- packages/essnmx/.github/workflows/docs.yml | 2 +- .../.github/workflows/nightly_at_main_lower_bound.yml | 2 +- packages/essnmx/.pre-commit-config.yaml | 10 +++++----- packages/essnmx/src/ess/nmx/__init__.py | 2 +- packages/essnmx/tox.ini | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/essnmx/.copier-answers.yml b/packages/essnmx/.copier-answers.yml index dc6b394f..aecca8d4 100644 --- a/packages/essnmx/.copier-answers.yml +++ b/packages/essnmx/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY -_commit: 024a41b +_commit: 0dae45f _src_path: gh:scipp/copier_template description: Data reduction for NMX at the European Spallation Source. max_python: '3.13' diff --git a/packages/essnmx/.github/workflows/docs.yml b/packages/essnmx/.github/workflows/docs.yml index 47294b92..3a303f84 100644 --- a/packages/essnmx/.github/workflows/docs.yml +++ b/packages/essnmx/.github/workflows/docs.yml @@ -71,7 +71,7 @@ jobs: path: html/ - run: echo "::notice::https://remote-unzip.deno.dev/${{ github.repository }}/artifacts/${{ steps.artifact-upload-step.outputs.artifact-id }}" - - uses: JamesIves/github-pages-deploy-action@v4.7.3 + - uses: JamesIves/github-pages-deploy-action@v4.8.0 if: ${{ inputs.publish }} with: branch: gh-pages diff --git a/packages/essnmx/.github/workflows/nightly_at_main_lower_bound.yml b/packages/essnmx/.github/workflows/nightly_at_main_lower_bound.yml index c13c3f78..c086e3cc 100644 --- a/packages/essnmx/.github/workflows/nightly_at_main_lower_bound.yml +++ b/packages/essnmx/.github/workflows/nightly_at_main_lower_bound.yml @@ -31,7 +31,7 @@ jobs: ESS_PROTECTED_FILESTORE_PASSWORD: ${{ secrets.ESS_PROTECTED_FILESTORE_PASSWORD }} steps: - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v6 + - uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.python.version }} - run: uv run --extra=test --resolution=lowest-direct pytest diff --git a/packages/essnmx/.pre-commit-config.yaml b/packages/essnmx/.pre-commit-config.yaml index 045e4a46..683b94d7 100644 --- a/packages/essnmx/.pre-commit-config.yaml +++ b/packages/essnmx/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -15,14 +15,14 @@ repos: args: [ --markdown-linebreak-ext=md ] exclude: '\.svg' - repo: https://github.com/kynan/nbstripout - rev: 0.7.1 + rev: 0.8.2 hooks: - id: nbstripout types: [ "jupyter" ] args: [ "--drop-empty-cells", "--extra-keys 'metadata.language_info.version cell.metadata.jp-MarkdownHeadingCollapsed cell.metadata.pycharm'" ] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.0 + rev: v0.14.6 hooks: - id: ruff args: [ --fix ] @@ -30,7 +30,7 @@ repos: - id: ruff-format types_or: [ python, pyi ] - repo: https://github.com/codespell-project/codespell - rev: v2.3.0 + rev: v2.4.1 hooks: - id: codespell additional_dependencies: @@ -48,7 +48,7 @@ repos: - id: rst-inline-touching-normal - id: text-unicode-replacement-char - repo: https://github.com/rhysd/actionlint - rev: v1.7.3 + rev: v1.7.9 hooks: - id: actionlint # Disable because of false-positive SC2046 diff --git a/packages/essnmx/src/ess/nmx/__init__.py b/packages/essnmx/src/ess/nmx/__init__.py index b3941868..d6e82906 100644 --- a/packages/essnmx/src/ess/nmx/__init__.py +++ b/packages/essnmx/src/ess/nmx/__init__.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) -# ruff: noqa: E402, I +# ruff: noqa: RUF100, E402, I import importlib.metadata diff --git a/packages/essnmx/tox.ini b/packages/essnmx/tox.ini index bc04863e..3fbfcaed 100644 --- a/packages/essnmx/tox.ini +++ b/packages/essnmx/tox.ini @@ -26,8 +26,8 @@ commands = pytest {posargs} description = invoke sphinx-build to build the HTML docs deps = -r requirements/docs.txt allowlist_externals=find -commands = python -m sphinx -j2 -v -b html -d {toxworkdir}/docs_doctrees docs html - python -m sphinx -j2 -v -b doctest -d {toxworkdir}/docs_doctrees docs html +commands = python -m sphinx -W -j2 -v -b html -d {toxworkdir}/docs_doctrees docs html + python -m sphinx -W -j2 -v -b doctest -d {toxworkdir}/docs_doctrees docs html find html -type f -name "*.ipynb" -not -path "html/_sources/*" -delete [testenv:releasedocs] From 4aa7a3f093a575c2e7a808813d49ee8bcf3ad44d Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 14 Jan 2026 13:19:53 +0100 Subject: [PATCH 342/403] Suppress only E501, long line --- packages/essnmx/src/ess/nmx/executables.py | 1 - .../essnmx/src/ess/nmx/mcstas/executables.py | 13 ++++++------- .../essnmx/src/ess/nmx/mcstas/streaming.py | 1 - packages/essnmx/src/ess/nmx/workflows.py | 1 - packages/essnmx/tests/executable_test.py | 9 +++++---- packages/essnmx/tests/mcstas/exporter_test.py | 19 +++++++++++-------- .../mcstas/mcstas_description_examples.py | 2 +- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 2d68928d..1c224fea 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -6,7 +6,6 @@ import scipp as sc import scippnexus as snx - from ess.reduce.nexus.types import Filename, NeXusName, SampleRun from ess.reduce.time_of_flight.types import TimeOfFlightLookupTable, TofDetector diff --git a/packages/essnmx/src/ess/nmx/mcstas/executables.py b/packages/essnmx/src/ess/nmx/mcstas/executables.py index 690f6b5f..3bef4e6e 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/executables.py +++ b/packages/essnmx/src/ess/nmx/mcstas/executables.py @@ -8,7 +8,6 @@ import sciline as sl import scipp as sc - from ess.reduce.streaming import ( EternalAccumulator, MaxAccumulator, @@ -49,9 +48,9 @@ from .xml import McStasInstrument -def _build_metadata_streaming_processor_helper() -> ( - Callable[[sl.Pipeline], StreamProcessor] -): +def _build_metadata_streaming_processor_helper() -> Callable[ + [sl.Pipeline], StreamProcessor +]: return partial( StreamProcessor, dynamic_keys=(RawEventProbability,), @@ -64,9 +63,9 @@ def _build_metadata_streaming_processor_helper() -> ( ) -def _build_final_streaming_processor_helper() -> ( - Callable[[sl.Pipeline], StreamProcessor] -): +def _build_final_streaming_processor_helper() -> Callable[ + [sl.Pipeline], StreamProcessor +]: return partial( StreamProcessor, dynamic_keys=(RawEventProbability,), diff --git a/packages/essnmx/src/ess/nmx/mcstas/streaming.py b/packages/essnmx/src/ess/nmx/mcstas/streaming.py index 2ea46257..dbf3da51 100644 --- a/packages/essnmx/src/ess/nmx/mcstas/streaming.py +++ b/packages/essnmx/src/ess/nmx/mcstas/streaming.py @@ -4,7 +4,6 @@ import scipp as sc import scippnexus as snx - from ess.reduce.streaming import Accumulator from .load import _validate_chunk_size, load_event_data_bank_name diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index f6417c96..c1506e92 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -6,7 +6,6 @@ import scipp as sc import scippnexus as snx import tof - from ess.reduce.nexus.types import ( EmptyDetector, Filename, diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index f12a14db..8b855bc2 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -71,9 +71,9 @@ def _check_non_default_config(testing_config: ReductionConfig) -> None: # This value may be None or default, so we skip the check. continue default_value = default_model[key] - assert ( - testing_value != default_value - ), f"Value for '{key}' is default: {testing_value}" + assert testing_value != default_value, ( + f"Value for '{key}' is default: {testing_value}" + ) def test_reduction_config() -> None: @@ -227,9 +227,10 @@ def tof_lut_file_path(tmp_path: pathlib.Path): """Fixture to provide the path to the small NMX NeXus file.""" from dataclasses import is_dataclass - from ess.nmx.workflows import initialize_nmx_workflow from ess.reduce.time_of_flight import TimeOfFlightLookupTable + from ess.nmx.workflows import initialize_nmx_workflow + # Simply use the default workflow for testing. workflow = initialize_nmx_workflow(config=WorkflowConfig()) tof_lut: TimeOfFlightLookupTable = workflow.compute(TimeOfFlightLookupTable) diff --git a/packages/essnmx/tests/mcstas/exporter_test.py b/packages/essnmx/tests/mcstas/exporter_test.py index 1d30cacc..576c5f23 100644 --- a/packages/essnmx/tests/mcstas/exporter_test.py +++ b/packages/essnmx/tests/mcstas/exporter_test.py @@ -80,16 +80,19 @@ def test_mcstas_reduction_export_to_bytestream( ] with io.BytesIO() as bio: - with pytest.warns( - DeprecationWarning, match='Please use ``export_as_nxlauetof`` instead.' - ): - if not _is_bitshuffle_available(): - # bitshuffle does not build correctly on Windows and ARM machines - # We are keeping this test here to catch when it builds correctly - # in the future. + if not _is_bitshuffle_available(): + # bitshuffle does not build correctly on Windows and ARM machines + # We are keeping this test here to catch when it builds correctly + # in the future. + with pytest.warns( + DeprecationWarning, match='Please use ``export_as_nxlauetof`` instead.' + ): with pytest.warns(UserWarning, match='bitshuffle.h5'): export_as_nexus(reduced_data, bio) - else: + else: + with pytest.warns( + DeprecationWarning, match='Please use ``export_as_nxlauetof`` instead.' + ): export_as_nexus(reduced_data, bio) with h5py.File(bio, 'r') as f: diff --git a/packages/essnmx/tests/mcstas/mcstas_description_examples.py b/packages/essnmx/tests/mcstas/mcstas_description_examples.py index beb9a24e..40315ead 100644 --- a/packages/essnmx/tests/mcstas/mcstas_description_examples.py +++ b/packages/essnmx/tests/mcstas/mcstas_description_examples.py @@ -1,4 +1,4 @@ -# flake8: noqa +# flake8: noqa: E501 no_detectors = """ SPLIT 999 COMPONENT Xtal = Single_crystal( From 30560e30f9bc54faad56b468c3f97ae572f4a541 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 14 Jan 2026 13:22:53 +0100 Subject: [PATCH 343/403] Update CI dependencies. --- packages/essnmx/requirements/base.txt | 22 ++++++++----------- packages/essnmx/requirements/ci.txt | 12 +++++----- packages/essnmx/requirements/dev.txt | 16 ++++++-------- packages/essnmx/requirements/docs.txt | 12 +++++----- packages/essnmx/requirements/mypy.txt | 4 ++-- packages/essnmx/requirements/nightly.txt | 28 ++++++++++-------------- packages/essnmx/requirements/static.txt | 8 +++---- packages/essnmx/requirements/wheels.txt | 2 +- 8 files changed, 46 insertions(+), 58 deletions(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 748853ba..bd6402e8 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -9,7 +9,7 @@ annotated-types==0.7.0 # via pydantic bitshuffle==0.5.2 ; os_name == "posix" # via -r base.in -certifi==2025.11.12 +certifi==2026.1.4 # via requests charset-normalizer==3.4.4 # via requests @@ -23,7 +23,7 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -cython==3.2.3 +cython==3.2.4 # via bitshuffle dask==2025.12.0 # via -r base.in @@ -33,11 +33,11 @@ dnspython==2.8.0 # via email-validator email-validator==2.3.0 # via scippneutron -essreduce==25.12.1 +essreduce==26.1.0 # via -r base.in fonttools==4.61.1 # via matplotlib -fsspec==2025.12.0 +fsspec==2026.1.0 # via dask gemmi==0.7.4 # via -r base.in @@ -52,8 +52,6 @@ idna==3.11 # via # email-validator # requests -importlib-metadata==8.7.0 - # via dask kiwisolver==1.4.9 # via matplotlib lazy-loader==0.4 @@ -73,7 +71,7 @@ msgpack==1.1.2 # via -r base.in networkx==3.6.1 # via cyclebane -numpy==2.3.5 +numpy==2.4.1 # via # bitshuffle # contourpy @@ -93,7 +91,7 @@ pandas==2.3.3 # via -r base.in partd==1.4.2 # via dask -pillow==12.0.0 +pillow==12.1.0 # via matplotlib platformdirs==4.5.1 # via pooch @@ -110,7 +108,7 @@ pydantic==2.12.5 # via scippneutron pydantic-core==2.41.5 # via pydantic -pyparsing==3.2.5 +pyparsing==3.3.1 # via matplotlib python-dateutil==2.9.0.post0 # via @@ -141,7 +139,7 @@ scippnexus==25.11.0 # -r base.in # essreduce # scippneutron -scipy==1.16.3 +scipy==1.17.0 # via # scippneutron # scippnexus @@ -162,10 +160,8 @@ typing-inspection==0.4.2 # via pydantic tzdata==2025.3 # via pandas -urllib3==2.6.2 +urllib3==2.6.3 # via requests -zipp==3.23.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index 685f1892..75ea55d3 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -7,7 +7,7 @@ # cachetools==6.2.4 # via tox -certifi==2025.11.12 +certifi==2026.1.4 # via requests chardet==5.2.0 # via tox @@ -17,13 +17,13 @@ colorama==0.4.6 # via tox distlib==0.4.0 # via virtualenv -filelock==3.20.1 +filelock==3.20.3 # via # tox # virtualenv gitdb==4.0.12 # via gitpython -gitpython==3.1.45 +gitpython==3.1.46 # via -r ci.in idna==3.11 # via requests @@ -44,9 +44,9 @@ requests==2.32.5 # via -r ci.in smmap==5.0.2 # via gitdb -tox==4.32.0 +tox==4.34.1 # via -r ci.in -urllib3==2.6.2 +urllib3==2.6.3 # via requests -virtualenv==20.35.4 +virtualenv==20.36.1 # via tox diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index fe6d0b48..52c2efa7 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -12,7 +12,7 @@ -r static.txt -r test.txt -r wheels.txt -anyio==4.12.0 +anyio==4.12.1 # via # httpx # jupyter-server @@ -26,7 +26,7 @@ async-lru==2.0.5 # via jupyterlab cffi==2.0.0 # via argon2-cffi-bindings -copier==9.11.0 +copier==9.11.1 # via -r dev.in dunamai==1.25.0 # via copier @@ -44,11 +44,11 @@ isoduration==20.11.0 # via jsonschema jinja2-ansible-filters==1.3.2 # via copier -json5==0.12.1 +json5==0.13.0 # via jupyterlab-server jsonpointer==3.0.0 # via jsonschema -jsonschema[format-nongpl]==4.25.1 +jsonschema[format-nongpl]==4.26.0 # via # jupyter-events # jupyterlab-server @@ -65,7 +65,7 @@ jupyter-server==2.17.0 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.5.1 +jupyterlab==4.5.2 # via -r dev.in jupyterlab-server==2.28.0 # via jupyterlab @@ -73,15 +73,13 @@ lark==1.3.1 # via rfc3987-syntax notebook-shim==0.2.4 # via jupyterlab -overrides==7.7.0 - # via jupyter-server pip-compile-multi==3.2.2 # via -r dev.in pip-tools==7.5.2 # via pip-compile-multi plumbum==1.10.0 # via copier -prometheus-client==0.23.1 +prometheus-client==0.24.0 # via jupyter-server pycparser==2.23 # via cffi @@ -99,7 +97,7 @@ rfc3986-validator==0.1.1 # jupyter-events rfc3987-syntax==1.1.0 # via jsonschema -send2trash==1.8.3 +send2trash==2.1.0 # via jupyter-server terminado==0.18.1 # via diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 511343b1..388bfaa0 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -54,7 +54,7 @@ ipydatawidgets==4.3.5 # via pythreejs ipykernel==7.1.0 # via -r docs.in -ipython==9.8.0 +ipython==9.9.0 # via # -r docs.in # ipykernel @@ -73,11 +73,11 @@ jinja2==3.1.6 # nbconvert # nbsphinx # sphinx -jsonschema==4.25.1 +jsonschema==4.26.0 # via nbformat jsonschema-specifications==2025.9.1 # via jsonschema -jupyter-client==8.7.0 +jupyter-client==8.8.0 # via # ipykernel # nbclient @@ -108,11 +108,11 @@ mdit-py-plugins==0.5.0 # via myst-parser mdurl==0.1.2 # via markdown-it-py -mistune==3.1.4 +mistune==3.2.0 # via nbconvert myst-parser==4.0.1 # via -r docs.in -nbclient==0.10.2 +nbclient==0.10.4 # via nbconvert nbconvert==7.16.6 # via nbsphinx @@ -133,7 +133,7 @@ pexpect==4.9.0 # via ipython prompt-toolkit==3.0.52 # via ipython -psutil==7.1.3 +psutil==7.2.1 # via ipykernel ptyprocess==0.7.0 # via pexpect diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index 7c71bd8e..61506e9f 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -6,13 +6,13 @@ # requirements upgrade # -r test.txt -librt==0.7.4 +librt==0.7.7 # via mypy mypy==1.19.1 # via -r mypy.in mypy-extensions==1.1.0 # via mypy -pathspec==0.12.1 +pathspec==1.0.3 # via mypy # The following packages are considered to be unsafe in a requirements file: diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 0421653e..b205d935 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -12,7 +12,7 @@ annotated-types==0.7.0 # via pydantic bitshuffle==0.5.2 ; os_name == "posix" # via -r nightly.in -certifi==2025.11.12 +certifi==2026.1.4 # via requests charset-normalizer==3.4.4 # via requests @@ -26,7 +26,7 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -cython==3.2.3 +cython==3.2.4 # via bitshuffle dask==2025.12.0 # via -r nightly.in @@ -36,11 +36,11 @@ dnspython==2.8.0 # via email-validator email-validator==2.3.0 # via scippneutron -essreduce==25.12.1 +essreduce==26.1.0 # via -r nightly.in fonttools==4.61.1 # via matplotlib -fsspec==2025.12.0 +fsspec==2026.1.0 # via dask gemmi==0.7.4 # via -r nightly.in @@ -55,8 +55,6 @@ idna==3.11 # via # email-validator # requests -importlib-metadata==8.7.0 - # via dask iniconfig==2.3.0 # via pytest kiwisolver==1.4.10rc0 @@ -78,7 +76,7 @@ msgpack==1.1.2 # via -r nightly.in networkx==3.6.1 # via cyclebane -numpy==2.4.0rc1 +numpy==2.4.1 # via # bitshuffle # contourpy @@ -88,18 +86,18 @@ numpy==2.4.0rc1 # scipp # scippneutron # scipy -packaging==25.0 +packaging==26.0rc2 # via # dask # lazy-loader # matplotlib # pooch # pytest -pandas==3.0.0rc0 +pandas==3.0.0rc1 # via -r nightly.in partd==1.4.2 # via dask -pillow==12.0.0 +pillow==12.1.0 # via matplotlib platformdirs==4.5.1 # via pooch @@ -120,7 +118,7 @@ pydantic-core==2.41.5 # via pydantic pygments==2.19.2 # via pytest -pyparsing==3.3.0b1 +pyparsing==3.3.1 # via matplotlib pytest==9.0.2 # via -r nightly.in @@ -151,7 +149,7 @@ scippnexus @ git+https://github.com/scipp/scippnexus@main # -r nightly.in # essreduce # scippneutron -scipy==1.17.0rc1 +scipy==1.17.0 # via # scippneutron # scippnexus @@ -170,12 +168,8 @@ typing-extensions==4.15.0 # typing-inspection typing-inspection==0.4.2 # via pydantic -tzdata==2025.3 - # via pandas -urllib3==2.6.2 +urllib3==2.6.3 # via requests -zipp==3.23.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index c71d195d..75cd3c9b 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -9,11 +9,11 @@ cfgv==3.5.0 # via pre-commit distlib==0.4.0 # via virtualenv -filelock==3.20.1 +filelock==3.20.3 # via virtualenv -identify==2.6.15 +identify==2.6.16 # via pre-commit -nodeenv==1.9.1 +nodeenv==1.10.0 # via pre-commit platformdirs==4.5.1 # via virtualenv @@ -21,5 +21,5 @@ pre-commit==4.5.1 # via -r static.in pyyaml==6.0.3 # via pre-commit -virtualenv==20.35.4 +virtualenv==20.36.1 # via pre-commit diff --git a/packages/essnmx/requirements/wheels.txt b/packages/essnmx/requirements/wheels.txt index 3558aae2..3e37dfed 100644 --- a/packages/essnmx/requirements/wheels.txt +++ b/packages/essnmx/requirements/wheels.txt @@ -5,7 +5,7 @@ # # requirements upgrade # -build==1.3.0 +build==1.4.0 # via -r wheels.in packaging==25.0 # via build From ab1091745e8b36652867e59a73a17772e87cd856 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 15 Jan 2026 15:52:05 +0100 Subject: [PATCH 344/403] Use user configuration properly for time-bin-edges. --- packages/essnmx/src/ess/nmx/configurations.py | 10 +- packages/essnmx/src/ess/nmx/executables.py | 116 +++++++++++++++--- packages/essnmx/src/ess/nmx/nexus.py | 22 +++- packages/essnmx/tests/executable_test.py | 53 +++++++- 4 files changed, 177 insertions(+), 24 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/configurations.py b/packages/essnmx/src/ess/nmx/configurations.py index 2470f6b0..08e821b6 100644 --- a/packages/essnmx/src/ess/nmx/configurations.py +++ b/packages/essnmx/src/ess/nmx/configurations.py @@ -69,8 +69,14 @@ class WorkflowConfig(BaseModel): model_config = {"title": "Workflow Configuration"} time_bin_coordinate: TimeBinCoordinate = Field( title="Time Bin Coordinate", - description="Coordinate to bin the time data.", - default=TimeBinCoordinate.event_time_offset, + description="Coordinate to bin the time data. " + "Selecting `event_time_offset` means " + "reduction steps are skipped, " + "i.e. calculating `time of flight(tof)` " + "and simply saves histograms of the raw data.", + default=TimeBinCoordinate.time_of_flight, + # Default is time of flight since + # DIALS should expect the time of flight. ) nbins: int = Field( title="Number of Time Bins", diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 1c224fea..096b4ee1 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -2,11 +2,12 @@ # Copyright (c) 2025 Scipp contributors (https://github.com/scipp) import logging import pathlib +import warnings from collections.abc import Callable import scipp as sc import scippnexus as snx -from ess.reduce.nexus.types import Filename, NeXusName, SampleRun +from ess.reduce.nexus.types import Filename, NeXusName, RawDetector, SampleRun from ess.reduce.time_of_flight.types import TimeOfFlightLookupTable, TofDetector from ._executable_helper import ( @@ -15,7 +16,12 @@ collect_matching_input_files, reduction_config_from_args, ) -from .configurations import OutputConfig, ReductionConfig +from .configurations import ( + OutputConfig, + ReductionConfig, + TimeBinCoordinate, + WorkflowConfig, +) from .nexus import ( export_detector_metadata_as_nxlauetof, export_monitor_metadata_as_nxlauetof, @@ -32,6 +38,8 @@ _TOF_COORD_NAME = 'tof' """Name of the TOF coordinate used in DataArrays.""" +_ETO_COORD_NAME = 'event_time_offset' +"""Name of the Event Time Offset Coordinate used in Nexus.""" def _retrieve_input_file(input_file: list[str]) -> pathlib.Path: @@ -68,6 +76,74 @@ def _retrieve_display( return logging.getLogger(__name__).info +def _retrieve_time_bin_coordinate_name(wf_config: WorkflowConfig) -> str: + if wf_config.time_bin_coordinate == TimeBinCoordinate.time_of_flight: + return _TOF_COORD_NAME + elif wf_config.time_bin_coordinate == TimeBinCoordinate.event_time_offset: + return _ETO_COORD_NAME + + +def _build_time_bin_edges( + *, + wf_config: WorkflowConfig, + result_das: sc.DataGroup, + t_coord_name: str, +) -> sc.Variable: + # Calculate the min and max of the data itself. + da_min_t = min(da.bins.coords[t_coord_name].nanmin() for da in result_das.values()) + da_max_t = max(da.bins.coords[t_coord_name].nanmax() for da in result_das.values()) + + # Use the user-set parameters if available + # and validate them according to the data. + if wf_config.min_time_bin is not None: + min_t = sc.scalar(wf_config.min_time_bin, unit=wf_config.time_bin_unit).to( + unit=da_min_t.unit, dtype=da_min_t.dtype + ) + # If the user-set minimum time bin value + # is bigger than all time-bin-coordinate values. + if min_t >= da_max_t: + warnings.warn( + message=f"{min_t} is bigger than all " + f"{wf_config.time_bin_coordinate} values.\n" + "The histogram will all have zero values.", + category=UserWarning, + stacklevel=4, + ) + else: + min_t = da_min_t + + if wf_config.max_time_bin is not None: + max_tof = sc.scalar(wf_config.max_time_bin, unit=wf_config.time_bin_unit).to( + unit=da_max_t.unit + ) + # If the user-set maximum time bin value + # is smaller than all time-bin-coordinate values. + if max_tof <= da_min_t: + warnings.warn( + message=f"{max_tof} is smaller than all " + f"{wf_config.time_bin_coordinate} values.\n" + "The histogram will all have zero values.", + category=UserWarning, + stacklevel=3, + ) + else: + max_tof = da_max_t + + # Validate the results. + if min_t >= max_tof: + raise ValueError( + f"Minimum time bin edge, {min_t} " + "is bigger or equal than the " + f"maximum time bin edge, {max_tof}.\n" + "Cannot build a time bin edges coordinate.\n" + "Please check your configurations again." + ) + + # Build the bin-edges to histogram the results. + n_edges = wf_config.nbins + 1 + return sc.linspace(dim=t_coord_name, start=min_t, stop=max_tof, num=n_edges) + + def reduction( *, config: ReductionConfig, @@ -113,42 +189,52 @@ def reduction( base_wf = initialize_nmx_workflow(config=config.workflow) # Insert parameters and cache intermediate results base_wf[Filename[SampleRun]] = input_file_path - base_wf[TimeOfFlightLookupTable] = base_wf.compute(TimeOfFlightLookupTable) + + if config.workflow.time_bin_coordinate == TimeBinCoordinate.time_of_flight: + # We cache the time of flight look up table + # only if we need to calculate time-of-flight coordinates. + # If `event_time_offset` was requested, + # we do not have to calculate the look up table at all. + base_wf[TimeOfFlightLookupTable] = base_wf.compute(TimeOfFlightLookupTable) metadatas = base_wf.compute((NMXSampleMetadata, NMXSourceMetadata)) + tof_das = sc.DataGroup() detector_metas = sc.DataGroup() + + if config.workflow.time_bin_coordinate == TimeBinCoordinate.event_time_offset: + target_type = RawDetector[SampleRun] + elif config.workflow.time_bin_coordinate == TimeBinCoordinate.time_of_flight: + target_type = TofDetector[SampleRun] + for detector_name in detector_names: cur_wf = base_wf.copy() cur_wf[NeXusName[snx.NXdetector]] = detector_name - results = cur_wf.compute((TofDetector[SampleRun], NMXDetectorMetadata)) + results = cur_wf.compute((target_type, NMXDetectorMetadata)) detector_metas[detector_name] = results[NMXDetectorMetadata] # Binning into 1 bin and getting final tof bin edges later. - tof_das[detector_name] = results[TofDetector[SampleRun]] + tof_das[detector_name] = results[target_type] # Make tof bin edges covering all detectors - # TODO: Allow user to specify tof binning parameters from config - min_tof = min(da.bins.coords[_TOF_COORD_NAME].min() for da in tof_das.values()) - max_tof = max(da.bins.coords[_TOF_COORD_NAME].max() for da in tof_das.values()) - n_edges = config.workflow.nbins + 1 - tof_bin_edges = sc.linspace( - dim=_TOF_COORD_NAME, start=min_tof, stop=max_tof, num=n_edges + t_coord_name = _retrieve_time_bin_coordinate_name(wf_config=config.workflow) + t_bin_edges = _build_time_bin_edges( + wf_config=config.workflow, result_das=tof_das, t_coord_name=t_coord_name ) monitor_metadata = NMXMonitorMetadata( - tof_bin_coord=_TOF_COORD_NAME, + tof_bin_coord=t_coord_name, # TODO: Use real monitor data # Currently NMX simulations or experiments do not have monitors monitor_histogram=sc.DataArray( - coords={_TOF_COORD_NAME: tof_bin_edges}, - data=sc.ones_like(tof_bin_edges[:-1]), + coords={t_coord_name: t_bin_edges}, + data=sc.ones_like(t_bin_edges[:-1]), ), ) # Histogram detector counts tof_histograms = sc.DataGroup() for detector_name, tof_da in tof_das.items(): - histogram = tof_da.hist(tof=tof_bin_edges) + histogram = tof_da.hist({t_coord_name: t_bin_edges}) tof_histograms[detector_name] = histogram results = sc.DataGroup( diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/nexus.py index 8de79d4d..1402d98c 100644 --- a/packages/essnmx/src/ess/nmx/nexus.py +++ b/packages/essnmx/src/ess/nmx/nexus.py @@ -415,8 +415,20 @@ def export_reduced_data_as_nxlauetof( ) data_dset.attrs["signal"] = 1 - _create_dataset_from_var( - name='time_of_flight', - root_entry=nx_detector, - var=sc.midpoints(da.coords['tof'], dim='tof'), - ) + + if 'tof' in da.coords: + _create_dataset_from_var( + name='time_of_flight', + root_entry=nx_detector, + var=sc.midpoints(da.coords['tof'], dim='tof'), + ) + elif 'event_time_offset' in da.coords: + _create_dataset_from_var( + name='event_time_offset', + root_entry=nx_detector, + var=sc.midpoints( + da.coords['event_time_offset'], dim='event_time_offset' + ), + ) + else: + raise ValueError("Could not find time-related bin edges to store.") diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index 8b855bc2..e184f0de 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -91,7 +91,7 @@ def test_reduction_config() -> None: nbins=100, min_time_bin=10, max_time_bin=100_000, - time_bin_coordinate=TimeBinCoordinate.time_of_flight, + time_bin_coordinate=TimeBinCoordinate.event_time_offset, time_bin_unit=TimeBinUnit.us, tof_simulation_num_neutrons=700_000, tof_simulation_max_wavelength=5.0, @@ -214,7 +214,6 @@ def test_reduction_default_settings(reduction_config: ReductionConfig) -> None: def test_reduction_only_number_of_time_bins(reduction_config: ReductionConfig) -> None: reduction_config.workflow.nbins = 20 - reduction_config.workflow.time_bin_coordinate = TimeBinCoordinate.time_of_flight with known_warnings(): hist = _retrieve_one_hist(reduction(config=reduction_config)) @@ -222,6 +221,56 @@ def test_reduction_only_number_of_time_bins(reduction_config: ReductionConfig) - assert len(hist.coords['tof']) == 21 # nbins + 1 edges +def test_histogram_invalid_min_max_raises(reduction_config: ReductionConfig) -> None: + reduction_config.workflow.nbins = 20 + reduction_config.workflow.min_time_bin = 120 + reduction_config.workflow.max_time_bin = 100 + with pytest.raises(ValueError, match='Cannot build a time bin edges coordinate'): + with known_warnings(): + reduction(config=reduction_config) + + +def test_histogram_out_of_range_min_warns(reduction_config: ReductionConfig) -> None: + reduction_config.workflow.nbins = 20 + reduction_config.workflow.min_time_bin = 1_000 + reduction_config.workflow.max_time_bin = 2_000 + with pytest.warns(UserWarning, match='is bigger than all'): + with known_warnings(): + results = reduction(config=reduction_config) + + for da in results['histogram'].values(): + assert_identical( + da.data.sum(), sc.scalar(0.0, unit='counts', dtype='float32', variance=0.0) + ) + + +def test_histogram_out_of_range_max_warns(reduction_config: ReductionConfig) -> None: + reduction_config.workflow.nbins = 20 + reduction_config.workflow.min_time_bin = 1 + reduction_config.workflow.max_time_bin = 2 + with pytest.warns(UserWarning, match='is smaller than all'): + with known_warnings(): + results = reduction(config=reduction_config) + + for da in results['histogram'].values(): + assert_identical( + da.data.sum(), sc.scalar(0.0, unit='counts', dtype='float32', variance=0.0) + ) + + +def test_histogram_event_time_offset(reduction_config: ReductionConfig) -> None: + reduction_config.workflow.nbins = 20 + reduction_config.workflow.time_bin_coordinate = TimeBinCoordinate.event_time_offset + with known_warnings(): + hist = _retrieve_one_hist(reduction(config=reduction_config)) + + # Check that the number of time bins is as expected. + assert len(hist.coords['event_time_offset']) == 21 # nbins + 1 edges + # Check if the histogram result is reasonable + zero = sc.scalar(0.0, unit='counts', dtype='float32', variance=0.0) + assert bool(hist.data.sum() > zero) + + @pytest.fixture def tof_lut_file_path(tmp_path: pathlib.Path): """Fixture to provide the path to the small NMX NeXus file.""" From 676f084dccbed89c230023b434d61b2a2b43d57a Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Fri, 16 Jan 2026 09:04:50 +0100 Subject: [PATCH 345/403] Fix variable name and grammar. Co-authored-by: Neil Vaytet <39047984+nvaytet@users.noreply.github.com> --- packages/essnmx/src/ess/nmx/executables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 096b4ee1..8420e157 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -113,7 +113,7 @@ def _build_time_bin_edges( min_t = da_min_t if wf_config.max_time_bin is not None: - max_tof = sc.scalar(wf_config.max_time_bin, unit=wf_config.time_bin_unit).to( + max_t = sc.scalar(wf_config.max_time_bin, unit=wf_config.time_bin_unit).to( unit=da_max_t.unit ) # If the user-set maximum time bin value @@ -133,7 +133,7 @@ def _build_time_bin_edges( if min_t >= max_tof: raise ValueError( f"Minimum time bin edge, {min_t} " - "is bigger or equal than the " + "is bigger than or equal to the " f"maximum time bin edge, {max_tof}.\n" "Cannot build a time bin edges coordinate.\n" "Please check your configurations again." From b37dafd4fe9e624b7c6d9327439408c563a85cf1 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Fri, 16 Jan 2026 09:33:35 +0100 Subject: [PATCH 346/403] Fix upper bin edge and update warn stack level to be consistent. --- packages/essnmx/src/ess/nmx/executables.py | 61 +++++++++++++--------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 8420e157..00287488 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -5,6 +5,7 @@ import warnings from collections.abc import Callable +import numpy as np import scipp as sc import scippnexus as snx from ess.reduce.nexus.types import Filename, NeXusName, RawDetector, SampleRun @@ -83,6 +84,22 @@ def _retrieve_time_bin_coordinate_name(wf_config: WorkflowConfig) -> str: return _ETO_COORD_NAME +def _warn_bin_edge_out_of_range( + *, edge: sc.Variable, coord_name: str, desc: str +) -> None: + warnings.warn( + message=f"{edge} is {desc} than all " + f"{coord_name} values.\n" + "The histogram will all have zero values.", + category=UserWarning, + stacklevel=4, + ) + + +def _match_data_unit_dtype(config_var: sc.Variable, da: sc.Variable) -> sc.Variable: + return config_var.to(unit=da.unit, dtype=da.dtype) + + def _build_time_bin_edges( *, wf_config: WorkflowConfig, @@ -95,53 +112,49 @@ def _build_time_bin_edges( # Use the user-set parameters if available # and validate them according to the data. + # Lower Time Bin Edge if wf_config.min_time_bin is not None: - min_t = sc.scalar(wf_config.min_time_bin, unit=wf_config.time_bin_unit).to( - unit=da_min_t.unit, dtype=da_min_t.dtype - ) + min_t = sc.scalar(wf_config.min_time_bin, unit=wf_config.time_bin_unit) + min_t = _match_data_unit_dtype(min_t, da=da_min_t) # If the user-set minimum time bin value # is bigger than all time-bin-coordinate values. - if min_t >= da_max_t: - warnings.warn( - message=f"{min_t} is bigger than all " - f"{wf_config.time_bin_coordinate} values.\n" - "The histogram will all have zero values.", - category=UserWarning, - stacklevel=4, + if min_t > da_max_t: + _warn_bin_edge_out_of_range( + edge=min_t, coord_name=wf_config.time_bin_coordinate, desc='bigger' ) else: min_t = da_min_t + # Upper Time Bin Edge if wf_config.max_time_bin is not None: - max_t = sc.scalar(wf_config.max_time_bin, unit=wf_config.time_bin_unit).to( - unit=da_max_t.unit - ) + max_t = sc.scalar(wf_config.max_time_bin, unit=wf_config.time_bin_unit) + max_t = _match_data_unit_dtype(max_t, da=da_max_t) # If the user-set maximum time bin value # is smaller than all time-bin-coordinate values. - if max_tof <= da_min_t: - warnings.warn( - message=f"{max_tof} is smaller than all " - f"{wf_config.time_bin_coordinate} values.\n" - "The histogram will all have zero values.", - category=UserWarning, - stacklevel=3, + if max_t <= da_min_t: + _warn_bin_edge_out_of_range( + edge=max_t, coord_name=wf_config.time_bin_coordinate, desc='smaller' ) else: - max_tof = da_max_t + max_t = da_max_t + + # Avoid dropping the event that has the exact same + # `event_time_offset`` or `tof` value as the upper bin edge. + max_t.value = np.nextafter(max_t.value, np.inf) # Validate the results. - if min_t >= max_tof: + if min_t >= max_t: raise ValueError( f"Minimum time bin edge, {min_t} " "is bigger than or equal to the " - f"maximum time bin edge, {max_tof}.\n" + f"maximum time bin edge, {max_t}.\n" "Cannot build a time bin edges coordinate.\n" "Please check your configurations again." ) # Build the bin-edges to histogram the results. n_edges = wf_config.nbins + 1 - return sc.linspace(dim=t_coord_name, start=min_t, stop=max_tof, num=n_edges) + return sc.linspace(dim=t_coord_name, start=min_t, stop=max_t, num=n_edges) def reduction( From 7749cd8e29a4a94841e8ba785f71c187df4c7406 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Fri, 16 Jan 2026 10:59:55 +0100 Subject: [PATCH 347/403] Save lookup table only if time_of_flight was calculated. --- packages/essnmx/src/ess/nmx/executables.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 00287488..834888fa 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -256,8 +256,11 @@ def reduction( sample=metadatas[NMXSampleMetadata], source=metadatas[NMXSourceMetadata], monitor=monitor_metadata, - lookup_table=base_wf.compute(TimeOfFlightLookupTable), ) + + if config.workflow.time_bin_coordinate == TimeBinCoordinate.time_of_flight: + results["lookup_table"] = base_wf.compute(TimeOfFlightLookupTable) + if not config.output.skip_file_output: save_results(results=results, output_config=config.output) From 14a9ca9cfef6f06421867c884fdb1ea3960419f3 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Fri, 16 Jan 2026 11:00:07 +0100 Subject: [PATCH 348/403] Add more tests using eto/tof --- packages/essnmx/tests/executable_test.py | 62 ++++++++++++++++-------- 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index e184f0de..36881c3b 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -221,8 +221,20 @@ def test_reduction_only_number_of_time_bins(reduction_config: ReductionConfig) - assert len(hist.coords['tof']) == 21 # nbins + 1 edges -def test_histogram_invalid_min_max_raises(reduction_config: ReductionConfig) -> None: +def test_histogram_event_time_offset(reduction_config: ReductionConfig) -> None: reduction_config.workflow.nbins = 20 + reduction_config.workflow.time_bin_coordinate = TimeBinCoordinate.event_time_offset + with known_warnings(): + hist = _retrieve_one_hist(reduction(config=reduction_config)) + + # Check that the number of time bins is as expected. + assert len(hist.coords['event_time_offset']) == 21 # nbins + 1 edges + # Check if the histogram result is reasonable + zero = sc.scalar(0.0, unit='counts', dtype='float32', variance=0.0) + assert bool(hist.data.sum() > zero) + + +def test_histogram_invalid_min_max_raises(reduction_config: ReductionConfig) -> None: reduction_config.workflow.min_time_bin = 120 reduction_config.workflow.max_time_bin = 100 with pytest.raises(ValueError, match='Cannot build a time bin edges coordinate'): @@ -230,7 +242,25 @@ def test_histogram_invalid_min_max_raises(reduction_config: ReductionConfig) -> reduction(config=reduction_config) -def test_histogram_out_of_range_min_warns(reduction_config: ReductionConfig) -> None: +def test_histogram_invalid_min_max_raises_eto( + reduction_config: ReductionConfig, +) -> None: + reduction_config.workflow.time_bin_coordinate = TimeBinCoordinate.event_time_offset + reduction_config.workflow.min_time_bin = 50 + reduction_config.workflow.max_time_bin = 40 + with pytest.raises(ValueError, match='Cannot build a time bin edges coordinate'): + with known_warnings(): + reduction(config=reduction_config) + + +@pytest.mark.parametrize( + argnames="t_coord", + argvalues=[TimeBinCoordinate.time_of_flight, TimeBinCoordinate.event_time_offset], +) +def test_histogram_out_of_range_min_warns( + reduction_config: ReductionConfig, t_coord: TimeBinCoordinate +) -> None: + reduction_config.workflow.time_bin_coordinate = t_coord reduction_config.workflow.nbins = 20 reduction_config.workflow.min_time_bin = 1_000 reduction_config.workflow.max_time_bin = 2_000 @@ -244,10 +274,17 @@ def test_histogram_out_of_range_min_warns(reduction_config: ReductionConfig) -> ) -def test_histogram_out_of_range_max_warns(reduction_config: ReductionConfig) -> None: - reduction_config.workflow.nbins = 20 - reduction_config.workflow.min_time_bin = 1 - reduction_config.workflow.max_time_bin = 2 +@pytest.mark.parametrize( + argnames="t_coord", + argvalues=[TimeBinCoordinate.time_of_flight, TimeBinCoordinate.event_time_offset], +) +def test_histogram_out_of_range_max_warns( + reduction_config: ReductionConfig, t_coord: TimeBinCoordinate +) -> None: + reduction_config.workflow.time_bin_coordinate = t_coord + reduction_config.workflow.nbins = 10 + reduction_config.workflow.min_time_bin = -1 + reduction_config.workflow.max_time_bin = 0 with pytest.warns(UserWarning, match='is smaller than all'): with known_warnings(): results = reduction(config=reduction_config) @@ -258,19 +295,6 @@ def test_histogram_out_of_range_max_warns(reduction_config: ReductionConfig) -> ) -def test_histogram_event_time_offset(reduction_config: ReductionConfig) -> None: - reduction_config.workflow.nbins = 20 - reduction_config.workflow.time_bin_coordinate = TimeBinCoordinate.event_time_offset - with known_warnings(): - hist = _retrieve_one_hist(reduction(config=reduction_config)) - - # Check that the number of time bins is as expected. - assert len(hist.coords['event_time_offset']) == 21 # nbins + 1 edges - # Check if the histogram result is reasonable - zero = sc.scalar(0.0, unit='counts', dtype='float32', variance=0.0) - assert bool(hist.data.sum() > zero) - - @pytest.fixture def tof_lut_file_path(tmp_path: pathlib.Path): """Fixture to provide the path to the small NMX NeXus file.""" From b402f35c78e00ec54d495ddb8a21542bef664c6f Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 15 Jan 2026 16:00:31 +0100 Subject: [PATCH 349/403] Fail earlier when output file exists. --- packages/essnmx/src/ess/nmx/executables.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 834888fa..ce182148 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -24,6 +24,7 @@ WorkflowConfig, ) from .nexus import ( + _check_file, export_detector_metadata_as_nxlauetof, export_monitor_metadata_as_nxlauetof, export_reduced_data_as_nxlauetof, @@ -189,6 +190,9 @@ def reduction( A DataGroup containing the reduced data for each selected detector. """ + # Check the file output configuration before we start heavy computation. + _check_file(config.output.output_file, config.output.overwrite) + display = _retrieve_display(logger, display) input_file_path = _retrieve_input_file(config.inputs.input_file).resolve() display(f"Input file: {input_file_path}") From f15196d22246bbbe30b4d9d85d113bb1d624436a Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Fri, 16 Jan 2026 09:41:44 +0100 Subject: [PATCH 350/403] Check output file path only if it is going to write into them. --- packages/essnmx/src/ess/nmx/executables.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index ce182148..8dac1fbf 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -191,7 +191,8 @@ def reduction( """ # Check the file output configuration before we start heavy computation. - _check_file(config.output.output_file, config.output.overwrite) + if not config.output.skip_file_output: + _check_file(config.output.output_file, config.output.overwrite) display = _retrieve_display(logger, display) input_file_path = _retrieve_input_file(config.inputs.input_file).resolve() From e01192d121e1eb5e01b2a864ebe57bb48a0f696a Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Fri, 16 Jan 2026 10:21:44 +0100 Subject: [PATCH 351/403] Add tests for the file existence check. --- packages/essnmx/tests/executable_test.py | 36 ++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index 36881c3b..fb5aa7de 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -345,3 +345,39 @@ def test_reduction_with_tof_lut_file( tof_edges = hist.coords['tof'] assert_identical(default_hist.data, hist.data) assert_identical(tof_edges_default * 2, tof_edges) + + +def test_reduction_suceed_when_skipping_evenif_output_file_exists( + reduction_config: ReductionConfig, temp_output_file: pathlib.Path +) -> None: + # Make sure the file exists + temp_output_file.touch(exist_ok=True) + # Make sure the file output is skipped. + reduction_config.output.skip_file_output = True + + # Adjust workflow setting to finish fast. + reduction_config.workflow.nbins = 2 + reduction_config.workflow.time_bin_coordinate = TimeBinCoordinate.event_time_offset + with known_warnings(): + reduction(config=reduction_config) + + +def test_reduction_fails_fast_if_output_file_exists( + reduction_config: ReductionConfig, temp_output_file: pathlib.Path +) -> None: + import time + + # Make sure the file exists + temp_output_file.touch() + # Make sure file output is NOT skipped. + reduction_config.output.skip_file_output = False + + start = time.time() + with pytest.raises(FileExistsError): + reduction(config=reduction_config) + finish = time.time() + + # Check if the `reduction` call fails within 1 second. + # There is no special reason why it is 1 second. + # It should just fail as fast as possible. + assert finish - start < 1 From 51e07a98ddf21c9bfb0d82729043ab455a472eaf Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Fri, 16 Jan 2026 10:37:51 +0100 Subject: [PATCH 352/403] Fix typo --- packages/essnmx/tests/executable_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index fb5aa7de..2443cb20 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -347,7 +347,7 @@ def test_reduction_with_tof_lut_file( assert_identical(tof_edges_default * 2, tof_edges) -def test_reduction_suceed_when_skipping_evenif_output_file_exists( +def test_reduction_succeed_when_skipping_evenif_output_file_exists( reduction_config: ReductionConfig, temp_output_file: pathlib.Path ) -> None: # Make sure the file exists From f02e9f0df15be357567bcca3d61d0c384fb2ea97 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Fri, 16 Jan 2026 15:27:29 +0100 Subject: [PATCH 353/403] Use scippnexus to write files - static metadata. --- packages/essnmx/src/ess/nmx/nexus.py | 86 +++--------------------- packages/essnmx/src/ess/nmx/types.py | 43 +++++++++++- packages/essnmx/src/ess/nmx/workflows.py | 10 ++- 3 files changed, 61 insertions(+), 78 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/nexus.py index 1402d98c..a7f72990 100644 --- a/packages/essnmx/src/ess/nmx/nexus.py +++ b/packages/essnmx/src/ess/nmx/nexus.py @@ -8,6 +8,7 @@ import h5py import numpy as np import scipp as sc +import scippnexus as snx from .configurations import Compression from .types import ( @@ -89,16 +90,6 @@ def _retrieve_compression_arguments(compress_mode: Compression) -> dict: return {"compression": compression_filter, "compression_opts": compression_opts} -def _create_lauetof_data_entry(file_obj: h5py.File) -> h5py.Group: - nx_entry = file_obj.create_group("entry") - nx_entry.attrs["NX_class"] = "NXentry" - return nx_entry - - -def _add_lauetof_definition(nx_entry: h5py.Group) -> None: - _create_dataset_from_string(root_entry=nx_entry, name="definition", var="NXlauetof") - - def _add_lauetof_instrument(nx_entry: h5py.Group) -> h5py.Group: nx_instrument = nx_entry.create_group("instrument") nx_instrument.attrs["NX_class"] = "NXinstrument" @@ -106,25 +97,6 @@ def _add_lauetof_instrument(nx_entry: h5py.Group) -> h5py.Group: return nx_instrument -def _add_lauetof_source_group( - source_position: sc.Variable, nx_instrument: h5py.Group -) -> None: - nx_source = nx_instrument.create_group("source") - nx_source.attrs["NX_class"] = "NXsource" - _create_dataset_from_string( - root_entry=nx_source, name="name", var="European Spallation Source" - ) - _create_dataset_from_string(root_entry=nx_source, name="short_name", var="ESS") - _create_dataset_from_string( - root_entry=nx_source, name="type", var="Spallation Neutron Source" - ) - _create_dataset_from_var( - root_entry=nx_source, name="distance", var=sc.norm(source_position) - ) - # Legacy probe information. - _create_dataset_from_string(root_entry=nx_source, name="probe", var="neutron") - - def _add_lauetof_detector_group( *, detector_name: str, @@ -156,37 +128,6 @@ def _add_lauetof_detector_group( _create_dataset_from_var(root_entry=nx_det, name="slow_axis", var=slow_axis) -def _add_lauetof_sample_group( - *, - crystal_rotation: sc.Variable, - sample_name: str | sc.Variable, - sample_orientation_matrix: sc.Variable, - sample_unit_cell: sc.Variable, - nx_entry: h5py.Group, -) -> None: - nx_sample = nx_entry.create_group("sample") - nx_sample.attrs["NX_class"] = "NXsample" - _create_dataset_from_var( - root_entry=nx_sample, - var=crystal_rotation, - name='crystal_rotation', - long_name='crystal rotation in Phi (XYZ)', - ) - _create_dataset_from_string( - root_entry=nx_sample, - name='name', - var=sample_name if isinstance(sample_name, str) else sample_name.value, - ) - _create_dataset_from_var( - name='orientation_matrix', root_entry=nx_sample, var=sample_orientation_matrix - ) - _create_dataset_from_var( - name='unit_cell', - root_entry=nx_sample, - var=sample_unit_cell, - ) - - def _add_arbitrary_metadata( nx_entry: h5py.Group, **arbitrary_metadata: sc.Variable ) -> None: @@ -240,22 +181,15 @@ def export_static_metadata_as_nxlauetof( """ _check_file(output_file, overwrite=overwrite) - with h5py.File(output_file, "w") as f: - f.attrs["NX_class"] = "NXlauetof" - nx_entry = _create_lauetof_data_entry(f) - _add_lauetof_definition(nx_entry) - _add_lauetof_sample_group( - crystal_rotation=sample_metadata.crystal_rotation, - sample_name=sample_metadata.sample_name, - sample_orientation_matrix=sample_metadata.sample_orientation_matrix, - sample_unit_cell=sample_metadata.sample_unit_cell, - nx_entry=nx_entry, - ) - nx_instrument = _add_lauetof_instrument(nx_entry) - _add_lauetof_source_group(source_metadata.source_position, nx_instrument) - # Skipping ``NXdata``(name) field with data link - # Add arbitrary metadata - _add_arbitrary_metadata(nx_entry, **arbitrary_metadata) + with snx.File(output_file, "w") as f: + f._group.attrs["NX_class"] = "NXlauetof" + nx_entry = f.create_class(name='entry', class_name='NXlauetof') + nx_entry.create_field('definitions', value='NXlauetof') + nx_entry['sample'] = sample_metadata + + nx_instrument = nx_entry.create_class('instrument', snx.NXinstrument) + nx_instrument['source'] = source_metadata + _add_arbitrary_metadata(nx_entry._group, **arbitrary_metadata) def export_monitor_metadata_as_nxlauetof( diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 0336ec7c..40a15b51 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -2,7 +2,22 @@ from dataclasses import dataclass, field from typing import NewType +import h5py +import numpy as np import scipp as sc +import scippnexus as snx + + +def _create_field( + group: snx.typing.H5Group, + name: str, + data: np.ndarray | sc.Variable, + long_name: str = '', + **kwargs, +) -> None: + new_field = snx.create_field(group, name, data, **kwargs) + if long_name: + new_field.attrs['long_name'] = long_name class Compression(enum.StrEnum): @@ -24,9 +39,11 @@ class Compression(enum.StrEnum): @dataclass(kw_only=True) class NMXSampleMetadata: + nx_class = snx.NXsample + crystal_rotation: sc.Variable sample_position: sc.Variable - sample_name: sc.Variable | str + sample_name: str # Temporarily hardcoding some values # TODO: Remove hardcoded values sample_orientation_matrix: sc.Variable = field( @@ -45,11 +62,35 @@ class NMXSampleMetadata: ) ) + def __write_to_nexus_group__(self, group: h5py.Group): + _create_field( + group, + 'crystal_rotation', + self.crystal_rotation, + long_name='crystal rotation in Phi (XYZ)', + ) + _create_field( + group, + 'name', + self.sample_name + if isinstance(self.sample_name, str) + else self.sample_name.value, + ) + _create_field(group, 'orientation_matrix', self.sample_orientation_matrix) + _create_field(group, 'unit_cell', self.sample_unit_cell) + @dataclass(kw_only=True) class NMXSourceMetadata: + nx_class = snx.NXsource source_position: sc.Variable + def __write_to_nexus_group__(self, group: h5py.Group): + _create_field(group, 'name', 'European Spallation Source') + _create_field(group, 'type', 'Spallation Neutron Source') + _create_field(group, 'distance', sc.norm(self.source_position)) + _create_field(group, 'probe', 'neutron') + @dataclass(kw_only=True) class NMXMonitorMetadata: diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index c1506e92..e9b65f52 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -111,8 +111,16 @@ def assemble_sample_metadata( sample_component: NeXusComponent[snx.NXsample, SampleRun], ) -> NMXSampleMetadata: """Assemble sample metadata for NMX reduction workflow.""" + name = sample_component['name'] + if isinstance(name, sc.Variable) and name.dtype is str: + sample_name = name.value + elif isinstance(name, str): + sample_name = name + else: + raise TypeError(f'Sample name {name}is in a wrong type: ', type(name)) + return NMXSampleMetadata( - sample_name=sample_component['name'], + sample_name=sample_name, crystal_rotation=crystal_rotation, sample_position=sample_position, ) From 75c03a36f00bb0a6b5f989977e94c9077fc5f4b7 Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 19 Jan 2026 13:06:59 +0100 Subject: [PATCH 354/403] Fix dtype checking logic [skip ci] Co-authored-by: Jan-Lukas Wynen --- packages/essnmx/src/ess/nmx/workflows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index e9b65f52..9f3cf167 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -112,7 +112,7 @@ def assemble_sample_metadata( ) -> NMXSampleMetadata: """Assemble sample metadata for NMX reduction workflow.""" name = sample_component['name'] - if isinstance(name, sc.Variable) and name.dtype is str: + if isinstance(name, sc.Variable) and name.dtype == str: sample_name = name.value elif isinstance(name, str): sample_name = name From 9210b27a06a8d445e88e7e036dae8a23d3815ce2 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 19 Jan 2026 13:08:10 +0100 Subject: [PATCH 355/403] Remove helper function and use snx create_field directly. --- packages/essnmx/src/ess/nmx/types.py | 41 ++++++---------------------- 1 file changed, 9 insertions(+), 32 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 40a15b51..771a5a36 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -3,23 +3,10 @@ from typing import NewType import h5py -import numpy as np import scipp as sc import scippnexus as snx -def _create_field( - group: snx.typing.H5Group, - name: str, - data: np.ndarray | sc.Variable, - long_name: str = '', - **kwargs, -) -> None: - new_field = snx.create_field(group, name, data, **kwargs) - if long_name: - new_field.attrs['long_name'] = long_name - - class Compression(enum.StrEnum): """Compression type of the output file. @@ -63,21 +50,11 @@ class NMXSampleMetadata: ) def __write_to_nexus_group__(self, group: h5py.Group): - _create_field( - group, - 'crystal_rotation', - self.crystal_rotation, - long_name='crystal rotation in Phi (XYZ)', - ) - _create_field( - group, - 'name', - self.sample_name - if isinstance(self.sample_name, str) - else self.sample_name.value, - ) - _create_field(group, 'orientation_matrix', self.sample_orientation_matrix) - _create_field(group, 'unit_cell', self.sample_unit_cell) + cr_field = snx.create_field(group, 'crystal_rotation', self.crystal_rotation) + cr_field.attrs['long_name'] = 'crystal rotation in Phi (XYZ)' + snx.create_field(group, 'name', self.sample_name) + snx.create_field(group, 'orientation_matrix', self.sample_orientation_matrix) + snx.create_field(group, 'unit_cell', self.sample_unit_cell) @dataclass(kw_only=True) @@ -86,10 +63,10 @@ class NMXSourceMetadata: source_position: sc.Variable def __write_to_nexus_group__(self, group: h5py.Group): - _create_field(group, 'name', 'European Spallation Source') - _create_field(group, 'type', 'Spallation Neutron Source') - _create_field(group, 'distance', sc.norm(self.source_position)) - _create_field(group, 'probe', 'neutron') + snx.create_field(group, 'name', 'European Spallation Source') + snx.create_field(group, 'type', 'Spallation Neutron Source') + snx.create_field(group, 'distance', sc.norm(self.source_position)) + snx.create_field(group, 'probe', 'neutron') @dataclass(kw_only=True) From fd2f4c6d7212a4874dd5fbe2d918dc326ab47e02 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 19 Jan 2026 14:28:44 +0100 Subject: [PATCH 356/403] Use scippnexus to write NXlauetof file. --- packages/essnmx/src/ess/nmx/nexus.py | 97 ++-------------------------- packages/essnmx/src/ess/nmx/types.py | 22 +++++++ 2 files changed, 29 insertions(+), 90 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/nexus.py index a7f72990..8384c72e 100644 --- a/packages/essnmx/src/ess/nmx/nexus.py +++ b/packages/essnmx/src/ess/nmx/nexus.py @@ -31,10 +31,6 @@ def _check_file( return filename -def _create_dataset_from_string(*, root_entry: h5py.Group, name: str, var: str) -> None: - root_entry.create_dataset(name, dtype=h5py.string_dtype(), data=var) - - def _create_dataset_from_var( *, root_entry: h5py.Group, @@ -90,44 +86,6 @@ def _retrieve_compression_arguments(compress_mode: Compression) -> dict: return {"compression": compression_filter, "compression_opts": compression_opts} -def _add_lauetof_instrument(nx_entry: h5py.Group) -> h5py.Group: - nx_instrument = nx_entry.create_group("instrument") - nx_instrument.attrs["NX_class"] = "NXinstrument" - _create_dataset_from_string(root_entry=nx_instrument, name="name", var="NMX") - return nx_instrument - - -def _add_lauetof_detector_group( - *, - detector_name: str, - x_pixel_size: sc.Variable, - y_pixel_size: sc.Variable, - origin_position: sc.Variable, - fast_axis: sc.Variable, - slow_axis: sc.Variable, - distance: sc.Variable, - polar_angle: sc.Variable, - azimuthal_angle: sc.Variable, - nx_instrument: h5py.Group, -) -> None: - nx_det = nx_instrument.create_group(detector_name) # Detector name - nx_det.attrs["NX_class"] = "NXdetector" - _create_dataset_from_var(name="polar_angle", root_entry=nx_det, var=polar_angle) - _create_dataset_from_var( - name="azimuthal_angle", root_entry=nx_det, var=azimuthal_angle - ) - _create_dataset_from_var(name="x_pixel_size", root_entry=nx_det, var=x_pixel_size) - _create_dataset_from_var(name="y_pixel_size", root_entry=nx_det, var=y_pixel_size) - _create_dataset_from_var(name="distance", root_entry=nx_det, var=distance) - # Legacy geometry information until we have a better way to store it - _create_dataset_from_var(name="origin", root_entry=nx_det, var=origin_position) - # Fast axis, along where the pixel ID increases by 1 - _create_dataset_from_var(root_entry=nx_det, name="fast_axis", var=fast_axis) - # Slow axis, along where the pixel ID increases - # by the number of pixels in the fast axis - _create_dataset_from_var(root_entry=nx_det, name="slow_axis", var=slow_axis) - - def _add_arbitrary_metadata( nx_entry: h5py.Group, **arbitrary_metadata: sc.Variable ) -> None: @@ -213,14 +171,9 @@ def export_monitor_metadata_as_nxlauetof( if not append_mode: raise NotImplementedError("Only append mode is supported for now.") - with h5py.File(output_file, "r+") as f: + with snx.File(output_file, "r+") as f: nx_entry = f["entry"] - # Placeholder for ``monitor`` group - _add_lauetof_monitor_group( - tof_bin_coord=monitor_metadata.tof_bin_coord, - monitor_histogram=monitor_metadata.monitor_histogram, - nx_entry=nx_entry, - ) + nx_entry["control"] = monitor_metadata def export_detector_metadata_as_nxlauetof( @@ -245,51 +198,15 @@ def export_detector_metadata_as_nxlauetof( if not append_mode: raise NotImplementedError("Only append mode is supported for now.") - with h5py.File(output_file, "r+") as f: - nx_entry = f["entry"] + with snx.File(output_file, "r+") as f: + nx_entry: snx.Group = f["entry"] if "instrument" not in nx_entry: - nx_instrument = _add_lauetof_instrument(f["entry"]) + nx_instrument = nx_entry.create_class("instrument", 'NXinstrument') + nx_instrument.create_field(key='name', value='NMX') else: nx_instrument = nx_entry["instrument"] - # Add detector group metadata - _add_lauetof_detector_group( - detector_name=detector_metadata.detector_name, - x_pixel_size=detector_metadata.x_pixel_size, - y_pixel_size=detector_metadata.y_pixel_size, - origin_position=detector_metadata.origin_position, - fast_axis=detector_metadata.fast_axis, - slow_axis=detector_metadata.slow_axis, - distance=detector_metadata.distance, - polar_angle=detector_metadata.polar_angle, - azimuthal_angle=detector_metadata.azimuthal_angle, - nx_instrument=nx_instrument, - ) - - -def _add_lauetof_monitor_group( - *, - tof_bin_coord: str, - monitor_histogram: sc.DataArray, - nx_entry: h5py.Group, -) -> None: - nx_monitor = nx_entry.create_group("control") - nx_monitor.attrs["NX_class"] = "NXmonitor" - _create_dataset_from_string(root_entry=nx_monitor, name='mode', var='monitor') - nx_monitor["preset"] = 0.0 # Check if this is the correct value - data_dset = _create_dataset_from_var( - name='data', - root_entry=nx_monitor, - var=monitor_histogram.data, - ) - data_dset.attrs["signal"] = 1 - data_dset.attrs["primary"] = 1 - - _create_dataset_from_var( - name='time_of_flight', - root_entry=nx_monitor, - var=monitor_histogram.coords[tof_bin_coord], - ) + nx_instrument[detector_metadata.detector_name] = detector_metadata def export_reduced_data_as_nxlauetof( diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 771a5a36..e3a4e24e 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -71,6 +71,7 @@ def __write_to_nexus_group__(self, group: h5py.Group): @dataclass(kw_only=True) class NMXMonitorMetadata: + nx_class = snx.NXmonitor monitor_histogram: sc.DataArray tof_bin_coord: str = field( default='tof', @@ -80,9 +81,21 @@ class NMXMonitorMetadata: }, ) + def __write_to_nexus_group__(self, group: h5py.Group): + snx.create_field(group, 'mode', 'monitor') + snx.create_field(group, 'preset', 0.0) + data_field = snx.create_field(group, 'data', self.monitor_histogram.data) + data_field.attrs['signal'] = 1 + data_field.attrs['primary'] = 1 + snx.create_field( + group, 'time_of_flight', self.monitor_histogram.coords[self.tof_bin_coord] + ) + @dataclass(kw_only=True) class NMXDetectorMetadata: + nx_class = snx.NXdetector + detector_name: str x_pixel_size: sc.Variable y_pixel_size: sc.Variable @@ -95,3 +108,12 @@ class NMXDetectorMetadata: azimuthal_angle: sc.Variable = field( default_factory=lambda: sc.scalar(0, unit='deg') ) + + def __write_to_nexus_group__(self, group: h5py.Group): + snx.create_field(group, 'polar_angle', self.polar_angle) + snx.create_field(group, 'azimuthal_angle', self.azimuthal_angle) + snx.create_field(group, 'x_pixel_size', self.x_pixel_size) + snx.create_field(group, 'distance', self.distance) + snx.create_field(group, 'origin', self.origin_position) + snx.create_field(group, 'fast_axis', self.fast_axis) + snx.create_field(group, 'slow_axis', self.slow_axis) From 1f68210a23a3098a34b22057133dae6a14b5b4b9 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 19 Jan 2026 14:46:21 +0100 Subject: [PATCH 357/403] Reorganize field/class creation and add minimum checks of mandatory fields. --- packages/essnmx/src/ess/nmx/nexus.py | 25 +++++++++++++++++------- packages/essnmx/src/ess/nmx/types.py | 7 ++++--- packages/essnmx/tests/executable_test.py | 12 ++++++++++++ 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/nexus.py index 8384c72e..7e074787 100644 --- a/packages/essnmx/src/ess/nmx/nexus.py +++ b/packages/essnmx/src/ess/nmx/nexus.py @@ -108,6 +108,22 @@ def _add_arbitrary_metadata( ) +def _set_default_instrument(nx_entry: snx.Group) -> snx.Group: + """Return NXinstrument group. + + If 'instrument' exists in the NXentry group, it returns the existing one. + Otherwise, new NXinstrument group is created and returned. + The default NXinstrument group has a field 'name' with the instrument name, 'NMX'. + """ + if "instrument" not in nx_entry: + nx_instrument = nx_entry.create_class("instrument", 'NXinstrument') + nx_instrument.create_field(key='name', value='NMX') + else: + nx_instrument = nx_entry["instrument"] + + return nx_instrument + + def export_static_metadata_as_nxlauetof( *, sample_metadata: NMXSampleMetadata, @@ -145,7 +161,7 @@ def export_static_metadata_as_nxlauetof( nx_entry.create_field('definitions', value='NXlauetof') nx_entry['sample'] = sample_metadata - nx_instrument = nx_entry.create_class('instrument', snx.NXinstrument) + nx_instrument = _set_default_instrument(nx_entry) nx_instrument['source'] = source_metadata _add_arbitrary_metadata(nx_entry._group, **arbitrary_metadata) @@ -200,12 +216,7 @@ def export_detector_metadata_as_nxlauetof( with snx.File(output_file, "r+") as f: nx_entry: snx.Group = f["entry"] - if "instrument" not in nx_entry: - nx_instrument = nx_entry.create_class("instrument", 'NXinstrument') - nx_instrument.create_field(key='name', value='NMX') - else: - nx_instrument = nx_entry["instrument"] - + nx_instrument = _set_default_instrument(nx_entry) nx_instrument[detector_metadata.detector_name] = detector_metadata diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index e3a4e24e..cb03de1a 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -110,10 +110,11 @@ class NMXDetectorMetadata: ) def __write_to_nexus_group__(self, group: h5py.Group): - snx.create_field(group, 'polar_angle', self.polar_angle) - snx.create_field(group, 'azimuthal_angle', self.azimuthal_angle) snx.create_field(group, 'x_pixel_size', self.x_pixel_size) - snx.create_field(group, 'distance', self.distance) + snx.create_field(group, 'y_pixel_size', self.y_pixel_size) snx.create_field(group, 'origin', self.origin_position) snx.create_field(group, 'fast_axis', self.fast_axis) snx.create_field(group, 'slow_axis', self.slow_axis) + snx.create_field(group, 'distance', self.distance) + snx.create_field(group, 'polar_angle', self.polar_angle) + snx.create_field(group, 'azimuthal_angle', self.azimuthal_angle) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index 2443cb20..0182666b 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -136,13 +136,25 @@ def small_nmx_nexus_path(): def _check_output_file(output_file_path: pathlib.Path, nbins: int): detector_names = [f'detector_panel_{i}' for i in range(3)] + mandatory_fields = ( + 'data', + 'distance', + 'fast_axis', + 'slow_axis', + 'origin', + 'x_pixel_size', + 'y_pixel_size', + 'origin', + ) with snx.File(output_file_path, 'r') as f: # Test + assert f['entry/instrument/name'][()] == 'NMX' for name in detector_names: det_gr = f[f'entry/instrument/{name}'] assert det_gr is not None toa_edges = det_gr['time_of_flight'][()] assert len(toa_edges) == nbins + assert all(field_name in det_gr for field_name in mandatory_fields) def test_executable_runs(small_nmx_nexus_path, tmp_path: pathlib.Path): From 081aab07a60f6669fdad8688418164a96d1ea153 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 21 Jan 2026 13:58:35 +0100 Subject: [PATCH 358/403] More explicit compression mode selection. --- packages/essnmx/src/ess/nmx/nexus.py | 57 ++++++++++++++++++---------- packages/essnmx/src/ess/nmx/types.py | 1 + 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/nexus.py index 7e074787..ee0c90f0 100644 --- a/packages/essnmx/src/ess/nmx/nexus.py +++ b/packages/essnmx/src/ess/nmx/nexus.py @@ -62,6 +62,12 @@ def _create_dataset_from_var( def _retrieve_compression_arguments(compress_mode: Compression) -> dict: + """Returns compression filter and opts arguments for the ``compress_mode``. + + Returns an empty dictionary if an unimplemented compression mode + or `NONE` compression mode is selected. + + """ if compress_mode == Compression.BITSHUFFLE_LZ4: try: import bitshuffle.h5 @@ -79,9 +85,21 @@ def _retrieve_compression_arguments(compress_mode: Compression) -> dict: ) compression_filter = "gzip" compression_opts = 4 + elif compress_mode == Compression.GZIP: + compression_filter = "gzip" + compression_opts = 4 + elif compress_mode == Compression.NONE: + return {} else: - compression_filter = None - compression_opts = None + warnings.warn( + UserWarning( + f"Compression Mode {compress_mode} is not implemented yet. " + "Not Compressing the dataset... " + "Try `GZIP` or `BITSHUFFLE_LZ4` if compression is needed." + ), + stacklevel=2, + ) + return {} return {"compression": compression_filter, "compression_opts": compression_opts} @@ -246,9 +264,13 @@ def export_reduced_data_as_nxlauetof( If ``False``, the file is opened in None-append mode. > None-append mode is not supported for now. > Only append mode is supported for now. - compress_counts: - If ``True``, the detector counts are compressed using bitshuffle. + compress_mode: + The detector counts are compressed using the ``compress_mode``. It is because only the detector counts are expected to be large. + If ``Compression.BITSHUFFLE_LZ4`` is selected + but the bitshuffle is not supported for the environment, + it will fall back to ``Compression.GZIP``. + Select ``Compression.NONE`` if compression is not needed. """ if not append_mode: @@ -259,22 +281,19 @@ def export_reduced_data_as_nxlauetof( # Data - shape: [n_x_pixels, n_y_pixels, n_tof_bins] # The actual application definition defines it as integer, # so we overwrite the dtype here. - num_x, num_y = da.sizes['x_pixel_offset'], da.sizes['y_pixel_offset'] - if compress_mode != Compression.NONE: - compression_args = _retrieve_compression_arguments(compress_mode) - data_dset = _create_dataset_from_var( - name="data", - root_entry=nx_detector, - var=da.data, - chunks=(num_x, num_y, 1), # Chunk along tof axis - dtype=np.uint, - **compression_args, - ) - else: - data_dset = _create_dataset_from_var( - name="data", root_entry=nx_detector, var=da.data, dtype=np.uint - ) + compression_args = _retrieve_compression_arguments(compress_mode) + if compress_mode != Compression.NONE: # Calculate the chunk sizes + num_x, num_y = da.sizes['x_pixel_offset'], da.sizes['y_pixel_offset'] + compression_args['chunks'] = (num_x, num_y, 1) # Chunk along tof axis + + data_dset = _create_dataset_from_var( + name="data", + root_entry=nx_detector, + var=da.data, + dtype=np.uint, + **compression_args, + ) data_dset.attrs["signal"] = 1 diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index cb03de1a..9c949571 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -14,6 +14,7 @@ class Compression(enum.StrEnum): """ NONE = 'NONE' + GZIP = 'GZIP' BITSHUFFLE_LZ4 = 'BITSHUFFLE_LZ4' From 5c38d6de291cc0aa274bbbc5536eba4c92ce9e05 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 21 Jan 2026 15:50:20 +0100 Subject: [PATCH 359/403] Add benchmark result in the documentation. --- .../essnmx/docs/user-guide/workflow.ipynb | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/packages/essnmx/docs/user-guide/workflow.ipynb b/packages/essnmx/docs/user-guide/workflow.ipynb index af3d2fc2..808d86fb 100644 --- a/packages/essnmx/docs/user-guide/workflow.ipynb +++ b/packages/essnmx/docs/user-guide/workflow.ipynb @@ -84,6 +84,7 @@ " InputConfig,\n", " WorkflowConfig,\n", " TimeBinCoordinate,\n", + " Compression,\n", " to_command_arguments,\n", ")\n", "\n", @@ -142,9 +143,41 @@ "source": [ "from ess.nmx.executables import save_results\n", "\n", - "output_config = OutputConfig(output_file=\"scipp_output.hdf\", overwrite=True)\n", + "output_config = OutputConfig(\n", + " output_file=\"scipp_output.hdf\", overwrite=True, compression=Compression.GZIP\n", + ")\n", "save_results(results=results, output_config=output_config)" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compression Modes\n", + "\n", + "There are multiple compression modes for `detector counts` data(other datasets are not compressed).
\n", + "The default mode is `BITSHUFFLE_LZ4` but
\n", + "note that `bitshuffle` may not be supported in cerntain environments, such as MacOS or Windows.\n", + "\n", + "It was decided because the first step of the reduction workflow (this workflow)
\n", + "is expected to be run by ESS in the specific environment that bitshuffle supports.
\n", + "\n", + "Here is the rough benchmark results with the small test dataset.
\n", + "With the result, users can decide which compression mode to use.\n", + "\n", + "| Compression Mode | Final Size [MB] | Compression Ratio | Writing Time [s] | Reading Time [s] |\n", + "| ---------------- | --------------- |------------------ | ---------------- | ---------------- |\n", + "| NONE | 1_966 | 1 | 4 | 1 |\n", + "| GZIP | 5 | 370 | 18 | 10 | 5 |\n", + "| BITSHUFFLE_LZ4 | 17 | 114 | 10 | 3 |\n", + "\n", + "> In the ESS standard VISA environment. (64 GB mem/6 VCPUs)\n", + "\n", + "`BITSHUFFLE_LZ4` showed much faster speed for writing/reading the reduced file.
\n", + "`GZIP` and `BITSHUFFLE` could both compress the data more than 99% (when the histogram was very empty)
\n", + "but `GZIP` had 3 times better compression ratio than `BITSHUFFLE` for this particular dataset.
\n", + "By default speed is prioritized to space." + ] } ], "metadata": { From b9e3f7334a319cf5082e1f36b9daa3f2b13d069c Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 21 Jan 2026 16:05:43 +0100 Subject: [PATCH 360/403] Add comma [skip ci] --- packages/essnmx/docs/user-guide/workflow.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/docs/user-guide/workflow.ipynb b/packages/essnmx/docs/user-guide/workflow.ipynb index 808d86fb..41e3b531 100644 --- a/packages/essnmx/docs/user-guide/workflow.ipynb +++ b/packages/essnmx/docs/user-guide/workflow.ipynb @@ -176,7 +176,7 @@ "`BITSHUFFLE_LZ4` showed much faster speed for writing/reading the reduced file.
\n", "`GZIP` and `BITSHUFFLE` could both compress the data more than 99% (when the histogram was very empty)
\n", "but `GZIP` had 3 times better compression ratio than `BITSHUFFLE` for this particular dataset.
\n", - "By default speed is prioritized to space." + "By default, speed is prioritized to space." ] } ], From e21d7d73f1d1158937d8a2f7cbe2e4b2f9fdcb70 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 21 Jan 2026 16:16:27 +0100 Subject: [PATCH 361/403] Add more context why bitshuffle is the default compression mode. --- .../essnmx/docs/user-guide/workflow.ipynb | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/essnmx/docs/user-guide/workflow.ipynb b/packages/essnmx/docs/user-guide/workflow.ipynb index 41e3b531..0fd92521 100644 --- a/packages/essnmx/docs/user-guide/workflow.ipynb +++ b/packages/essnmx/docs/user-guide/workflow.ipynb @@ -156,11 +156,7 @@ "## Compression Modes\n", "\n", "There are multiple compression modes for `detector counts` data(other datasets are not compressed).
\n", - "The default mode is `BITSHUFFLE_LZ4` but
\n", - "note that `bitshuffle` may not be supported in cerntain environments, such as MacOS or Windows.\n", - "\n", - "It was decided because the first step of the reduction workflow (this workflow)
\n", - "is expected to be run by ESS in the specific environment that bitshuffle supports.
\n", + "The default mode is `BITSHUFFLE_LZ4`.
\n", "\n", "Here is the rough benchmark results with the small test dataset.
\n", "With the result, users can decide which compression mode to use.\n", @@ -168,15 +164,27 @@ "| Compression Mode | Final Size [MB] | Compression Ratio | Writing Time [s] | Reading Time [s] |\n", "| ---------------- | --------------- |------------------ | ---------------- | ---------------- |\n", "| NONE | 1_966 | 1 | 4 | 1 |\n", - "| GZIP | 5 | 370 | 18 | 10 | 5 |\n", + "| GZIP | 5 | 370 | 18 | 5 |\n", "| BITSHUFFLE_LZ4 | 17 | 114 | 10 | 3 |\n", "\n", "> In the ESS standard VISA environment. (64 GB mem/6 VCPUs)\n", "\n", - "`BITSHUFFLE_LZ4` showed much faster speed for writing/reading the reduced file.
\n", + "`BITSHUFFLE_LZ4` showed almost twice faster speed for writing/reading the reduced file.
\n", "`GZIP` and `BITSHUFFLE` could both compress the data more than 99% (when the histogram was very empty)
\n", "but `GZIP` had 3 times better compression ratio than `BITSHUFFLE` for this particular dataset.
\n", - "By default, speed is prioritized to space." + "\n", + ".. note::\n", + "Why `BITSHUFFLE` is the default compression mode? -\n", + "`Bitshuffle` is compatible with `DIALS` and other crystallography packages.\n", + "It is the primary compression mode of all data collected on `DECTRIS EIGER` detectors,\n", + "which are the primary detectors used at synchrotron X-ray MX beamlines.\n", + "Most of these packages can also read `gzip` data but the slow readout makes `gzip` less attractive than `bitshuffle`.\n", + "\n", + "\n", + ".. warning::\n", + "`Bitshuffle` may not be supported in cerntain environments, such as MacOS or Windows.\n", + "It was accepted to be default because the first step of the reduction workflow (this workflow)\n", + "is expected to be run by ESS in the specific environment that bitshuffle supports." ] } ], From 4d190e9da49aca61433d24b67dff91586c2a1941 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 22 Jan 2026 11:11:05 +0100 Subject: [PATCH 362/403] Add test about compression modes. --- packages/essnmx/tests/executable_test.py | 121 ++++++++++++++++++++++- 1 file changed, 119 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index 0182666b..12e38d2b 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -3,9 +3,11 @@ import pathlib import subprocess +import time from contextlib import contextmanager from enum import Enum +import h5py import pydantic import pytest import scipp as sc @@ -377,8 +379,6 @@ def test_reduction_succeed_when_skipping_evenif_output_file_exists( def test_reduction_fails_fast_if_output_file_exists( reduction_config: ReductionConfig, temp_output_file: pathlib.Path ) -> None: - import time - # Make sure the file exists temp_output_file.touch() # Make sure file output is NOT skipped. @@ -393,3 +393,120 @@ def test_reduction_fails_fast_if_output_file_exists( # There is no special reason why it is 1 second. # It should just fail as fast as possible. assert finish - start < 1 + + +def test_reduction_compression_gzip( + reduction_config: ReductionConfig, tmp_path: pathlib.Path +) -> None: + reduction_config.output.skip_file_output = False + reduction_config.workflow.nbins = 5 # For faster test + file_paths: dict[Compression, pathlib.Path] = {} + + for compress_mode in (Compression.NONE, Compression.GZIP): + reduction_config.output.compression = compress_mode + cur_file_path = tmp_path / f'compress_{compress_mode}_output.hdf' + file_paths[compress_mode] = cur_file_path + assert not cur_file_path.exists() + reduction_config.output.output_file = cur_file_path.as_posix() + # Running the whole reduction instead of only saving the file on purpose. + with known_warnings(): + reduction(config=reduction_config) + assert cur_file_path.exists() + + assert ( + file_paths[Compression.NONE].stat().st_size + > file_paths[Compression.GZIP].stat().st_size + ) + with h5py.File(file_paths[Compression.NONE]) as file: + for i in range(3): + assert file[f'entry/instrument/detector_panel_{i}/data'].chunks is None + + with h5py.File(file_paths[Compression.GZIP]) as file: + for i in range(3): + data_path = f'entry/instrument/detector_panel_{i}/data' + assert file[data_path].chunks == (1280, 1280, 1) + assert file[data_path].compression == 'gzip' + assert file[data_path].compression_opts == 4 + + +try: + # Just checking availability + import bitshuffle.h5 # noqa: F401 +except ImportError: + BITSHUFFLE_AVAILABLE = False +else: + BITSHUFFLE_AVAILABLE = True + + +@pytest.mark.skipif( + not BITSHUFFLE_AVAILABLE, + reason="Bitshuffle is not available in this environment.", +) +def test_reduction_compression_bitshuffle_smaller_than_gzip( + reduction_config: ReductionConfig, tmp_path: pathlib.Path +) -> None: + reduction_config.output.skip_file_output = False + reduction_config.workflow.nbins = 5 # For faster test + file_paths: dict[Compression, pathlib.Path] = {} + total_times: dict[Compression, pathlib.Path] = {} + + for compress_mode in (Compression.GZIP, Compression.BITSHUFFLE_LZ4): + reduction_config.output.compression = compress_mode + cur_file_path = tmp_path / f'compress_{compress_mode}_output.hdf' + file_paths[compress_mode] = cur_file_path + assert not cur_file_path.exists() + reduction_config.output.output_file = cur_file_path.as_posix() + # Running the whole reduction instead of only saving the file on purpose. + with known_warnings(): + start = time.time() + reduction(config=reduction_config) + end = time.time() + + assert cur_file_path.exists() + total_times[compress_mode] = end - start + + # GZIP is expected to have better compression ratio than BITSHUFFLE + assert ( + file_paths[Compression.BITSHUFFLE_LZ4].stat().st_size + > file_paths[Compression.GZIP].stat().st_size + ) + # BITSHUFFLE is expected to be faster than GZIP + assert total_times[Compression.BITSHUFFLE_LZ4] < total_times[Compression.GZIP] + + with h5py.File(file_paths[Compression.GZIP]) as file: + for i in range(3): + data_path = f'entry/instrument/detector_panel_{i}/data' + assert file[data_path].chunks == (1280, 1280, 1) + assert file[data_path].compression == 'gzip' + + with h5py.File(file_paths[Compression.BITSHUFFLE_LZ4]) as file: + for i in range(3): + data_path = f'entry/instrument/detector_panel_{i}/data' + assert file[data_path].chunks == (1280, 1280, 1) + # For some reason it doesn't write the compression. + # so we check the filter instead. + # assert file[data_path].compression == 'bitshuffle' + assert '32008' in file[data_path]._filters + + +@pytest.mark.skipif( + BITSHUFFLE_AVAILABLE, + reason="Bitshuffle is available in this environment so it won't fall back.", +) +def test_reduction_compression_bitshuffle_fall_back_to_gzip( + reduction_config: ReductionConfig, temp_output_file: pathlib.Path +) -> None: + reduction_config.output.skip_file_output = False + reduction_config.workflow.nbins = 5 # For faster test + reduction_config.output.compression = Compression.BITSHUFFLE_LZ4 + reduction_config.output.output_file = temp_output_file.as_posix() + + with known_warnings(): + with pytest.warns(UserWarning, match='bitshuffle.h5'): + reduction(config=reduction_config) + + with h5py.File(temp_output_file) as file: + for i in range(3): + data_path = f'entry/instrument/detector_panel_{i}/data' + assert file[data_path].chunks == (1280, 1280, 1) + assert file[data_path].compression == 'gzip' From bd6e31af36382888326a937240768908576fa280 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 22 Jan 2026 15:45:55 +0100 Subject: [PATCH 363/403] Update lower pin of essreduce. --- packages/essnmx/pyproject.toml | 2 +- packages/essnmx/requirements/base.in | 2 +- packages/essnmx/requirements/base.txt | 20 ++++++++------------ packages/essnmx/requirements/basetest.txt | 2 +- packages/essnmx/requirements/ci.txt | 2 +- packages/essnmx/requirements/dev.txt | 14 ++++++++------ packages/essnmx/requirements/docs.txt | 18 ++++++++---------- packages/essnmx/requirements/mypy.txt | 2 +- packages/essnmx/requirements/nightly.in | 2 +- packages/essnmx/requirements/nightly.txt | 14 +++++++------- packages/essnmx/requirements/wheels.txt | 2 +- packages/essnmx/src/ess/nmx/workflows.py | 1 - 12 files changed, 38 insertions(+), 43 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 25279351..274b47c8 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -31,7 +31,7 @@ requires-python = ">=3.11" # Make sure to list one dependency per line. dependencies = [ "dask>=2022.1.0", - "essreduce>=25.11.5", # New domain types implemented, see https://scipp.github.io/essreduce/user-guide/reduction-workflow-guidelines.html#c-6-domain-types-for-data-products + "essreduce>=26.1.1", # New domain types implemented, see https://scipp.github.io/essreduce/user-guide/reduction-workflow-guidelines.html#c-6-domain-types-for-data-products "graphviz", "plopp>=24.7.0", "sciline>=24.06.0", diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index e873e726..e1ea74c7 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -3,7 +3,7 @@ # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask>=2022.1.0 -essreduce>=25.11.5 +essreduce>=26.1.1 graphviz plopp>=24.7.0 sciline>=24.06.0 diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index bd6402e8..e58d2b6c 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:4a430e06cfd011c7ff94e1985071c7b73750560a +# SHA1:14fea58f5e863d1b2f92c4bb7dd114ad641cd304 # # This file was generated by pip-compile-multi. # To update, run: @@ -25,7 +25,7 @@ cycler==0.12.1 # via matplotlib cython==3.2.4 # via bitshuffle -dask==2025.12.0 +dask==2026.1.1 # via -r base.in defusedxml==0.7.1 # via -r base.in @@ -33,7 +33,7 @@ dnspython==2.8.0 # via email-validator email-validator==2.3.0 # via scippneutron -essreduce==26.1.0 +essreduce==26.1.1 # via -r base.in fonttools==4.61.1 # via matplotlib @@ -81,13 +81,13 @@ numpy==2.4.1 # scipp # scippneutron # scipy -packaging==25.0 +packaging==26.0 # via # dask # lazy-loader # matplotlib # pooch -pandas==2.3.3 +pandas==3.0.0 # via -r base.in partd==1.4.2 # via dask @@ -108,15 +108,13 @@ pydantic==2.12.5 # via scippneutron pydantic-core==2.41.5 # via pydantic -pyparsing==3.3.1 +pyparsing==3.3.2 # via matplotlib python-dateutil==2.9.0.post0 # via # matplotlib # pandas # scippneutron -pytz==2025.2 - # via pandas pyyaml==6.0.3 # via dask requests==2.32.5 @@ -134,7 +132,7 @@ scipp==25.12.0 # tof scippneutron==25.11.2 # via essreduce -scippnexus==25.11.0 +scippnexus==26.1.0 # via # -r base.in # essreduce @@ -145,7 +143,7 @@ scipy==1.17.0 # scippnexus six==1.17.0 # via python-dateutil -tof==25.12.1 +tof==26.1.0 # via -r base.in toolz==1.1.0 # via @@ -158,8 +156,6 @@ typing-extensions==4.15.0 # typing-inspection typing-inspection==0.4.2 # via pydantic -tzdata==2025.3 - # via pandas urllib3==2.6.3 # via requests diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt index e9fde5ff..7950f419 100644 --- a/packages/essnmx/requirements/basetest.txt +++ b/packages/essnmx/requirements/basetest.txt @@ -7,7 +7,7 @@ # iniconfig==2.3.0 # via pytest -packaging==25.0 +packaging==26.0 # via pytest pluggy==1.6.0 # via pytest diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index 75ea55d3..f90a93aa 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -27,7 +27,7 @@ gitpython==3.1.46 # via -r ci.in idna==3.11 # via requests -packaging==25.0 +packaging==26.0 # via # -r ci.in # pyproject-api diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index 52c2efa7..41e16424 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -22,11 +22,11 @@ argon2-cffi-bindings==25.1.0 # via argon2-cffi arrow==1.4.0 # via isoduration -async-lru==2.0.5 +async-lru==2.1.0 # via jupyterlab cffi==2.0.0 # via argon2-cffi-bindings -copier==9.11.1 +copier==9.11.2 # via -r dev.in dunamai==1.25.0 # via copier @@ -63,7 +63,7 @@ jupyter-server==2.17.0 # jupyterlab # jupyterlab-server # notebook-shim -jupyter-server-terminals==0.5.3 +jupyter-server-terminals==0.5.4 # via jupyter-server jupyterlab==4.5.2 # via -r dev.in @@ -79,9 +79,9 @@ pip-tools==7.5.2 # via pip-compile-multi plumbum==1.10.0 # via copier -prometheus-client==0.24.0 +prometheus-client==0.24.1 # via jupyter-server -pycparser==2.23 +pycparser==3.0 # via cffi python-json-logger==4.0.0 # via jupyter-events @@ -105,13 +105,15 @@ terminado==0.18.1 # jupyter-server-terminals toposort==1.10 # via pip-compile-multi +tzdata==2025.3 + # via arrow uri-template==1.3.0 # via jsonschema webcolors==25.10.0 # via jsonschema websocket-client==1.9.0 # via jupyter-server -wheel==0.45.1 +wheel==0.46.3 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 388bfaa0..eb6fb979 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -38,7 +38,7 @@ debugpy==1.8.19 # via ipykernel decorator==5.2.1 # via ipython -docutils==0.21.2 +docutils==0.22.4 # via # myst-parser # nbsphinx @@ -92,7 +92,7 @@ jupyterlab-pygments==0.3.0 # via nbconvert jupyterlab-widgets==3.0.16 # via ipywidgets -markdown-it-py==3.0.0 +markdown-it-py==4.0.0 # via # mdit-py-plugins # myst-parser @@ -110,7 +110,7 @@ mdurl==0.1.2 # via markdown-it-py mistune==3.2.0 # via nbconvert -myst-parser==4.0.1 +myst-parser==5.0.0 # via -r docs.in nbclient==0.10.4 # via nbconvert @@ -164,8 +164,6 @@ referencing==0.37.0 # jsonschema # jsonschema-specifications roman-numerals==4.1.0 - # via roman-numerals-py -roman-numerals-py==4.1.0 # via sphinx rpds-py==0.30.0 # via @@ -173,9 +171,9 @@ rpds-py==0.30.0 # referencing snowballstemmer==3.0.1 # via sphinx -soupsieve==2.8.1 +soupsieve==2.8.3 # via beautifulsoup4 -sphinx==8.2.3 +sphinx==9.1.0 # via # -r docs.in # autodoc-pydantic @@ -185,11 +183,11 @@ sphinx==8.2.3 # sphinx-autodoc-typehints # sphinx-copybutton # sphinx-design -sphinx-autodoc-typehints==3.5.2 +sphinx-autodoc-typehints==3.6.2 # via -r docs.in sphinx-copybutton==0.5.2 # via -r docs.in -sphinx-design==0.6.1 +sphinx-design==0.7.0 # via -r docs.in sphinxcontrib-applehelp==2.0.0 # via sphinx @@ -227,7 +225,7 @@ traitlets==5.14.3 # traittypes traittypes==0.2.3 # via ipydatawidgets -wcwidth==0.2.14 +wcwidth==0.3.0 # via prompt-toolkit webencodings==0.5.1 # via diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index 61506e9f..1ade2762 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -6,7 +6,7 @@ # requirements upgrade # -r test.txt -librt==0.7.7 +librt==0.7.8 # via mypy mypy==1.19.1 # via -r mypy.in diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index ac3d7c95..51f61b0e 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -2,7 +2,7 @@ # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask>=2022.1.0 -essreduce>=25.11.5 +essreduce>=26.1.1 graphviz pooch>=1.5 pandas>=2.1.2 diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index b205d935..367219e2 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:b96d14f543449e35d860f235d9259b0b1504fb14 +# SHA1:ca33b3456dae9dd4723dba3f6a4cff5056530bf0 # # This file was generated by pip-compile-multi. # To update, run: @@ -28,7 +28,7 @@ cycler==0.12.1 # via matplotlib cython==3.2.4 # via bitshuffle -dask==2025.12.0 +dask==2026.1.1 # via -r nightly.in defusedxml==0.8.0rc2 # via -r nightly.in @@ -36,7 +36,7 @@ dnspython==2.8.0 # via email-validator email-validator==2.3.0 # via scippneutron -essreduce==26.1.0 +essreduce==26.1.1 # via -r nightly.in fonttools==4.61.1 # via matplotlib @@ -86,14 +86,14 @@ numpy==2.4.1 # scipp # scippneutron # scipy -packaging==26.0rc2 +packaging==26.0 # via # dask # lazy-loader # matplotlib # pooch # pytest -pandas==3.0.0rc1 +pandas==3.0.0 # via -r nightly.in partd==1.4.2 # via dask @@ -118,7 +118,7 @@ pydantic-core==2.41.5 # via pydantic pygments==2.19.2 # via pytest -pyparsing==3.3.1 +pyparsing==3.3.2 # via matplotlib pytest==9.0.2 # via -r nightly.in @@ -155,7 +155,7 @@ scipy==1.17.0 # scippnexus six==1.17.0 # via python-dateutil -tof==25.12.1 +tof==26.1.0 # via -r nightly.in toolz==1.1.0 # via diff --git a/packages/essnmx/requirements/wheels.txt b/packages/essnmx/requirements/wheels.txt index 3e37dfed..0d70d60c 100644 --- a/packages/essnmx/requirements/wheels.txt +++ b/packages/essnmx/requirements/wheels.txt @@ -7,7 +7,7 @@ # build==1.4.0 # via -r wheels.in -packaging==25.0 +packaging==26.0 # via build pyproject-hooks==1.2.0 # via build diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index 9f3cf167..0c463d8c 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -73,7 +73,6 @@ def _simulate_fixed_wavelength_tof( events = events[~events.masks["blocked_by_others"]] return SimulationResults( time_of_arrival=events.coords["toa"], - speed=events.coords["speed"], wavelength=events.coords["wavelength"], weight=events.data, distance=results["detector"].distance, From 3503902ed2bfd0ee456070752b2b9d81d868898b Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 22 Jan 2026 15:52:19 +0100 Subject: [PATCH 364/403] Remove lowerbound comment about domain type. --- packages/essnmx/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 274b47c8..bb1c93f4 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -31,7 +31,7 @@ requires-python = ">=3.11" # Make sure to list one dependency per line. dependencies = [ "dask>=2022.1.0", - "essreduce>=26.1.1", # New domain types implemented, see https://scipp.github.io/essreduce/user-guide/reduction-workflow-guidelines.html#c-6-domain-types-for-data-products + "essreduce>=26.1.1", "graphviz", "plopp>=24.7.0", "sciline>=24.06.0", From 69177f51bc8d65e60f6533269ef0b921ebba97a5 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 22 Jan 2026 16:06:56 +0100 Subject: [PATCH 365/403] Lock dependencies using py311 --- packages/essnmx/requirements/base.txt | 4 ++++ packages/essnmx/requirements/dev.txt | 2 ++ packages/essnmx/requirements/docs.txt | 4 ++-- packages/essnmx/requirements/nightly.txt | 4 ++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index e58d2b6c..da2e9dac 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -52,6 +52,8 @@ idna==3.11 # via # email-validator # requests +importlib-metadata==8.7.1 + # via dask kiwisolver==1.4.9 # via matplotlib lazy-loader==0.4 @@ -158,6 +160,8 @@ typing-inspection==0.4.2 # via pydantic urllib3==2.6.3 # via requests +zipp==3.23.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index 41e16424..2dea046b 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -73,6 +73,8 @@ lark==1.3.1 # via rfc3987-syntax notebook-shim==0.2.4 # via jupyterlab +overrides==7.7.0 + # via jupyter-server pip-compile-multi==3.2.2 # via -r dev.in pip-tools==7.5.2 diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index eb6fb979..e49781b4 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -173,7 +173,7 @@ snowballstemmer==3.0.1 # via sphinx soupsieve==2.8.3 # via beautifulsoup4 -sphinx==9.1.0 +sphinx==9.0.4 # via # -r docs.in # autodoc-pydantic @@ -183,7 +183,7 @@ sphinx==9.1.0 # sphinx-autodoc-typehints # sphinx-copybutton # sphinx-design -sphinx-autodoc-typehints==3.6.2 +sphinx-autodoc-typehints==3.6.1 # via -r docs.in sphinx-copybutton==0.5.2 # via -r docs.in diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 367219e2..6304dcf7 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -55,6 +55,8 @@ idna==3.11 # via # email-validator # requests +importlib-metadata==8.7.1 + # via dask iniconfig==2.3.0 # via pytest kiwisolver==1.4.10rc0 @@ -170,6 +172,8 @@ typing-inspection==0.4.2 # via pydantic urllib3==2.6.3 # via requests +zipp==3.23.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools From 689ab96e551a692737e968dc749ded974e4d604b Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Fri, 23 Jan 2026 10:37:32 +0100 Subject: [PATCH 366/403] Update lower pin of pytest. (#192) --- packages/essnmx/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index bb1c93f4..c54c8007 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -54,7 +54,7 @@ essnmx-reduce = "ess.nmx.executables:main" [project.optional-dependencies] test = [ - "pytest>=7.0", + "pytest>=8.0", ] [project.urls] From cf4aa8f9642b01235aa60665188b57a2dcc756f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Jan 2026 17:52:19 +0000 Subject: [PATCH 367/403] Bump scipp from 25.12.0 to 26.1.0 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 25.12.0 to 26.1.0. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/25.12.0...26.1.0) --- updated-dependencies: - dependency-name: scipp dependency-version: 26.1.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index da2e9dac..9d27a096 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -125,7 +125,7 @@ sciline==25.11.1 # via # -r base.in # essreduce -scipp==25.12.0 +scipp==26.1.0 # via # -r base.in # essreduce From ccd3d5a418f404fba47b223297d8d9f5f50c8bad Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Fri, 30 Jan 2026 12:48:13 +0100 Subject: [PATCH 368/403] Add duplicated file path test. (#194) --- packages/essnmx/src/ess/nmx/executables.py | 10 ++++++++++ packages/essnmx/tests/executable_test.py | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 8dac1fbf..e9adde16 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -48,6 +48,16 @@ def _retrieve_input_file(input_file: list[str]) -> pathlib.Path: """Temporary helper to retrieve a single input file from the list Until multiple input file support is implemented. """ + from collections import Counter + + # Check duplicated pattern or paths + _counts = Counter(input_file) + duplicating_patterns = {pattern for pattern, num in _counts.items() if num > 1} + if duplicating_patterns: + raise ValueError( + f"Duplicated file paths or pattern found. {duplicating_patterns}" + ) + if isinstance(input_file, list): input_files = collect_matching_input_files(*input_file) if len(input_files) == 0: diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index 12e38d2b..c36d3743 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -510,3 +510,12 @@ def test_reduction_compression_bitshuffle_fall_back_to_gzip( data_path = f'entry/instrument/detector_panel_{i}/data' assert file[data_path].chunks == (1280, 1280, 1) assert file[data_path].compression == 'gzip' + + +def test_reduction_duplicated_path_raises(reduction_config: ReductionConfig) -> None: + # Run with two files with same names. + reduction_config.inputs.input_file = reduction_config.inputs.input_file * 2 + with pytest.raises( + ValueError, match=r'Duplicated file paths or pattern found.*small_nmx_nexus.hdf' + ): + reduction(config=reduction_config) From 33085bc26bd72bb7b8c6687d2a4e1f8821ef2dc1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 30 Jan 2026 17:53:54 +0000 Subject: [PATCH 369/403] Bump scipp from 26.1.0 to 26.1.1 in /requirements Bumps [scipp](https://github.com/scipp/scipp) from 26.1.0 to 26.1.1. - [Release notes](https://github.com/scipp/scipp/releases) - [Commits](https://github.com/scipp/scipp/compare/26.1.0...26.1.1) --- updated-dependencies: - dependency-name: scipp dependency-version: 26.1.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- packages/essnmx/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 9d27a096..5caf01b5 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -125,7 +125,7 @@ sciline==25.11.1 # via # -r base.in # essreduce -scipp==26.1.0 +scipp==26.1.1 # via # -r base.in # essreduce From 50f34c2713e37bc2526cc549fedadc93b4edd9ce Mon Sep 17 00:00:00 2001 From: Neil Vaytet Date: Thu, 12 Feb 2026 15:28:48 +0100 Subject: [PATCH 370/403] fix simulation results for full beamline table using new format after essreduce update --- packages/essnmx/pyproject.toml | 2 +- packages/essnmx/requirements/base.in | 2 +- packages/essnmx/requirements/base.txt | 20 ++++++------ packages/essnmx/requirements/basetest.in | 2 +- packages/essnmx/requirements/basetest.txt | 2 +- packages/essnmx/requirements/ci.txt | 2 +- packages/essnmx/requirements/dev.txt | 6 ++-- packages/essnmx/requirements/docs.txt | 18 +++++------ packages/essnmx/requirements/mypy.txt | 2 +- packages/essnmx/requirements/nightly.in | 4 +-- packages/essnmx/requirements/nightly.txt | 14 ++++----- packages/essnmx/src/ess/nmx/workflows.py | 37 ++++++++++++++--------- 12 files changed, 58 insertions(+), 53 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index c54c8007..a2e05cc1 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -31,7 +31,7 @@ requires-python = ">=3.11" # Make sure to list one dependency per line. dependencies = [ "dask>=2022.1.0", - "essreduce>=26.1.1", + "essreduce>=26.2.1", "graphviz", "plopp>=24.7.0", "sciline>=24.06.0", diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index e1ea74c7..c828022c 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -3,7 +3,7 @@ # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask>=2022.1.0 -essreduce>=26.1.1 +essreduce>=26.2.1 graphviz plopp>=24.7.0 sciline>=24.06.0 diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 5caf01b5..b14873e7 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:14fea58f5e863d1b2f92c4bb7dd114ad641cd304 +# SHA1:690cd6e0a77540a2e9d253fc6be5f73d41322b4f # # This file was generated by pip-compile-multi. # To update, run: @@ -25,7 +25,7 @@ cycler==0.12.1 # via matplotlib cython==3.2.4 # via bitshuffle -dask==2026.1.1 +dask==2026.1.2 # via -r base.in defusedxml==0.7.1 # via -r base.in @@ -33,11 +33,11 @@ dnspython==2.8.0 # via email-validator email-validator==2.3.0 # via scippneutron -essreduce==26.1.1 +essreduce==26.2.2 # via -r base.in fonttools==4.61.1 # via matplotlib -fsspec==2026.1.0 +fsspec==2026.2.0 # via dask gemmi==0.7.4 # via -r base.in @@ -73,7 +73,7 @@ msgpack==1.1.2 # via -r base.in networkx==3.6.1 # via cyclebane -numpy==2.4.1 +numpy==2.4.2 # via # bitshuffle # contourpy @@ -93,16 +93,16 @@ pandas==3.0.0 # via -r base.in partd==1.4.2 # via dask -pillow==12.1.0 +pillow==12.1.1 # via matplotlib platformdirs==4.5.1 # via pooch -plopp==25.11.0 +plopp==26.2.0 # via # -r base.in # scippneutron # tof -pooch==1.8.2 +pooch==1.9.0 # via # -r base.in # tof @@ -125,7 +125,7 @@ sciline==25.11.1 # via # -r base.in # essreduce -scipp==26.1.1 +scipp==26.2.0 # via # -r base.in # essreduce @@ -134,7 +134,7 @@ scipp==26.1.1 # tof scippneutron==25.11.2 # via essreduce -scippnexus==26.1.0 +scippnexus==26.1.1 # via # -r base.in # essreduce diff --git a/packages/essnmx/requirements/basetest.in b/packages/essnmx/requirements/basetest.in index 231016ec..692bca17 100644 --- a/packages/essnmx/requirements/basetest.in +++ b/packages/essnmx/requirements/basetest.in @@ -7,4 +7,4 @@ # will not be touched by ``make_base.py`` # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! -pytest>=7.0 +pytest>=8.0 diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt index 7950f419..2509493d 100644 --- a/packages/essnmx/requirements/basetest.txt +++ b/packages/essnmx/requirements/basetest.txt @@ -1,4 +1,4 @@ -# SHA1:8287decb8676bd4ad5934cc138073b38af537418 +# SHA1:a183f2aaeff6c4f418995809266f8825553a276e # # This file was generated by pip-compile-multi. # To update, run: diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index f90a93aa..fd69cd42 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -5,7 +5,7 @@ # # requirements upgrade # -cachetools==6.2.4 +cachetools==7.0.1 # via tox certifi==2026.1.4 # via requests diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index 2dea046b..1b6bda4c 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -26,7 +26,7 @@ async-lru==2.1.0 # via jupyterlab cffi==2.0.0 # via argon2-cffi-bindings -copier==9.11.2 +copier==9.11.3 # via -r dev.in dunamai==1.25.0 # via copier @@ -65,7 +65,7 @@ jupyter-server==2.17.0 # notebook-shim jupyter-server-terminals==0.5.4 # via jupyter-server -jupyterlab==4.5.2 +jupyterlab==4.5.4 # via -r dev.in jupyterlab-server==2.28.0 # via jupyterlab @@ -77,7 +77,7 @@ overrides==7.7.0 # via jupyter-server pip-compile-multi==3.2.2 # via -r dev.in -pip-tools==7.5.2 +pip-tools==7.5.3 # via pip-compile-multi plumbum==1.10.0 # via copier diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index e49781b4..c425ce7d 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -10,8 +10,6 @@ accessible-pygments==0.0.5 # via pydata-sphinx-theme alabaster==1.0.0 # via sphinx -appnope==0.1.4 - # via ipykernel asttokens==3.0.1 # via stack-data attrs==25.4.0 @@ -20,7 +18,7 @@ attrs==25.4.0 # referencing autodoc-pydantic==2.2.0 # via -r docs.in -babel==2.17.0 +babel==2.18.0 # via # pydata-sphinx-theme # sphinx @@ -34,7 +32,7 @@ comm==0.2.3 # via # ipykernel # ipywidgets -debugpy==1.8.19 +debugpy==1.8.20 # via ipykernel decorator==5.2.1 # via ipython @@ -52,9 +50,9 @@ imagesize==1.4.1 # via sphinx ipydatawidgets==4.3.5 # via pythreejs -ipykernel==7.1.0 +ipykernel==7.2.0 # via -r docs.in -ipython==9.9.0 +ipython==9.10.0 # via # -r docs.in # ipykernel @@ -114,7 +112,7 @@ myst-parser==5.0.0 # via -r docs.in nbclient==0.10.4 # via nbconvert -nbconvert==7.16.6 +nbconvert==7.17.0 # via nbsphinx nbformat==5.10.4 # via @@ -127,13 +125,13 @@ nest-asyncio==1.6.0 # via ipykernel pandocfilters==1.5.1 # via nbconvert -parso==0.8.5 +parso==0.8.6 # via jedi pexpect==4.9.0 # via ipython prompt-toolkit==3.0.52 # via ipython -psutil==7.2.1 +psutil==7.2.2 # via ipykernel ptyprocess==0.7.0 # via pexpect @@ -225,7 +223,7 @@ traitlets==5.14.3 # traittypes traittypes==0.2.3 # via ipydatawidgets -wcwidth==0.3.0 +wcwidth==0.6.0 # via prompt-toolkit webencodings==0.5.1 # via diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index 1ade2762..83b4a62c 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -12,7 +12,7 @@ mypy==1.19.1 # via -r mypy.in mypy-extensions==1.1.0 # via mypy -pathspec==1.0.3 +pathspec==1.0.4 # via mypy # The following packages are considered to be unsafe in a requirements file: diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index 51f61b0e..1e2414a6 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -2,7 +2,7 @@ # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! dask>=2022.1.0 -essreduce>=26.1.1 +essreduce>=26.2.1 graphviz pooch>=1.5 pandas>=2.1.2 @@ -11,7 +11,7 @@ defusedxml>=0.7.1 msgpack>=1.0.8 tof>=25.12.1 bitshuffle>=0.5.2;os_name == 'posix' -pytest>=7.0 +pytest>=8.0 scipp --index-url=https://pypi.anaconda.org/scipp-nightly-wheels/simple/ --extra-index-url=https://pypi.org/simple diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 6304dcf7..a7250f3c 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:ca33b3456dae9dd4723dba3f6a4cff5056530bf0 +# SHA1:4e708d759352a2191073c0fe9616799eae59cf90 # # This file was generated by pip-compile-multi. # To update, run: @@ -28,7 +28,7 @@ cycler==0.12.1 # via matplotlib cython==3.2.4 # via bitshuffle -dask==2026.1.1 +dask==2026.1.2 # via -r nightly.in defusedxml==0.8.0rc2 # via -r nightly.in @@ -36,11 +36,11 @@ dnspython==2.8.0 # via email-validator email-validator==2.3.0 # via scippneutron -essreduce==26.1.1 +essreduce==26.2.2 # via -r nightly.in fonttools==4.61.1 # via matplotlib -fsspec==2026.1.0 +fsspec==2026.2.0 # via dask gemmi==0.7.4 # via -r nightly.in @@ -78,7 +78,7 @@ msgpack==1.1.2 # via -r nightly.in networkx==3.6.1 # via cyclebane -numpy==2.4.1 +numpy==2.4.2 # via # bitshuffle # contourpy @@ -99,7 +99,7 @@ pandas==3.0.0 # via -r nightly.in partd==1.4.2 # via dask -pillow==12.1.0 +pillow==12.1.1 # via matplotlib platformdirs==4.5.1 # via pooch @@ -110,7 +110,7 @@ plopp @ git+https://github.com/scipp/plopp@main # tof pluggy==1.6.0 # via pytest -pooch==1.8.2 +pooch==1.9.0 # via # -r nightly.in # tof diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index 0c463d8c..2a3daf20 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -22,6 +22,7 @@ SimulationSeed, TofLookupTableWorkflow, ) +from ess.reduce.time_of_flight.lut import BeamlineComponentReading from ess.reduce.time_of_flight.types import TimeOfFlightLookupTableFilename from ess.reduce.workflow import register_workflow @@ -43,16 +44,24 @@ def _simulate_fixed_wavelength_tof( wmin: TofSimulationMinWavelength, wmax: TofSimulationMaxWavelength, - ltotal_range: LtotalRange, neutrons: NumberOfSimulatedNeutrons, seed: SimulationSeed, ) -> SimulationResults: """ - Simulate a pulse of neutrons propagating through a chopper cascade using the + Simulate a pulse of neutrons propagating through the instrument using the ``tof`` package (https://tof.readthedocs.io). + This runs a simulation assuming there are no choppers in the instrument. Parameters ---------- + wmin: + Minimum wavelength of the simulated neutrons. + wmax: + Maximum wavelength of the simulated neutrons. + neutrons: + Number of neutrons to simulate. + seed: + Random seed for the simulation. """ source = tof.Source( facility="ess", @@ -62,20 +71,18 @@ def _simulate_fixed_wavelength_tof( wmax=wmax, wmin=wmin, ) - nmx_det = tof.Detector(distance=max(ltotal_range), name="detector") - model = tof.Model(source=source, choppers=[], detectors=[nmx_det]) - results = model.run() - events = results["detector"].data.squeeze().flatten(to="event") - # If there are any blocked neutrons, remove them - # it is not expected to have any in this simulation - # since it is not using any choppers - # but just in case we ever add any in the future - events = events[~events.masks["blocked_by_others"]] + events = source.data.squeeze().flatten(to="event") + return SimulationResults( - time_of_arrival=events.coords["toa"], - wavelength=events.coords["wavelength"], - weight=events.data, - distance=results["detector"].distance, + readings={ + "source": BeamlineComponentReading( + time_of_arrival=events.coords["birth_time"], + wavelength=events.coords["wavelength"], + weight=events.data, + distance=source.distance, + ) + }, + choppers=None, ) From 12a25f2a9422645338161292e1fa17d6eea7524a Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 18 Feb 2026 15:17:21 +0100 Subject: [PATCH 371/403] Export first pixel position for DIALS --- packages/essnmx/src/ess/nmx/types.py | 6 ++++++ packages/essnmx/src/ess/nmx/workflows.py | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 9c949571..63f2f87e 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -104,6 +104,11 @@ class NMXDetectorMetadata: fast_axis: sc.Variable slow_axis: sc.Variable distance: sc.Variable + first_pixel_position: sc.Variable + """First pixel position with respect to the sample. + + Additional field for DIALS. It should be a 3D vector. + """ # TODO: Remove hardcoded values polar_angle: sc.Variable = field(default_factory=lambda: sc.scalar(0, unit='deg')) azimuthal_angle: sc.Variable = field( @@ -117,5 +122,6 @@ def __write_to_nexus_group__(self, group: h5py.Group): snx.create_field(group, 'fast_axis', self.fast_axis) snx.create_field(group, 'slow_axis', self.slow_axis) snx.create_field(group, 'distance', self.distance) + snx.create_field(group, 'first_pixel_position', self.first_pixel_position) snx.create_field(group, 'polar_angle', self.polar_angle) snx.create_field(group, 'azimuthal_angle', self.azimuthal_angle) diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index 2a3daf20..304e0756 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -196,6 +196,7 @@ def _retrieve_crystal_rotation( def assemble_detector_metadata( detector_component: NeXusComponent[snx.NXdetector, SampleRun], transformation: NeXusTransformation[snx.NXdetector, SampleRun], + sample_position: Position[snx.NXsample, SampleRun], source_position: Position[snx.NXsource, SampleRun], empty_detector: EmptyDetector[SampleRun], ) -> NMXDetectorMetadata: @@ -220,6 +221,14 @@ def assemble_detector_metadata( y_pixel_size = _decide_step(empty_detector.coords['y_pixel_offset']) distance = sc.norm(origin - source_position.to(unit=origin.unit)) + # We save the first pixel position so that DIALS can read use it. + flattened = empty_detector.flatten(to='detector_number') + first_pixel_number = flattened.coords['detector_number'].min() + first_pixel_position = flattened['detector_number', first_pixel_number].coords[ + 'position' + ] + first_pixel_position_from_sample = first_pixel_position - sample_position + return NMXDetectorMetadata( detector_name=detector_component['nexus_component_name'], x_pixel_size=x_pixel_size, @@ -228,6 +237,7 @@ def assemble_detector_metadata( fast_axis=_normalize_vector(fast_axis_vector), slow_axis=_normalize_vector(slow_axis_vector), distance=distance, + first_pixel_position=first_pixel_position_from_sample, ) From 1941e7a007f8386762e19e06bc9cccdb0f678ec6 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 19 Feb 2026 10:16:30 +0100 Subject: [PATCH 372/403] Save first pixel position as an attribute not a field. --- packages/essnmx/src/ess/nmx/types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 63f2f87e..5e5ecc3a 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -118,10 +118,10 @@ class NMXDetectorMetadata: def __write_to_nexus_group__(self, group: h5py.Group): snx.create_field(group, 'x_pixel_size', self.x_pixel_size) snx.create_field(group, 'y_pixel_size', self.y_pixel_size) - snx.create_field(group, 'origin', self.origin_position) + origin = snx.create_field(group, 'origin', self.origin_position) + origin.attrs['first_pixel_position'] = self.first_pixel_position.values snx.create_field(group, 'fast_axis', self.fast_axis) snx.create_field(group, 'slow_axis', self.slow_axis) snx.create_field(group, 'distance', self.distance) - snx.create_field(group, 'first_pixel_position', self.first_pixel_position) snx.create_field(group, 'polar_angle', self.polar_angle) snx.create_field(group, 'azimuthal_angle', self.azimuthal_angle) From 4129692dd47ceddb6181d7131a9bac0a4fbabfdc Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 23 Feb 2026 11:53:22 +0100 Subject: [PATCH 373/403] Rename the result field name to match the output file structure. --- packages/essnmx/src/ess/nmx/executables.py | 6 ++--- packages/essnmx/src/ess/nmx/types.py | 28 ++++++++++++++++++++-- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index e9adde16..a7a8fa61 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -270,7 +270,7 @@ def reduction( detector=detector_metas, sample=metadatas[NMXSampleMetadata], source=metadatas[NMXSourceMetadata], - monitor=monitor_metadata, + control=monitor_metadata, ) if config.workflow.time_bin_coordinate == TimeBinCoordinate.time_of_flight: @@ -284,7 +284,7 @@ def reduction( def save_results(*, results: sc.DataGroup, output_config: OutputConfig) -> None: # Validate if results have expected fields - for mandatory_key in ['histogram', 'detector', 'sample', 'source', 'monitor']: + for mandatory_key in ['histogram', 'detector', 'sample', 'source', 'control']: if mandatory_key not in results: raise ValueError(f"Missing '{mandatory_key}' in results to save.") @@ -295,7 +295,7 @@ def save_results(*, results: sc.DataGroup, output_config: OutputConfig) -> None: overwrite=output_config.overwrite, ) export_monitor_metadata_as_nxlauetof( - monitor_metadata=results['monitor'], + monitor_metadata=results['control'], output_file=output_config.output_file, ) for detector_name, detector_meta in results['detector'].items(): diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 5e5ecc3a..67515323 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -25,6 +25,18 @@ class Compression(enum.StrEnum): """Maximum wavelength for tof simulation to calculate look up table.""" +class ControlMode(enum.StrEnum): + """Control mode of counting. + + Based on the NXlauetof definition of `control`(NXmonitor) field. + """ + + monitor = 'monitor' + """Count to a preset value based on received monitor counts.""" + timer = 'timer' + """Count to a preset value based on clock time""" + + @dataclass(kw_only=True) class NMXSampleMetadata: nx_class = snx.NXsample @@ -70,6 +82,10 @@ def __write_to_nexus_group__(self, group: h5py.Group): snx.create_field(group, 'probe', 'neutron') +def _zero_float_count() -> sc.Variable: + return sc.scalar(0.0, unit='count') + + @dataclass(kw_only=True) class NMXMonitorMetadata: nx_class = snx.NXmonitor @@ -81,10 +97,18 @@ class NMXMonitorMetadata: "in the monitor histogram." }, ) + mode: ControlMode = field( + default=ControlMode.monitor, + metadata={"description": "Mode of counting. One of `monitor` or `timer`."}, + ) + preset: sc.Variable = field( + default_factory=_zero_float_count, + metadata={"description": "Preset value of counting for the `mode`."}, + ) def __write_to_nexus_group__(self, group: h5py.Group): - snx.create_field(group, 'mode', 'monitor') - snx.create_field(group, 'preset', 0.0) + snx.create_field(group, 'mode', str(self.mode)) + snx.create_field(group, 'preset', self.preset) data_field = snx.create_field(group, 'data', self.monitor_histogram.data) data_field.attrs['signal'] = 1 data_field.attrs['primary'] = 1 From 06206090c77ad983e5bd538201668126471ab233 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 23 Feb 2026 15:34:42 +0100 Subject: [PATCH 374/403] Wrap reduced result into a dataclass and add a helper function for easier display. --- .../essnmx/docs/user-guide/workflow.ipynb | 6 +- .../essnmx/src/ess/nmx/_display_helper.py | 30 +++++++ packages/essnmx/src/ess/nmx/executables.py | 56 +++++++------ packages/essnmx/src/ess/nmx/types.py | 78 +++++++++++++++---- packages/essnmx/src/ess/nmx/workflows.py | 4 +- 5 files changed, 134 insertions(+), 40 deletions(-) create mode 100644 packages/essnmx/src/ess/nmx/_display_helper.py diff --git a/packages/essnmx/docs/user-guide/workflow.ipynb b/packages/essnmx/docs/user-guide/workflow.ipynb index 0fd92521..cf33aed5 100644 --- a/packages/essnmx/docs/user-guide/workflow.ipynb +++ b/packages/essnmx/docs/user-guide/workflow.ipynb @@ -52,8 +52,10 @@ " ),\n", ")\n", "\n", - "# Run Reduction\n", - "reduction(config=config, display=display)" + "# Run reduction and display the result.\n", + "result = reduction(config=config, display=display)\n", + "dg = result.to_datagroup()\n", + "dg" ] }, { diff --git a/packages/essnmx/src/ess/nmx/_display_helper.py b/packages/essnmx/src/ess/nmx/_display_helper.py new file mode 100644 index 00000000..676c3b9e --- /dev/null +++ b/packages/essnmx/src/ess/nmx/_display_helper.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Scipp contributors (https://github.com/scipp) +from dataclasses import fields, is_dataclass + +import scipp as sc + + +def _is_nested(obj) -> bool: + return is_dataclass(obj) or isinstance(obj, sc.DataGroup | dict) + + +def to_datagroup(obj) -> sc.DataGroup: + if is_dataclass(obj): + return sc.DataGroup( + { + field.name: to_datagroup(value) + if _is_nested(value := getattr(obj, field.name)) + else value + for field in fields(obj) + } + ) + elif isinstance(obj, sc.DataGroup | dict): + return sc.DataGroup( + { + name: to_datagroup(value) if _is_nested(value) else value + for name, value in obj.items() + } + ) + else: + return obj diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index a7a8fa61..d63923a0 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -32,7 +32,10 @@ ) from .types import ( NMXDetectorMetadata, + NMXInstrument, + NMXLauetof, NMXMonitorMetadata, + NMXReducedDetector, NMXSampleMetadata, NMXSourceMetadata, ) @@ -173,7 +176,7 @@ def reduction( config: ReductionConfig, logger: logging.Logger | None = None, display: Callable | None = None, -) -> sc.DataGroup: +) -> NMXLauetof: """Reduce NMX data from a Nexus file and export to NXLauetof(ESS NMX specific) file. Parameters @@ -265,16 +268,25 @@ def reduction( histogram = tof_da.hist({t_coord_name: t_bin_edges}) tof_histograms[detector_name] = histogram - results = sc.DataGroup( - histogram=tof_histograms, - detector=detector_metas, - sample=metadatas[NMXSampleMetadata], - source=metadatas[NMXSourceMetadata], + detector_results = sc.DataGroup( + { + detector_name: NMXReducedDetector( + data=histogram, metadata=detector_metas[detector_name] + ) + for detector_name, histogram in tof_histograms.items() + } + ) + source_meta: NMXSourceMetadata = metadatas[NMXSourceMetadata] + sample_meta: NMXSampleMetadata = metadatas[NMXSampleMetadata] + + results = NMXLauetof( control=monitor_metadata, + instrument=NMXInstrument(detectors=detector_results, source=source_meta), + sample=sample_meta, ) if config.workflow.time_bin_coordinate == TimeBinCoordinate.time_of_flight: - results["lookup_table"] = base_wf.compute(TimeOfFlightLookupTable) + results.lookup_table = base_wf.compute(TimeOfFlightLookupTable) if not config.output.skip_file_output: save_results(results=results, output_config=config.output) @@ -282,33 +294,33 @@ def reduction( return results -def save_results(*, results: sc.DataGroup, output_config: OutputConfig) -> None: +def save_results(*, results: NMXLauetof, output_config: OutputConfig) -> None: # Validate if results have expected fields - for mandatory_key in ['histogram', 'detector', 'sample', 'source', 'control']: - if mandatory_key not in results: - raise ValueError(f"Missing '{mandatory_key}' in results to save.") export_static_metadata_as_nxlauetof( - sample_metadata=results['sample'], - source_metadata=results['source'], + sample_metadata=results.sample, + source_metadata=results.instrument.source, output_file=output_config.output_file, overwrite=output_config.overwrite, ) export_monitor_metadata_as_nxlauetof( - monitor_metadata=results['control'], + monitor_metadata=results.control, output_file=output_config.output_file, ) - for detector_name, detector_meta in results['detector'].items(): + for detector_name, detector_result in results.instrument.detectors.items(): export_detector_metadata_as_nxlauetof( - detector_metadata=detector_meta, + detector_metadata=detector_result.metadata, output_file=output_config.output_file, ) - export_reduced_data_as_nxlauetof( - detector_name=detector_name, - da=results['histogram'][detector_name], - output_file=output_config.output_file, - compress_mode=output_config.compression, - ) + if isinstance(detector_result.data, sc.DataArray): + export_reduced_data_as_nxlauetof( + detector_name=detector_name, + da=detector_result.data, + output_file=output_config.output_file, + compress_mode=output_config.compression, + ) + else: + raise ValueError(f"Detector counts histogram missing in {detector_name}") def main() -> None: diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 67515323..5a869207 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -1,10 +1,13 @@ import enum from dataclasses import dataclass, field -from typing import NewType +from typing import Literal, NewType import h5py import scipp as sc import scippnexus as snx +from ess.reduce.time_of_flight.types import TofLookupTable + +from ._display_helper import to_datagroup class Compression(enum.StrEnum): @@ -37,23 +40,25 @@ class ControlMode(enum.StrEnum): """Count to a preset value based on clock time""" +def _unit_matrix() -> sc.Variable: + return sc.array( + dims=['i', 'j'], + values=[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], + unit="dimensionless", + ) + + @dataclass(kw_only=True) class NMXSampleMetadata: nx_class = snx.NXsample crystal_rotation: sc.Variable - sample_position: sc.Variable - sample_name: str + name: str + position: sc.Variable # Temporarily hardcoding some values # TODO: Remove hardcoded values - sample_orientation_matrix: sc.Variable = field( - default_factory=lambda: sc.array( - dims=['i', 'j'], - values=[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], - unit="dimensionless", - ) - ) - sample_unit_cell: sc.Variable = field( + orientation_matrix: sc.Variable = field(default_factory=_unit_matrix) + unit_cell: sc.Variable = field( default_factory=lambda: sc.array( dims=['i'], values=[1.0, 1.0, 1.0, 90.0, 90.0, 90.0], @@ -65,9 +70,10 @@ class NMXSampleMetadata: def __write_to_nexus_group__(self, group: h5py.Group): cr_field = snx.create_field(group, 'crystal_rotation', self.crystal_rotation) cr_field.attrs['long_name'] = 'crystal rotation in Phi (XYZ)' - snx.create_field(group, 'name', self.sample_name) - snx.create_field(group, 'orientation_matrix', self.sample_orientation_matrix) - snx.create_field(group, 'unit_cell', self.sample_unit_cell) + snx.create_field(group, 'name', self.name) + snx.create_field(group, 'position', self.position) + snx.create_field(group, 'orientation_matrix', self.orientation_matrix) + snx.create_field(group, 'unit_cell', self.unit_cell) @dataclass(kw_only=True) @@ -149,3 +155,47 @@ def __write_to_nexus_group__(self, group: h5py.Group): snx.create_field(group, 'distance', self.distance) snx.create_field(group, 'polar_angle', self.polar_angle) snx.create_field(group, 'azimuthal_angle', self.azimuthal_angle) + + +@dataclass(kw_only=True) +class NMXReducedDetector: + """Reduced Detector data and metadata container. + + In an output file, all metadata fields are stored on the same level as the `data`. + However, in this reduced detector data container, the `data` and `metadata` are + separated with an extra hierarchy. + It is because the `data` needs more control how to be stored, + i.e. compression option. + Also, the histogram may need chunk-wise processing + and therefore metadata may need to be written in advance so that + the `data` can be appended to the existing `NXdetector` HDF5 Group. + + """ + + data: sc.DataArray | None = None + """3D Histogram of the detector counts or its place holder.""" + metadata: NMXDetectorMetadata + """NMX Detector metadata.""" + + +@dataclass(kw_only=True) +class NMXInstrument: + nx_class = snx.NXinstrument + + detectors: sc.DataGroup[NMXReducedDetector] + name: str = "NMX" + source: NMXSourceMetadata + + +@dataclass(kw_only=True) +class NMXLauetof: + nx_class = "NXlauetof" + + control: NMXMonitorMetadata + definitions: Literal['NXlauetof'] = 'NXlauetof' + instrument: NMXInstrument + sample: NMXSampleMetadata + lookup_table: TofLookupTable | None = None + + def to_datagroup(self) -> sc.DataGroup: + return to_datagroup(self) diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index 304e0756..9d702e19 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -126,9 +126,9 @@ def assemble_sample_metadata( raise TypeError(f'Sample name {name}is in a wrong type: ', type(name)) return NMXSampleMetadata( - sample_name=sample_name, + name=sample_name, crystal_rotation=crystal_rotation, - sample_position=sample_position, + position=sample_position, ) From 8aa8447da2bffed63b9720bd340a6b619e0823c8 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 23 Feb 2026 15:35:03 +0100 Subject: [PATCH 375/403] Update test according to the new data structure for the result. --- packages/essnmx/tests/executable_test.py | 28 +++++++++++++++--------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/packages/essnmx/tests/executable_test.py b/packages/essnmx/tests/executable_test.py index c36d3743..5edf8b02 100644 --- a/packages/essnmx/tests/executable_test.py +++ b/packages/essnmx/tests/executable_test.py @@ -24,7 +24,7 @@ ) from ess.nmx.configurations import TimeBinCoordinate, TimeBinUnit, to_command_arguments from ess.nmx.executables import reduction -from ess.nmx.types import Compression +from ess.nmx.types import Compression, NMXLauetof def _build_arg_list_from_pydantic_instance(*instances: pydantic.BaseModel) -> list[str]: @@ -215,9 +215,12 @@ def reduction_config( return ReductionConfig(inputs=input_config, output=output_config) -def _retrieve_one_hist(results: sc.DataGroup) -> sc.DataArray: +def _retrieve_one_hist(results: NMXLauetof) -> sc.DataArray: """Helper to retrieve the first DataArray from the results dictionary.""" - return results['histogram']['detector_panel_0'] + da = results.instrument.detectors['detector_panel_0'].data + if not isinstance(da, sc.DataArray): + raise TypeError("Histogram is not a DataArray.") + return da def test_reduction_default_settings(reduction_config: ReductionConfig) -> None: @@ -282,7 +285,9 @@ def test_histogram_out_of_range_min_warns( with known_warnings(): results = reduction(config=reduction_config) - for da in results['histogram'].values(): + for histogram in results.instrument.detectors.values(): + assert isinstance(histogram.data, sc.DataArray) + da = histogram.data assert_identical( da.data.sum(), sc.scalar(0.0, unit='counts', dtype='float32', variance=0.0) ) @@ -303,7 +308,9 @@ def test_histogram_out_of_range_max_warns( with known_warnings(): results = reduction(config=reduction_config) - for da in results['histogram'].values(): + for det in results.instrument.detectors.values(): + da = det.data + assert isinstance(da, sc.DataArray) assert_identical( da.data.sum(), sc.scalar(0.0, unit='counts', dtype='float32', variance=0.0) ) @@ -350,11 +357,12 @@ def test_reduction_with_tof_lut_file( with known_warnings(): results = reduction(config=reduction_config) - for default_hist, hist in zip( - default_results['histogram'].values(), - results['histogram'].values(), - strict=True, - ): + default_hists = [det.data for det in default_results.instrument.detectors.values()] + hists = [det.data for det in results.instrument.detectors.values()] + + for default_hist, hist in zip(default_hists, hists, strict=True): + assert isinstance(default_hist, sc.DataArray) + assert isinstance(hist, sc.DataArray) tof_edges_default = default_hist.coords['tof'] tof_edges = hist.coords['tof'] assert_identical(default_hist.data, hist.data) From dfa1d5ac9b28b90cb1f62b29b37c534f8a671651 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 23 Feb 2026 15:41:38 +0100 Subject: [PATCH 376/403] Adjust reduction configuration for documentation build. --- packages/essnmx/docs/user-guide/workflow.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/docs/user-guide/workflow.ipynb b/packages/essnmx/docs/user-guide/workflow.ipynb index cf33aed5..89bae585 100644 --- a/packages/essnmx/docs/user-guide/workflow.ipynb +++ b/packages/essnmx/docs/user-guide/workflow.ipynb @@ -40,7 +40,7 @@ " detector_ids=[0, 1, 2],\n", " ),\n", " output=OutputConfig(\n", - " output_file=\"scipp_output.hdf\", skip_file_output=False, overwrite=True\n", + " output_file=\"scipp_output.hdf\", skip_file_output=True, overwrite=True\n", " ),\n", " workflow=WorkflowConfig(\n", " time_bin_coordinate=TimeBinCoordinate.time_of_flight,\n", From 41953d06bc4ae6c6acdf64e241406dc973b7b328 Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 23 Feb 2026 15:41:58 +0100 Subject: [PATCH 377/403] Update source metadata to match the file structure. --- packages/essnmx/src/ess/nmx/types.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 5a869207..851ff3ba 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -80,12 +80,16 @@ def __write_to_nexus_group__(self, group: h5py.Group): class NMXSourceMetadata: nx_class = snx.NXsource source_position: sc.Variable + name: Literal['European Spallation Source'] = "European Spallation Source" + type: Literal['Spallation Neutron Source'] = "Spallation Neutron Source" + probe: Literal['neutron'] = "neutron" def __write_to_nexus_group__(self, group: h5py.Group): - snx.create_field(group, 'name', 'European Spallation Source') - snx.create_field(group, 'type', 'Spallation Neutron Source') - snx.create_field(group, 'distance', sc.norm(self.source_position)) - snx.create_field(group, 'probe', 'neutron') + snx.create_field(group, 'name', self.name) + snx.create_field(group, 'type', self.type) + distance = snx.create_field(group, 'distance', sc.norm(self.source_position)) + distance.attrs['position'] = self.source_position.values + snx.create_field(group, 'probe', self.probe) def _zero_float_count() -> sc.Variable: From e351c9c196be676dfc7fde419ca4e07c452e82dd Mon Sep 17 00:00:00 2001 From: YooSunyoung <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 23 Feb 2026 17:37:58 +0100 Subject: [PATCH 378/403] Custom loader for NMX NXlauetof file and test. --- packages/essnmx/src/ess/nmx/_nxlauetof_io.py | 45 ++++++++++++++++ packages/essnmx/src/ess/nmx/types.py | 5 +- .../essnmx/tests/nxlauetof_io_helper_test.py | 52 +++++++++++++++++++ 3 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 packages/essnmx/src/ess/nmx/_nxlauetof_io.py create mode 100644 packages/essnmx/tests/nxlauetof_io_helper_test.py diff --git a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py new file mode 100644 index 00000000..00ef2244 --- /dev/null +++ b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Scipp contributors (https://github.com/scipp) +import warnings + +import scipp as sc +import scippnexus as snx +from ess.reduce.nexus.types import FilePath, NeXusFile + + +def _validate_entry(entry: snx.Group) -> None: + if str(entry.attrs['NX_class']) != 'NXlauetof': + raise ValueError("File entry is not NXlauetof.") + _MANDATORY_FIELDS = ('control', 'instrument', 'sample') + missing_fields = [field for field in _MANDATORY_FIELDS if field not in entry] + if any(missing_fields): + raise ValueError("File entry missing mandatory fields, ", missing_fields) + + +def _as_vector(var: sc.Variable) -> sc.Variable: + if var.dims == () and var.dtype == sc.DType.vector3: + return var + elif len(var.dims) == 1 and var.sizes[var.dim] == 3: + return sc.vector(value=var.values, unit=var.unit) + else: + warnings.warn( + f"Cannot convert to vector3 scalar: {var}. " + "Falling back to the original form.", + UserWarning, + stacklevel=3, + ) + return var + + +def load_essnmx_nxlauetof(file: str | FilePath | NeXusFile) -> sc.DataGroup: + dg = snx.load(file) + + with snx.File(file) as f: + _validate_entry(entry := f['entry']) + sample = entry['sample'][...] + sample['crystal_rotation'] = _as_vector(sample['crystal_rotation']) + sample['position'] = _as_vector(sample['position']) + sample['unit_cell'] = sample['unit_cell'].rename_dims(dim_0='i') + dg['entry']['sample'] = sample + + return dg['entry'] diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 851ff3ba..a59ffc10 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -41,9 +41,8 @@ class ControlMode(enum.StrEnum): def _unit_matrix() -> sc.Variable: - return sc.array( - dims=['i', 'j'], - values=[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], + return sc.spatial.linear_transform( + value=[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], unit="dimensionless", ) diff --git a/packages/essnmx/tests/nxlauetof_io_helper_test.py b/packages/essnmx/tests/nxlauetof_io_helper_test.py new file mode 100644 index 00000000..9d185636 --- /dev/null +++ b/packages/essnmx/tests/nxlauetof_io_helper_test.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Scipp contributors (https://github.com/scipp) +import pathlib +from contextlib import contextmanager + +import pytest +from scipp.testing.assertions import assert_identical + +from ess.nmx._nxlauetof_io import load_essnmx_nxlauetof +from ess.nmx.configurations import InputConfig, OutputConfig, ReductionConfig +from ess.nmx.executables import reduction +from ess.nmx.types import Compression + + +@pytest.fixture +def temp_output_file(tmp_path: pathlib.Path): + output_file_path = tmp_path / "scipp_output.h5" + yield output_file_path + if output_file_path.exists(): + output_file_path.unlink() + + +@pytest.fixture +def reduction_config(temp_output_file: pathlib.Path) -> ReductionConfig: + from ess.nmx.data import get_small_nmx_nexus + + input_config = InputConfig(input_file=[get_small_nmx_nexus().as_posix()]) + output_config = OutputConfig( + output_file=temp_output_file.as_posix(), + compression=Compression.NONE, + skip_file_output=False, + ) + return ReductionConfig(inputs=input_config, output=output_config) + + +@contextmanager +def known_warnings(): + with pytest.warns(RuntimeWarning, match="No crystal rotation*"): + yield + + +def test_loaded_data_same_as_in_memory_result( + reduction_config: ReductionConfig, +) -> None: + with known_warnings(): + result = reduction(config=reduction_config) + original_result_dg = result.to_datagroup() + + with pytest.warns(UserWarning, match=r'Could not determine'): + loaded_dg = load_essnmx_nxlauetof(reduction_config.output.output_file) + + assert_identical(loaded_dg['sample'], original_result_dg['sample']) From 769d5dcd2c0dc76cd68c8a63aba5fe5c1d8cbc49 Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 24 Feb 2026 14:07:18 +0100 Subject: [PATCH 379/403] Handle unit cell separately. --- packages/essnmx/src/ess/nmx/_nxlauetof_io.py | 20 ++++++++++---- packages/essnmx/src/ess/nmx/types.py | 29 ++++++++++++++------ 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py index 00ef2244..13c98d40 100644 --- a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py +++ b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py @@ -31,15 +31,25 @@ def _as_vector(var: sc.Variable) -> sc.Variable: return var +def _handle_sample(sample: snx.Group) -> sc.DataGroup: + sample_dg = sample[...] + sample_dg['crystal_rotation'] = _as_vector(sample_dg['crystal_rotation']) + sample_dg['position'] = _as_vector(sample_dg['position']) + unit_cell = sample_dg.pop('unit_cell') + sample_dg['unit_cell_length'] = sc.vector( + unit_cell[:3], unit=sample['unit_cell'].attrs['length-unit'] + ) + sample_dg['unit_cell_angle'] = sc.vector( + unit_cell[3:], unit=sample['unit_cell'].attrs['angle-unit'] + ) + return sample_dg + + def load_essnmx_nxlauetof(file: str | FilePath | NeXusFile) -> sc.DataGroup: dg = snx.load(file) with snx.File(file) as f: _validate_entry(entry := f['entry']) - sample = entry['sample'][...] - sample['crystal_rotation'] = _as_vector(sample['crystal_rotation']) - sample['position'] = _as_vector(sample['position']) - sample['unit_cell'] = sample['unit_cell'].rename_dims(dim_0='i') - dg['entry']['sample'] = sample + dg['entry']['sample'] = _handle_sample(entry['sample']) return dg['entry'] diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index a59ffc10..220d3ccd 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -3,6 +3,7 @@ from typing import Literal, NewType import h5py +import numpy as np import scipp as sc import scippnexus as snx from ess.reduce.time_of_flight.types import TofLookupTable @@ -47,6 +48,14 @@ def _unit_matrix() -> sc.Variable: ) +def _uniform_unit_cell_length() -> sc.Variable: + return sc.vector([1.0, 1.0, 1.0], unit='dimensionless') + + +def _cube_unit_cell_angle() -> sc.Variable: + return sc.vector([90.0, 90.0, 90.0], unit='deg') + + @dataclass(kw_only=True) class NMXSampleMetadata: nx_class = snx.NXsample @@ -57,14 +66,14 @@ class NMXSampleMetadata: # Temporarily hardcoding some values # TODO: Remove hardcoded values orientation_matrix: sc.Variable = field(default_factory=_unit_matrix) - unit_cell: sc.Variable = field( - default_factory=lambda: sc.array( - dims=['i'], - values=[1.0, 1.0, 1.0, 90.0, 90.0, 90.0], - unit="dimensionless", # TODO: Add real data, - # a, b, c, alpha, beta, gamma - ) - ) + unit_cell_length: sc.Variable = field(default_factory=_uniform_unit_cell_length) + unit_cell_angle: sc.Variable = field(default_factory=_cube_unit_cell_angle) + + @property + def unit_cell(self) -> sc.Variable: + """a, b, c, alpha, beta, gamma.""" + + return np.concat([self.unit_cell_length.values, self.unit_cell_angle.values]) def __write_to_nexus_group__(self, group: h5py.Group): cr_field = snx.create_field(group, 'crystal_rotation', self.crystal_rotation) @@ -72,7 +81,9 @@ def __write_to_nexus_group__(self, group: h5py.Group): snx.create_field(group, 'name', self.name) snx.create_field(group, 'position', self.position) snx.create_field(group, 'orientation_matrix', self.orientation_matrix) - snx.create_field(group, 'unit_cell', self.unit_cell) + unit_cell = snx.create_field(group, 'unit_cell', self.unit_cell) + unit_cell.attrs['length-unit'] = str(self.unit_cell_length.unit) + unit_cell.attrs['angle-unit'] = str(self.unit_cell_angle.unit) @dataclass(kw_only=True) From 0e9a8535d2caf2d92c0f6855932e49ca09d3da78 Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 24 Feb 2026 14:38:56 +0100 Subject: [PATCH 380/403] Handle monitor group. --- packages/essnmx/src/ess/nmx/_nxlauetof_io.py | 19 +++++++++++++++++++ packages/essnmx/src/ess/nmx/executables.py | 2 +- packages/essnmx/src/ess/nmx/types.py | 16 +++++++++++----- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py index 13c98d40..23d01d0b 100644 --- a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py +++ b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py @@ -6,6 +6,8 @@ import scippnexus as snx from ess.reduce.nexus.types import FilePath, NeXusFile +from .types import ControlMode + def _validate_entry(entry: snx.Group) -> None: if str(entry.attrs['NX_class']) != 'NXlauetof': @@ -45,11 +47,28 @@ def _handle_sample(sample: snx.Group) -> sc.DataGroup: return sample_dg +def _handle_monitor(control_dg: sc.DataGroup, control: snx.Group) -> sc.DataGroup: + tof_bin_coord_key = 'tof_bin_coord' + + if tof_bin_coord_key in control.attrs: + tof_bin_coord = control.attrs['tof_bin_coord'] + control_dg['tof_bin_coord'] = tof_bin_coord + data: sc.DataArray = control_dg['data'] + data.coords[tof_bin_coord] = data.coords.pop('time_of_flight') + + control_dg['mode'] = ControlMode[control_dg['mode']] + + return control_dg + + def load_essnmx_nxlauetof(file: str | FilePath | NeXusFile) -> sc.DataGroup: dg = snx.load(file) with snx.File(file) as f: _validate_entry(entry := f['entry']) dg['entry']['sample'] = _handle_sample(entry['sample']) + dg['entry']['control'] = _handle_monitor( + dg['entry']['control'], entry['control'] + ) return dg['entry'] diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index d63923a0..749113a4 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -256,7 +256,7 @@ def reduction( tof_bin_coord=t_coord_name, # TODO: Use real monitor data # Currently NMX simulations or experiments do not have monitors - monitor_histogram=sc.DataArray( + data=sc.DataArray( coords={t_coord_name: t_bin_edges}, data=sc.ones_like(t_bin_edges[:-1]), ), diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 220d3ccd..767a7107 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -109,7 +109,13 @@ def _zero_float_count() -> sc.Variable: @dataclass(kw_only=True) class NMXMonitorMetadata: nx_class = snx.NXmonitor - monitor_histogram: sc.DataArray + data: sc.DataArray + """Monitor counts.""" + + @property + def time_of_flight(self) -> sc.Variable: + return self.data.coords[self.tof_bin_coord] + tof_bin_coord: str = field( default='tof', metadata={ @@ -127,14 +133,14 @@ class NMXMonitorMetadata: ) def __write_to_nexus_group__(self, group: h5py.Group): + group.attrs['axes'] = self.data.dims + group.attrs['tof_bin_coord'] = self.tof_bin_coord snx.create_field(group, 'mode', str(self.mode)) snx.create_field(group, 'preset', self.preset) - data_field = snx.create_field(group, 'data', self.monitor_histogram.data) + data_field = snx.create_field(group, 'data', self.data.data) data_field.attrs['signal'] = 1 data_field.attrs['primary'] = 1 - snx.create_field( - group, 'time_of_flight', self.monitor_histogram.coords[self.tof_bin_coord] - ) + snx.create_field(group, 'time_of_flight', self.time_of_flight) @dataclass(kw_only=True) From 200bfe1a6f33d03c636e27255f7a4e29f644a9c2 Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 24 Feb 2026 15:19:22 +0100 Subject: [PATCH 381/403] Handle source and detector data to match the original data group. --- packages/essnmx/src/ess/nmx/_nxlauetof_io.py | 30 +++++++++++++++++++- packages/essnmx/src/ess/nmx/nexus.py | 1 + packages/essnmx/src/ess/nmx/types.py | 10 +++++-- packages/essnmx/src/ess/nmx/workflows.py | 2 +- 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py index 23d01d0b..0ebd6202 100644 --- a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py +++ b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py @@ -61,14 +61,42 @@ def _handle_monitor(control_dg: sc.DataGroup, control: snx.Group) -> sc.DataGrou return control_dg +def _handle_source(instrument_dg: sc.DataGroup, instrument: snx.Group) -> sc.DataGroup: + distance = instrument_dg['source'].pop('distance') + position = sc.vector( + instrument['source']['distance'].attrs['position'], unit=distance.unit + ) + instrument_dg['source']['position'] = position + + +def _handle_detector_data( + instrument_dg: sc.DataGroup, instrument: snx.Group +) -> sc.DataGroup: + detectors: sc.DataGroup[sc.DataGroup] = sc.DataGroup( + { + det_name: instrument_dg.pop(det_name) + for det_name in instrument[snx.NXdetector].keys() + } + ) + instrument_dg['detectors'] = detectors + for det_gr in detectors.values(): + all_keys = list(filter(lambda key: key != 'data', det_gr.keys())) + metadatas = sc.DataGroup() + for key in all_keys: + metadatas[key] = det_gr.pop(key) + det_gr['metadata'] = metadatas + + def load_essnmx_nxlauetof(file: str | FilePath | NeXusFile) -> sc.DataGroup: dg = snx.load(file) - with snx.File(file) as f: + with snx.File(file, mode='r') as f: _validate_entry(entry := f['entry']) dg['entry']['sample'] = _handle_sample(entry['sample']) dg['entry']['control'] = _handle_monitor( dg['entry']['control'], entry['control'] ) + _handle_source(dg['entry']['instrument'], entry['instrument']) + _handle_detector_data(dg['entry']['instrument'], entry['instrument']) return dg['entry'] diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/nexus.py index ee0c90f0..d6117a93 100644 --- a/packages/essnmx/src/ess/nmx/nexus.py +++ b/packages/essnmx/src/ess/nmx/nexus.py @@ -296,6 +296,7 @@ def export_reduced_data_as_nxlauetof( ) data_dset.attrs["signal"] = 1 + data_dset.attrs["axes"] = list(da.dims) if 'tof' in da.coords: _create_dataset_from_var( diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 767a7107..206d8949 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -89,16 +89,20 @@ def __write_to_nexus_group__(self, group: h5py.Group): @dataclass(kw_only=True) class NMXSourceMetadata: nx_class = snx.NXsource - source_position: sc.Variable + position: sc.Variable name: Literal['European Spallation Source'] = "European Spallation Source" type: Literal['Spallation Neutron Source'] = "Spallation Neutron Source" probe: Literal['neutron'] = "neutron" + @property + def distance(self) -> sc.Variable: + return sc.norm(self.position) + def __write_to_nexus_group__(self, group: h5py.Group): snx.create_field(group, 'name', self.name) snx.create_field(group, 'type', self.type) - distance = snx.create_field(group, 'distance', sc.norm(self.source_position)) - distance.attrs['position'] = self.source_position.values + distance = snx.create_field(group, 'distance', self.distance) + distance.attrs['position'] = self.position.values snx.create_field(group, 'probe', self.probe) diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index 9d702e19..2e5b0330 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -136,7 +136,7 @@ def assemble_source_metadata( source_position: Position[snx.NXsource, SampleRun], ) -> NMXSourceMetadata: """Assemble source metadata for NMX reduction workflow.""" - return NMXSourceMetadata(source_position=source_position) + return NMXSourceMetadata(position=source_position) def _decide_fast_axis(da: sc.DataArray) -> str: From 170c01e553ee04e33abc67838c3b813796732a64 Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 24 Feb 2026 16:28:32 +0100 Subject: [PATCH 382/403] Add documentation about fast/slow axis. --- packages/essnmx/src/ess/nmx/types.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 206d8949..79e74bd6 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -156,7 +156,23 @@ class NMXDetectorMetadata: y_pixel_size: sc.Variable origin_position: sc.Variable fast_axis: sc.Variable + """The index of the fast axis changes fast along the detector number. + + i.e. When detector numbers grows: ``1, 2, 3, 4, 5, ...`` + and the size of the fast axis is ``3``, + the fast axis index will be: ``1, 2, 3, 1, 2, ...`` + for each detector number. + + """ slow_axis: sc.Variable + """The index of the slow axis changes slowly along the detector number. + + i.e.When detector numbers grows: ``1, 2, 3, 4, 5, 6, 7, ...`` + and the size of the fast axis is ``3``, + the slow axis index will be: ``1, 1, 1, 2, 2, 2, 3, ...`` + for each detector number. + + """ distance: sc.Variable first_pixel_position: sc.Variable """First pixel position with respect to the sample. From ae01de1b00dc3bc31e6a43baf1cd2a1a318f2510 Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 24 Feb 2026 16:32:27 +0100 Subject: [PATCH 383/403] Add documentation about fast/slow axis. --- packages/essnmx/src/ess/nmx/types.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 79e74bd6..9f75438b 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -156,20 +156,24 @@ class NMXDetectorMetadata: y_pixel_size: sc.Variable origin_position: sc.Variable fast_axis: sc.Variable - """The index of the fast axis changes fast along the detector number. + """Inner most dimension if the data is sorted by detector number. - i.e. When detector numbers grows: ``1, 2, 3, 4, 5, ...`` + The index of the fast axis changes fast along the detector number. + + i.e. When detector numbers grows: ``0, 1, 2, 3, 4, 5, 6, ...`` and the size of the fast axis is ``3``, - the fast axis index will be: ``1, 2, 3, 1, 2, ...`` + the fast axis index will be: ``0, 1, 2, 0, 1, 2, 0 ...`` for each detector number. """ slow_axis: sc.Variable - """The index of the slow axis changes slowly along the detector number. + """Outer most dimension if the data is sorted by detector number. + + The index of the slow axis changes slowly along the detector number. - i.e.When detector numbers grows: ``1, 2, 3, 4, 5, 6, 7, ...`` + i.e. When detector numbers grows: ``0, 1, 2, 3, 4, 5, 6, ...`` and the size of the fast axis is ``3``, - the slow axis index will be: ``1, 1, 1, 2, 2, 2, 3, ...`` + the slow axis index will be: ``0, 0, 0, 1, 1, 1, 2, ...`` for each detector number. """ From 5c861972fe487b4b0c0b2e92770d6ee7766081c7 Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Tue, 24 Feb 2026 17:56:17 +0100 Subject: [PATCH 384/403] Restore positions. --- packages/essnmx/src/ess/nmx/_nxlauetof_io.py | 96 ++++++++++++++++++-- packages/essnmx/src/ess/nmx/types.py | 8 +- packages/essnmx/src/ess/nmx/workflows.py | 2 + 3 files changed, 97 insertions(+), 9 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py index 0ebd6202..6a9d2f3f 100644 --- a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py +++ b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py @@ -33,8 +33,7 @@ def _as_vector(var: sc.Variable) -> sc.Variable: return var -def _handle_sample(sample: snx.Group) -> sc.DataGroup: - sample_dg = sample[...] +def _handle_sample(sample_dg: sc.DataGroup, sample: snx.Group) -> sc.DataGroup: sample_dg['crystal_rotation'] = _as_vector(sample_dg['crystal_rotation']) sample_dg['position'] = _as_vector(sample_dg['position']) unit_cell = sample_dg.pop('unit_cell') @@ -69,6 +68,68 @@ def _handle_source(instrument_dg: sc.DataGroup, instrument: snx.Group) -> sc.Dat instrument_dg['source']['position'] = position +def _restore_bin_edges(time_midpoints: sc.Variable) -> sc.Variable: + left = time_midpoints[:-1] + right = time_midpoints[1:] + widths = right - left + if not bool(sc.allclose(sc.broadcast(widths[0], sizes=widths.sizes), widths)): + warnings.warn( + "Time coordinate does not have uniform width. Cannot restore bid-edges. " + "Keeping midpoint values...", + UserWarning, + stacklevel=3, + ) + return time_midpoints + + half_width = widths[0] / 2 + return sc.concat( + [time_midpoints - half_width, time_midpoints[-1] + half_width], + dim=time_midpoints.dim, + ) + + +def _restore_positions( + *, metadatas: sc.DataGroup, fast_axis_dim: str, slow_axis_dim: str, sizes: dict +) -> sc.Variable: + fast_axis = metadatas['fast_axis'] + fast_axis_size = sizes[fast_axis_dim] + slow_axis = metadatas['slow_axis'] + slow_axis_size = sizes[slow_axis_dim] + + pixel_sizes = { + 'x_pixel_offset': metadatas['x_pixel_size'], + 'y_pixel_offset': metadatas['y_pixel_size'], + } + + fast_axis_offsets = ( + sc.arange(dim=fast_axis_dim, start=0.0, stop=fast_axis_size) + * pixel_sizes[fast_axis_dim] + * fast_axis + ) + slow_axis_offsets = ( + sc.arange(dim=slow_axis_dim, start=0.0, stop=slow_axis_size) + * pixel_sizes[slow_axis_dim] + * slow_axis + ) + # The slow axis should be the outer most dimension. + detector_sizes = {slow_axis_dim: slow_axis_size, fast_axis_dim: fast_axis_size} + + pixel_offsets = fast_axis_offsets.broadcast( + sizes=detector_sizes + ) + slow_axis_offsets.broadcast(sizes=detector_sizes) + + detetor_center = metadatas['origin'] + slow_axis_width = pixel_sizes[slow_axis_dim] * slow_axis_size + fast_axis_width = pixel_sizes[fast_axis_dim] * fast_axis_size + detector_corner = ( + detetor_center + - (slow_axis_width / 2) * slow_axis + - (fast_axis_width / 2) * fast_axis + ) + + return pixel_offsets + detector_corner + + def _handle_detector_data( instrument_dg: sc.DataGroup, instrument: snx.Group ) -> sc.DataGroup: @@ -79,12 +140,35 @@ def _handle_detector_data( } ) instrument_dg['detectors'] = detectors - for det_gr in detectors.values(): + time_coord_name = next(iter({'tof', 'event_time_offset'} & set(detectors.dims))) + time_field_name = ( + 'time_of_flight' if time_coord_name == 'tof' else 'event_time_offset' + ) + + for det_name, det_gr in detectors.items(): all_keys = list(filter(lambda key: key != 'data', det_gr.keys())) metadatas = sc.DataGroup() for key in all_keys: metadatas[key] = det_gr.pop(key) + + for vector_field in ('slow_axis', 'fast_axis', 'origin'): + metadatas[vector_field] = _as_vector(metadatas[vector_field]) + det_gr['metadata'] = metadatas + slow_axis_dim = instrument[det_name]['slow_axis'].attrs['dim'] + fast_axis_dim = instrument[det_name]['fast_axis'].attrs['dim'] + det_gr['data'] = sc.DataArray( + data=det_gr['data'], + coords={ + time_coord_name: _restore_bin_edges(metadatas[time_field_name]), + 'position': _restore_positions( + metadatas=metadatas, + fast_axis_dim=fast_axis_dim, + slow_axis_dim=slow_axis_dim, + sizes=det_gr['data'].sizes, + ), + }, + ) def load_essnmx_nxlauetof(file: str | FilePath | NeXusFile) -> sc.DataGroup: @@ -92,10 +176,8 @@ def load_essnmx_nxlauetof(file: str | FilePath | NeXusFile) -> sc.DataGroup: with snx.File(file, mode='r') as f: _validate_entry(entry := f['entry']) - dg['entry']['sample'] = _handle_sample(entry['sample']) - dg['entry']['control'] = _handle_monitor( - dg['entry']['control'], entry['control'] - ) + _handle_sample(dg['entry']['sample'], entry['sample']) + _handle_monitor(dg['entry']['control'], entry['control']) _handle_source(dg['entry']['instrument'], entry['instrument']) _handle_detector_data(dg['entry']['instrument'], entry['instrument']) diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 9f75438b..d2dbf45f 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -166,6 +166,7 @@ class NMXDetectorMetadata: for each detector number. """ + fast_axis_dim: str slow_axis: sc.Variable """Outer most dimension if the data is sorted by detector number. @@ -177,6 +178,7 @@ class NMXDetectorMetadata: for each detector number. """ + slow_axis_dim: str distance: sc.Variable first_pixel_position: sc.Variable """First pixel position with respect to the sample. @@ -194,8 +196,10 @@ def __write_to_nexus_group__(self, group: h5py.Group): snx.create_field(group, 'y_pixel_size', self.y_pixel_size) origin = snx.create_field(group, 'origin', self.origin_position) origin.attrs['first_pixel_position'] = self.first_pixel_position.values - snx.create_field(group, 'fast_axis', self.fast_axis) - snx.create_field(group, 'slow_axis', self.slow_axis) + fast_axis = snx.create_field(group, 'fast_axis', self.fast_axis) + fast_axis.attrs['dim'] = self.fast_axis_dim + slow_axis = snx.create_field(group, 'slow_axis', self.slow_axis) + slow_axis.attrs['dim'] = self.slow_axis_dim snx.create_field(group, 'distance', self.distance) snx.create_field(group, 'polar_angle', self.polar_angle) snx.create_field(group, 'azimuthal_angle', self.azimuthal_angle) diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index 2e5b0330..7870086e 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -235,7 +235,9 @@ def assemble_detector_metadata( y_pixel_size=y_pixel_size, origin_position=origin, fast_axis=_normalize_vector(fast_axis_vector), + fast_axis_dim=_fast_axis + '_pixel_offset', slow_axis=_normalize_vector(slow_axis_vector), + slow_axis_dim=_slow_axis + '_pixel_offset', distance=distance, first_pixel_position=first_pixel_position_from_sample, ) From 464c9d04f464d55ad28bb4aae439c3e6a8c879a6 Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 25 Feb 2026 15:22:57 +0100 Subject: [PATCH 385/403] Fix geometry reconstruction. --- packages/essnmx/src/ess/nmx/_nxlauetof_io.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py index 6a9d2f3f..93d87697 100644 --- a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py +++ b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py @@ -124,7 +124,9 @@ def _restore_positions( detector_corner = ( detetor_center - (slow_axis_width / 2) * slow_axis + + pixel_sizes[slow_axis_dim] * slow_axis / 2 - (fast_axis_width / 2) * fast_axis + + pixel_sizes[fast_axis_dim] * fast_axis / 2 ) return pixel_offsets + detector_corner From bd4a367926d3826b5d27b476f0d2440da2a4344a Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 25 Feb 2026 15:44:38 +0100 Subject: [PATCH 386/403] Store original time coordinate bin edges. --- packages/essnmx/src/ess/nmx/_nxlauetof_io.py | 25 +--------------- packages/essnmx/src/ess/nmx/nexus.py | 30 ++++++++++++-------- 2 files changed, 19 insertions(+), 36 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py index 93d87697..e00c6352 100644 --- a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py +++ b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py @@ -68,26 +68,6 @@ def _handle_source(instrument_dg: sc.DataGroup, instrument: snx.Group) -> sc.Dat instrument_dg['source']['position'] = position -def _restore_bin_edges(time_midpoints: sc.Variable) -> sc.Variable: - left = time_midpoints[:-1] - right = time_midpoints[1:] - widths = right - left - if not bool(sc.allclose(sc.broadcast(widths[0], sizes=widths.sizes), widths)): - warnings.warn( - "Time coordinate does not have uniform width. Cannot restore bid-edges. " - "Keeping midpoint values...", - UserWarning, - stacklevel=3, - ) - return time_midpoints - - half_width = widths[0] / 2 - return sc.concat( - [time_midpoints - half_width, time_midpoints[-1] + half_width], - dim=time_midpoints.dim, - ) - - def _restore_positions( *, metadatas: sc.DataGroup, fast_axis_dim: str, slow_axis_dim: str, sizes: dict ) -> sc.Variable: @@ -143,9 +123,6 @@ def _handle_detector_data( ) instrument_dg['detectors'] = detectors time_coord_name = next(iter({'tof', 'event_time_offset'} & set(detectors.dims))) - time_field_name = ( - 'time_of_flight' if time_coord_name == 'tof' else 'event_time_offset' - ) for det_name, det_gr in detectors.items(): all_keys = list(filter(lambda key: key != 'data', det_gr.keys())) @@ -162,7 +139,7 @@ def _handle_detector_data( det_gr['data'] = sc.DataArray( data=det_gr['data'], coords={ - time_coord_name: _restore_bin_edges(metadatas[time_field_name]), + time_coord_name: metadatas.pop('original_time_edges'), 'position': _restore_positions( metadatas=metadatas, fast_axis_dim=fast_axis_dim, diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/nexus.py index d6117a93..9ee2bca5 100644 --- a/packages/essnmx/src/ess/nmx/nexus.py +++ b/packages/essnmx/src/ess/nmx/nexus.py @@ -299,18 +299,24 @@ def export_reduced_data_as_nxlauetof( data_dset.attrs["axes"] = list(da.dims) if 'tof' in da.coords: - _create_dataset_from_var( - name='time_of_flight', - root_entry=nx_detector, - var=sc.midpoints(da.coords['tof'], dim='tof'), - ) + time_field_name = "time_of_flight" + time_coord_name = "tof" + time_dim = "tof" elif 'event_time_offset' in da.coords: - _create_dataset_from_var( - name='event_time_offset', - root_entry=nx_detector, - var=sc.midpoints( - da.coords['event_time_offset'], dim='event_time_offset' - ), - ) + time_field_name = "event_time_offset" + time_coord_name = "event_time_offset" + time_dim = "event_time_offset" else: raise ValueError("Could not find time-related bin edges to store.") + + _create_dataset_from_var( + name=time_field_name, + root_entry=nx_detector, + var=sc.midpoints(da.coords[time_coord_name], dim=time_dim), + ) + + _create_dataset_from_var( + name='original_time_edges', + root_entry=nx_detector, + var=da.coords[time_coord_name], + ) From 1d9dff000a7b25dd2c81ce5be326ccc53db27b32 Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 25 Feb 2026 16:23:17 +0100 Subject: [PATCH 387/403] Adjust detector metadata name to match the field name in the output file as much as possible and fix unit test according to the chnages. --- packages/essnmx/src/ess/nmx/_nxlauetof_io.py | 28 +++++++++++++++---- packages/essnmx/src/ess/nmx/types.py | 5 ++-- packages/essnmx/src/ess/nmx/workflows.py | 2 +- .../essnmx/tests/nxlauetof_io_helper_test.py | 27 ++++++++++++++++-- 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py index e00c6352..23cddb87 100644 --- a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py +++ b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py @@ -123,23 +123,39 @@ def _handle_detector_data( ) instrument_dg['detectors'] = detectors time_coord_name = next(iter({'tof', 'event_time_offset'} & set(detectors.dims))) + time_field_name = 'time_of_flight' if time_coord_name == 'tof' else time_coord_name for det_name, det_gr in detectors.items(): - all_keys = list(filter(lambda key: key != 'data', det_gr.keys())) - metadatas = sc.DataGroup() - for key in all_keys: - metadatas[key] = det_gr.pop(key) + # These fields are part of the histogram as data and coordinate. + non_meta_keys = ('data', 'time_of_flight', 'event_time_offset') + all_keys = list(filter(lambda key: key not in non_meta_keys, det_gr.keys())) + metadatas = sc.DataGroup({key: det_gr.pop(key) for key in all_keys}) for vector_field in ('slow_axis', 'fast_axis', 'origin'): metadatas[vector_field] = _as_vector(metadatas[vector_field]) det_gr['metadata'] = metadatas - slow_axis_dim = instrument[det_name]['slow_axis'].attrs['dim'] fast_axis_dim = instrument[det_name]['fast_axis'].attrs['dim'] + slow_axis_dim = instrument[det_name]['slow_axis'].attrs['dim'] + metadatas['fast_axis_dim'] = fast_axis_dim + metadatas['slow_axis_dim'] = slow_axis_dim + metadatas['detector_name'] = det_name + metadatas['first_pixel_position'] = sc.vector( + instrument[det_name]['origin'].attrs['first_pixel_position'], + unit=metadatas['origin'].unit, + ) + time_coord = metadatas.pop('original_time_edges') + mid_time = det_gr.pop(time_field_name) + if sc.any(sc.midpoints(time_coord) != mid_time): + warnings.warn( + "Time bin edges and mid point coordinates do not agree.", + UserWarning, + stacklevel=3, + ) det_gr['data'] = sc.DataArray( data=det_gr['data'], coords={ - time_coord_name: metadatas.pop('original_time_edges'), + time_coord_name: time_coord, 'position': _restore_positions( metadatas=metadatas, fast_axis_dim=fast_axis_dim, diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index d2dbf45f..8188cd5a 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -154,7 +154,8 @@ class NMXDetectorMetadata: detector_name: str x_pixel_size: sc.Variable y_pixel_size: sc.Variable - origin_position: sc.Variable + origin: sc.Variable + """Center of the detector panel.""" fast_axis: sc.Variable """Inner most dimension if the data is sorted by detector number. @@ -194,7 +195,7 @@ class NMXDetectorMetadata: def __write_to_nexus_group__(self, group: h5py.Group): snx.create_field(group, 'x_pixel_size', self.x_pixel_size) snx.create_field(group, 'y_pixel_size', self.y_pixel_size) - origin = snx.create_field(group, 'origin', self.origin_position) + origin = snx.create_field(group, 'origin', self.origin) origin.attrs['first_pixel_position'] = self.first_pixel_position.values fast_axis = snx.create_field(group, 'fast_axis', self.fast_axis) fast_axis.attrs['dim'] = self.fast_axis_dim diff --git a/packages/essnmx/src/ess/nmx/workflows.py b/packages/essnmx/src/ess/nmx/workflows.py index 7870086e..2004bf05 100644 --- a/packages/essnmx/src/ess/nmx/workflows.py +++ b/packages/essnmx/src/ess/nmx/workflows.py @@ -233,7 +233,7 @@ def assemble_detector_metadata( detector_name=detector_component['nexus_component_name'], x_pixel_size=x_pixel_size, y_pixel_size=y_pixel_size, - origin_position=origin, + origin=origin, fast_axis=_normalize_vector(fast_axis_vector), fast_axis_dim=_fast_axis + '_pixel_offset', slow_axis=_normalize_vector(slow_axis_vector), diff --git a/packages/essnmx/tests/nxlauetof_io_helper_test.py b/packages/essnmx/tests/nxlauetof_io_helper_test.py index 9d185636..b8db64a5 100644 --- a/packages/essnmx/tests/nxlauetof_io_helper_test.py +++ b/packages/essnmx/tests/nxlauetof_io_helper_test.py @@ -4,7 +4,7 @@ from contextlib import contextmanager import pytest -from scipp.testing.assertions import assert_identical +from scipp.testing.assertions import assert_allclose, assert_identical from ess.nmx._nxlauetof_io import load_essnmx_nxlauetof from ess.nmx.configurations import InputConfig, OutputConfig, ReductionConfig @@ -46,7 +46,30 @@ def test_loaded_data_same_as_in_memory_result( result = reduction(config=reduction_config) original_result_dg = result.to_datagroup() + # Adjust original result to be same as expected loaded data group. + original_result_dg.pop('lookup_table') + original_positions = {} + detectors = original_result_dg['instrument']['detectors'] + for det_name, det in detectors.items(): + # Removing coordinates that are not kept in the file or reconstructed. + det['data'].coords.pop('Ltotal') + det['data'].coords.pop('detector_number') + det['data'].coords.pop('x_pixel_offset') + det['data'].coords.pop('y_pixel_offset') + # Saving position coordinate to compare them by allclose instead of eq. + original_positions[det_name] = det['data'].coords.pop('position') + with pytest.warns(UserWarning, match=r'Could not determine'): loaded_dg = load_essnmx_nxlauetof(reduction_config.output.output_file) - assert_identical(loaded_dg['sample'], original_result_dg['sample']) + loaded_detector_positions = {} + for det_name, loaded_det in loaded_dg['instrument']['detectors'].items(): + loaded_detector_positions[det_name] = loaded_det['data'].coords.pop('position') + + assert_identical(loaded_dg, original_result_dg) + # Using the x_pixel_size of the first panel to get absolute tolerance. + pixel_size = next(iter(detectors.values()))['metadata']['x_pixel_size'] + atol = pixel_size / 10.0 + for det_name, original_position in original_positions.items(): + loaded_position = loaded_detector_positions[det_name] + assert_allclose(original_position, loaded_position, atol=atol) From d4b1d2b1a32ec96a1b19335dc30e1f1087a9305c Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 25 Feb 2026 16:44:12 +0100 Subject: [PATCH 388/403] Add program information. --- packages/essnmx/src/ess/nmx/executables.py | 1 + packages/essnmx/src/ess/nmx/nexus.py | 3 +++ packages/essnmx/src/ess/nmx/types.py | 15 +++++++++++++++ 3 files changed, 19 insertions(+) diff --git a/packages/essnmx/src/ess/nmx/executables.py b/packages/essnmx/src/ess/nmx/executables.py index 749113a4..21643caf 100644 --- a/packages/essnmx/src/ess/nmx/executables.py +++ b/packages/essnmx/src/ess/nmx/executables.py @@ -300,6 +300,7 @@ def save_results(*, results: NMXLauetof, output_config: OutputConfig) -> None: export_static_metadata_as_nxlauetof( sample_metadata=results.sample, source_metadata=results.instrument.source, + program=results.reducer, output_file=output_config.output_file, overwrite=output_config.overwrite, ) diff --git a/packages/essnmx/src/ess/nmx/nexus.py b/packages/essnmx/src/ess/nmx/nexus.py index 9ee2bca5..20a71d3e 100644 --- a/packages/essnmx/src/ess/nmx/nexus.py +++ b/packages/essnmx/src/ess/nmx/nexus.py @@ -14,6 +14,7 @@ from .types import ( NMXDetectorMetadata, NMXMonitorMetadata, + NMXProgram, NMXSampleMetadata, NMXSourceMetadata, ) @@ -146,6 +147,7 @@ def export_static_metadata_as_nxlauetof( *, sample_metadata: NMXSampleMetadata, source_metadata: NMXSourceMetadata, + program: NMXProgram, output_file: str | pathlib.Path | io.BytesIO, overwrite: bool = False, **arbitrary_metadata: sc.Variable, @@ -178,6 +180,7 @@ def export_static_metadata_as_nxlauetof( nx_entry = f.create_class(name='entry', class_name='NXlauetof') nx_entry.create_field('definitions', value='NXlauetof') nx_entry['sample'] = sample_metadata + nx_entry['reducer'] = program nx_instrument = _set_default_instrument(nx_entry) nx_instrument['source'] = source_metadata diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 8188cd5a..d6aefa9d 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -236,6 +236,19 @@ class NMXInstrument: source: NMXSourceMetadata +@dataclass(kw_only=True) +class NMXProgram: + nx_class = 'NXprogram' + + program: str = 'essnmx' + + def __write_to_nexus_group__(self, group: h5py.Group): + from ess.nmx import __version__ as essnmxversion + + prog = snx.create_field(group, 'program', self.program) + prog.attrs['version'] = essnmxversion + + @dataclass(kw_only=True) class NMXLauetof: nx_class = "NXlauetof" @@ -245,6 +258,8 @@ class NMXLauetof: instrument: NMXInstrument sample: NMXSampleMetadata lookup_table: TofLookupTable | None = None + reducer: NMXProgram = field(default_factory=NMXProgram) + "Information of the reduction software." def to_datagroup(self) -> sc.DataGroup: return to_datagroup(self) From 3741fa3a3800b201ead60efd6b8191964c0af5f8 Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 25 Feb 2026 17:04:43 +0100 Subject: [PATCH 389/403] Suppress warning since they will be handled. --- packages/essnmx/src/ess/nmx/_nxlauetof_io.py | 7 +++++-- packages/essnmx/tests/nxlauetof_io_helper_test.py | 3 +-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py index 23cddb87..8ceef362 100644 --- a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py +++ b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py @@ -167,9 +167,12 @@ def _handle_detector_data( def load_essnmx_nxlauetof(file: str | FilePath | NeXusFile) -> sc.DataGroup: - dg = snx.load(file) - with snx.File(file, mode='r') as f: + with warnings.catch_warnings(action='ignore'): + # Expecting warnings for loading NXdetectors. + # The data array reconstruction is handled manually later. + dg = f[()] + _validate_entry(entry := f['entry']) _handle_sample(dg['entry']['sample'], entry['sample']) _handle_monitor(dg['entry']['control'], entry['control']) diff --git a/packages/essnmx/tests/nxlauetof_io_helper_test.py b/packages/essnmx/tests/nxlauetof_io_helper_test.py index b8db64a5..46264f24 100644 --- a/packages/essnmx/tests/nxlauetof_io_helper_test.py +++ b/packages/essnmx/tests/nxlauetof_io_helper_test.py @@ -59,8 +59,7 @@ def test_loaded_data_same_as_in_memory_result( # Saving position coordinate to compare them by allclose instead of eq. original_positions[det_name] = det['data'].coords.pop('position') - with pytest.warns(UserWarning, match=r'Could not determine'): - loaded_dg = load_essnmx_nxlauetof(reduction_config.output.output_file) + loaded_dg = load_essnmx_nxlauetof(reduction_config.output.output_file) loaded_detector_positions = {} for det_name, loaded_det in loaded_dg['instrument']['detectors'].items(): From e0fc735a6d867d11b98d90303fc8ade436dbadee Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 25 Feb 2026 17:05:03 +0100 Subject: [PATCH 390/403] Add loader example. --- .../essnmx/docs/user-guide/workflow.ipynb | 77 ++++++++++++++++++- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/packages/essnmx/docs/user-guide/workflow.ipynb b/packages/essnmx/docs/user-guide/workflow.ipynb index 89bae585..a13747ad 100644 --- a/packages/essnmx/docs/user-guide/workflow.ipynb +++ b/packages/essnmx/docs/user-guide/workflow.ipynb @@ -40,7 +40,7 @@ " detector_ids=[0, 1, 2],\n", " ),\n", " output=OutputConfig(\n", - " output_file=\"scipp_output.hdf\", skip_file_output=True, overwrite=True\n", + " output_file=\"scipp_output.hdf\", skip_file_output=False, overwrite=True\n", " ),\n", " workflow=WorkflowConfig(\n", " time_bin_coordinate=TimeBinCoordinate.time_of_flight,\n", @@ -58,6 +58,26 @@ "dg" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from scippnexus import data\n", + "\n", + "filename = data.get_path('PG3_4844_event.nxs')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!cp '/home/sunyoungyoo/.cache/scippnexus/1/PG3_4844_event.nxs' ./" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -151,6 +171,57 @@ "save_results(results=results, output_config=output_config)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading Reduced File\n", + "\n", + "There is a custom loader for NXlauetof file for NMX.
\n", + "It reconstruct the position coordinates from the file and add it back to the data array.
\n", + "The data group should almost look the same as the in-memory results.
\n", + "The loaded data group will not have some coordinates compared to the in-memory results.
" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ess.nmx._nxlauetof_io import load_essnmx_nxlauetof\n", + "\n", + "loaded = load_essnmx_nxlauetof('scipp_output.hdf')\n", + "loaded" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can then plot the loaded data array exactly same as the in-memory results.\n", + "\n", + "For example, you can plot the 3D instrument view:\n", + "\n", + "```python\n", + "%matplotlib widget\n", + "import scipp as sc\n", + "import scippneutron as scn\n", + "\n", + "\n", + "dims=('y_pixel_offset', 'x_pixel_offset')\n", + "merged_2d_das = sc.concat(\n", + " [\n", + " det['data'].sum('tof').flatten(dims=dims, to='detector_number')\n", + " for det in loaded['instrument']['detectors'].values()\n", + " ],\n", + " dim='detector_number',\n", + ")\n", + "\n", + "scn.instrument_view(merged_2d_das)\n", + "```" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -192,7 +263,7 @@ ], "metadata": { "kernelspec": { - "display_name": "nmx-dev-313", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -206,7 +277,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.5" + "version": "3.12.12" } }, "nbformat": 4, From 3e64e7c464b8af97ff385270aaba3eb334eca225 Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 25 Feb 2026 17:31:28 +0100 Subject: [PATCH 391/403] Limit sphinx because of deprecation error. --- packages/essnmx/requirements/base.txt | 16 ++++++---------- packages/essnmx/requirements/ci.txt | 12 +++++------- packages/essnmx/requirements/dev.txt | 10 ++++------ packages/essnmx/requirements/docs.in | 2 +- packages/essnmx/requirements/docs.txt | 12 +++++++----- packages/essnmx/requirements/mypy.txt | 2 +- packages/essnmx/requirements/nightly.txt | 18 +++++++----------- packages/essnmx/requirements/static.txt | 6 +++--- 8 files changed, 34 insertions(+), 44 deletions(-) diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index b14873e7..490b115e 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -9,7 +9,7 @@ annotated-types==0.7.0 # via pydantic bitshuffle==0.5.2 ; os_name == "posix" # via -r base.in -certifi==2026.1.4 +certifi==2026.2.25 # via requests charset-normalizer==3.4.4 # via requests @@ -52,8 +52,6 @@ idna==3.11 # via # email-validator # requests -importlib-metadata==8.7.1 - # via dask kiwisolver==1.4.9 # via matplotlib lazy-loader==0.4 @@ -67,7 +65,7 @@ matplotlib==3.10.8 # via # mpltoolbox # plopp -mpltoolbox==25.10.0 +mpltoolbox==26.2.0 # via scippneutron msgpack==1.1.2 # via -r base.in @@ -89,15 +87,15 @@ packaging==26.0 # lazy-loader # matplotlib # pooch -pandas==3.0.0 +pandas==3.0.1 # via -r base.in partd==1.4.2 # via dask pillow==12.1.1 # via matplotlib -platformdirs==4.5.1 +platformdirs==4.9.2 # via pooch -plopp==26.2.0 +plopp==26.2.1 # via # -r base.in # scippneutron @@ -139,7 +137,7 @@ scippnexus==26.1.1 # -r base.in # essreduce # scippneutron -scipy==1.17.0 +scipy==1.17.1 # via # scippneutron # scippnexus @@ -160,8 +158,6 @@ typing-inspection==0.4.2 # via pydantic urllib3==2.6.3 # via requests -zipp==3.23.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index fd69cd42..c139b987 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -7,17 +7,15 @@ # cachetools==7.0.1 # via tox -certifi==2026.1.4 +certifi==2026.2.25 # via requests -chardet==5.2.0 - # via tox charset-normalizer==3.4.4 # via requests colorama==0.4.6 # via tox distlib==0.4.0 # via virtualenv -filelock==3.20.3 +filelock==3.24.3 # via # tox # virtualenv @@ -32,7 +30,7 @@ packaging==26.0 # -r ci.in # pyproject-api # tox -platformdirs==4.5.1 +platformdirs==4.9.2 # via # tox # virtualenv @@ -44,9 +42,9 @@ requests==2.32.5 # via -r ci.in smmap==5.0.2 # via gitdb -tox==4.34.1 +tox==4.46.3 # via -r ci.in urllib3==2.6.3 # via requests -virtualenv==20.36.1 +virtualenv==20.39.0 # via tox diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt index 1b6bda4c..7f1435b3 100644 --- a/packages/essnmx/requirements/dev.txt +++ b/packages/essnmx/requirements/dev.txt @@ -22,13 +22,13 @@ argon2-cffi-bindings==25.1.0 # via argon2-cffi arrow==1.4.0 # via isoduration -async-lru==2.1.0 +async-lru==2.2.0 # via jupyterlab cffi==2.0.0 # via argon2-cffi-bindings -copier==9.11.3 +copier==9.12.0 # via -r dev.in -dunamai==1.25.0 +dunamai==1.26.0 # via copier fqdn==1.5.1 # via jsonschema @@ -65,7 +65,7 @@ jupyter-server==2.17.0 # notebook-shim jupyter-server-terminals==0.5.4 # via jupyter-server -jupyterlab==4.5.4 +jupyterlab==4.5.5 # via -r dev.in jupyterlab-server==2.28.0 # via jupyterlab @@ -73,8 +73,6 @@ lark==1.3.1 # via rfc3987-syntax notebook-shim==0.2.4 # via jupyterlab -overrides==7.7.0 - # via jupyter-server pip-compile-multi==3.2.2 # via -r dev.in pip-tools==7.5.3 diff --git a/packages/essnmx/requirements/docs.in b/packages/essnmx/requirements/docs.in index 4693d337..9873d260 100644 --- a/packages/essnmx/requirements/docs.in +++ b/packages/essnmx/requirements/docs.in @@ -5,7 +5,7 @@ ipython!=8.7.0 # Breaks syntax highlighting in Jupyter code cells. myst-parser nbsphinx pydata-sphinx-theme>=0.14 -sphinx +sphinx<9.0.0 sphinx-autodoc-typehints sphinx-copybutton sphinx-design diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index c425ce7d..49132789 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -1,4 +1,4 @@ -# SHA1:ac8033c9d3c36ca4ae5c2f2b12733e14aa0155ff +# SHA1:e784872c1be8b2208cae1e80c661a0ec223964fc # # This file was generated by pip-compile-multi. # To update, run: @@ -36,7 +36,7 @@ debugpy==1.8.20 # via ipykernel decorator==5.2.1 # via ipython -docutils==0.22.4 +docutils==0.21.2 # via # myst-parser # nbsphinx @@ -137,7 +137,7 @@ ptyprocess==0.7.0 # via pexpect pure-eval==0.2.3 # via stack-data -pydantic-settings==2.12.0 +pydantic-settings==2.13.1 # via autodoc-pydantic pydata-sphinx-theme==0.16.1 # via -r docs.in @@ -162,6 +162,8 @@ referencing==0.37.0 # jsonschema # jsonschema-specifications roman-numerals==4.1.0 + # via roman-numerals-py +roman-numerals-py==4.1.0 # via sphinx rpds-py==0.30.0 # via @@ -171,7 +173,7 @@ snowballstemmer==3.0.1 # via sphinx soupsieve==2.8.3 # via beautifulsoup4 -sphinx==9.0.4 +sphinx==8.2.3 # via # -r docs.in # autodoc-pydantic @@ -181,7 +183,7 @@ sphinx==9.0.4 # sphinx-autodoc-typehints # sphinx-copybutton # sphinx-design -sphinx-autodoc-typehints==3.6.1 +sphinx-autodoc-typehints==3.5.2 # via -r docs.in sphinx-copybutton==0.5.2 # via -r docs.in diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index 83b4a62c..19fcddf5 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -6,7 +6,7 @@ # requirements upgrade # -r test.txt -librt==0.7.8 +librt==0.8.1 # via mypy mypy==1.19.1 # via -r mypy.in diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index a7250f3c..5178cd5a 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -12,7 +12,7 @@ annotated-types==0.7.0 # via pydantic bitshuffle==0.5.2 ; os_name == "posix" # via -r nightly.in -certifi==2026.1.4 +certifi==2026.2.25 # via requests charset-normalizer==3.4.4 # via requests @@ -55,8 +55,6 @@ idna==3.11 # via # email-validator # requests -importlib-metadata==8.7.1 - # via dask iniconfig==2.3.0 # via pytest kiwisolver==1.4.10rc0 @@ -72,7 +70,7 @@ matplotlib==3.10.8 # via # mpltoolbox # plopp -mpltoolbox==25.10.0 +mpltoolbox==26.2.0 # via scippneutron msgpack==1.1.2 # via -r nightly.in @@ -95,13 +93,13 @@ packaging==26.0 # matplotlib # pooch # pytest -pandas==3.0.0 +pandas==3.0.1 # via -r nightly.in partd==1.4.2 # via dask pillow==12.1.1 # via matplotlib -platformdirs==4.5.1 +platformdirs==4.9.2 # via pooch plopp @ git+https://github.com/scipp/plopp@main # via @@ -114,9 +112,9 @@ pooch==1.9.0 # via # -r nightly.in # tof -pydantic==2.12.5 +pydantic==2.13.0b2 # via scippneutron -pydantic-core==2.41.5 +pydantic-core==2.42.0 # via pydantic pygments==2.19.2 # via pytest @@ -151,7 +149,7 @@ scippnexus @ git+https://github.com/scipp/scippnexus@main # -r nightly.in # essreduce # scippneutron -scipy==1.17.0 +scipy==1.17.1 # via # scippneutron # scippnexus @@ -172,8 +170,6 @@ typing-inspection==0.4.2 # via pydantic urllib3==2.6.3 # via requests -zipp==3.23.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index 75cd3c9b..4c3fbc6f 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -9,17 +9,17 @@ cfgv==3.5.0 # via pre-commit distlib==0.4.0 # via virtualenv -filelock==3.20.3 +filelock==3.24.3 # via virtualenv identify==2.6.16 # via pre-commit nodeenv==1.10.0 # via pre-commit -platformdirs==4.5.1 +platformdirs==4.9.2 # via virtualenv pre-commit==4.5.1 # via -r static.in pyyaml==6.0.3 # via pre-commit -virtualenv==20.36.1 +virtualenv==20.39.0 # via pre-commit From d384859cb30f3aa459008fec985d6f317dba4a27 Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Wed, 25 Feb 2026 17:52:43 +0100 Subject: [PATCH 392/403] Fix docstring. --- packages/essnmx/src/ess/nmx/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index d6aefa9d..ca10f66f 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -32,7 +32,7 @@ class Compression(enum.StrEnum): class ControlMode(enum.StrEnum): """Control mode of counting. - Based on the NXlauetof definition of `control`(NXmonitor) field. + Based on the NXlauetof definition of ``control``(NXmonitor) field. """ monitor = 'monitor' From 8da05c8d406b2d4f417b990e4c9f3e524a0472a6 Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 26 Feb 2026 09:09:26 +0100 Subject: [PATCH 393/403] Fix docstring. --- packages/essnmx/src/ess/nmx/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index ca10f66f..2dec331d 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -32,7 +32,7 @@ class Compression(enum.StrEnum): class ControlMode(enum.StrEnum): """Control mode of counting. - Based on the NXlauetof definition of ``control``(NXmonitor) field. + Based on the NXlauetof definition of ``control`` (NXmonitor) field. """ monitor = 'monitor' From ad6355737b9dc011974ec85d2c24bedb59f7e009 Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 26 Feb 2026 14:13:17 +0100 Subject: [PATCH 394/403] Remove accidentally added cells. [skip ci] --- .../essnmx/docs/user-guide/workflow.ipynb | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/packages/essnmx/docs/user-guide/workflow.ipynb b/packages/essnmx/docs/user-guide/workflow.ipynb index a13747ad..1e3b05cf 100644 --- a/packages/essnmx/docs/user-guide/workflow.ipynb +++ b/packages/essnmx/docs/user-guide/workflow.ipynb @@ -58,26 +58,6 @@ "dg" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from scippnexus import data\n", - "\n", - "filename = data.get_path('PG3_4844_event.nxs')\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!cp '/home/sunyoungyoo/.cache/scippnexus/1/PG3_4844_event.nxs' ./" - ] - }, { "cell_type": "markdown", "metadata": {}, From d8d1e95190c23b71732cbea11051a71967079283 Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 26 Feb 2026 14:42:58 +0100 Subject: [PATCH 395/403] Fix grammar Co-authored-by: Jan-Lukas Wynen --- packages/essnmx/docs/user-guide/workflow.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/docs/user-guide/workflow.ipynb b/packages/essnmx/docs/user-guide/workflow.ipynb index 1e3b05cf..46084b32 100644 --- a/packages/essnmx/docs/user-guide/workflow.ipynb +++ b/packages/essnmx/docs/user-guide/workflow.ipynb @@ -158,7 +158,7 @@ "## Loading Reduced File\n", "\n", "There is a custom loader for NXlauetof file for NMX.
\n", - "It reconstruct the position coordinates from the file and add it back to the data array.
\n", + "It reconstructs the position coordinates from the file and adds them back to the data array.
\n", "The data group should almost look the same as the in-memory results.
\n", "The loaded data group will not have some coordinates compared to the in-memory results.
" ] From 593961d7e2d3c03c176fae37d9d20c05c206cd55 Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 26 Feb 2026 15:57:07 +0100 Subject: [PATCH 396/403] Use scippneutorn metadata types. --- packages/essnmx/pyproject.toml | 1 + packages/essnmx/src/ess/nmx/_nxlauetof_io.py | 8 ++++++-- packages/essnmx/src/ess/nmx/types.py | 20 ++++++++++++++++---- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index a2e05cc1..0c0aa161 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "sciline>=24.06.0", "scipp>=25.3.0", "scippnexus>=23.12.0", + "scippneutron>=26.02.0", "pooch>=1.5", "pandas>=2.1.2", "gemmi>=0.6.6", diff --git a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py index 8ceef362..f21dea0a 100644 --- a/packages/essnmx/src/ess/nmx/_nxlauetof_io.py +++ b/packages/essnmx/src/ess/nmx/_nxlauetof_io.py @@ -5,6 +5,7 @@ import scipp as sc import scippnexus as snx from ess.reduce.nexus.types import FilePath, NeXusFile +from scippneutron.metadata import RadiationProbe, SourceType from .types import ControlMode @@ -61,11 +62,14 @@ def _handle_monitor(control_dg: sc.DataGroup, control: snx.Group) -> sc.DataGrou def _handle_source(instrument_dg: sc.DataGroup, instrument: snx.Group) -> sc.DataGroup: - distance = instrument_dg['source'].pop('distance') + source_dg = instrument_dg['source'] + distance = source_dg.pop('distance') position = sc.vector( instrument['source']['distance'].attrs['position'], unit=distance.unit ) - instrument_dg['source']['position'] = position + source_dg['position'] = position + source_dg['source_type'] = SourceType(source_dg.pop('type')) + source_dg['probe'] = RadiationProbe(source_dg['probe']) def _restore_positions( diff --git a/packages/essnmx/src/ess/nmx/types.py b/packages/essnmx/src/ess/nmx/types.py index 2dec331d..cfc4301c 100644 --- a/packages/essnmx/src/ess/nmx/types.py +++ b/packages/essnmx/src/ess/nmx/types.py @@ -7,6 +7,7 @@ import scipp as sc import scippnexus as snx from ess.reduce.time_of_flight.types import TofLookupTable +from scippneutron.metadata import RadiationProbe, SourceType from ._display_helper import to_datagroup @@ -89,10 +90,21 @@ def __write_to_nexus_group__(self, group: h5py.Group): @dataclass(kw_only=True) class NMXSourceMetadata: nx_class = snx.NXsource + position: sc.Variable + """Position of the source (from the sample).""" + + # These three fields are matching fields as ``scippneutron.metadata.Source``. + # However, NMX needs to store `position` as a vector, + # not only the name, type and probe + # essnmx cannot use ``scippneutron.metadata.Source`` as it is. + # We will need to implement unpacking function for vector scalar value. + # Therefore we decided not to use the ``scippneutron.metadata.Source`` for now + # but the ``NMXSourceMetadata`` 's ``source_type`` and ``probe`` fields + # have the same Enum types as ``scippneutron.metadata.Source``. name: Literal['European Spallation Source'] = "European Spallation Source" - type: Literal['Spallation Neutron Source'] = "Spallation Neutron Source" - probe: Literal['neutron'] = "neutron" + source_type: SourceType = SourceType.SpallationNeutronSource + probe: RadiationProbe = RadiationProbe.Neutron @property def distance(self) -> sc.Variable: @@ -100,10 +112,10 @@ def distance(self) -> sc.Variable: def __write_to_nexus_group__(self, group: h5py.Group): snx.create_field(group, 'name', self.name) - snx.create_field(group, 'type', self.type) + snx.create_field(group, 'type', self.source_type.value) distance = snx.create_field(group, 'distance', self.distance) distance.attrs['position'] = self.position.values - snx.create_field(group, 'probe', self.probe) + snx.create_field(group, 'probe', self.probe.value) def _zero_float_count() -> sc.Variable: From 0fd72fda89e082db9fed650a488d0a75508c3fc0 Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Fri, 27 Feb 2026 16:23:53 +0100 Subject: [PATCH 397/403] Remove bitshuffle from the hard-dependencies. --- packages/essnmx/pyproject.toml | 1 - packages/essnmx/requirements/base.in | 2 +- packages/essnmx/requirements/base.txt | 17 +++++------------ packages/essnmx/requirements/ci.txt | 6 +++++- packages/essnmx/requirements/docs.txt | 3 --- packages/essnmx/requirements/mypy.txt | 3 --- packages/essnmx/requirements/nightly.in | 1 - packages/essnmx/requirements/nightly.txt | 13 ++----------- packages/essnmx/requirements/static.txt | 12 +++++++++--- packages/essnmx/requirements/test.txt | 3 --- 10 files changed, 22 insertions(+), 39 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 0c0aa161..03455125 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -44,7 +44,6 @@ dependencies = [ "defusedxml>=0.7.1", "msgpack>=1.0.8", "tof>=25.12.1", - "bitshuffle>=0.5.2;os_name == 'posix'" ] dynamic = ["version"] diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index c828022c..f3766be7 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -9,10 +9,10 @@ plopp>=24.7.0 sciline>=24.06.0 scipp>=25.3.0 scippnexus>=23.12.0 +scippneutron>=26.02.0 pooch>=1.5 pandas>=2.1.2 gemmi>=0.6.6 defusedxml>=0.7.1 msgpack>=1.0.8 tof>=25.12.1 -bitshuffle>=0.5.2;os_name == 'posix' diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 490b115e..66a35e75 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:690cd6e0a77540a2e9d253fc6be5f73d41322b4f +# SHA1:47709a55cff4a7391979e84117af1db9661bcab6 # # This file was generated by pip-compile-multi. # To update, run: @@ -7,8 +7,6 @@ # annotated-types==0.7.0 # via pydantic -bitshuffle==0.5.2 ; os_name == "posix" - # via -r base.in certifi==2026.2.25 # via requests charset-normalizer==3.4.4 @@ -23,8 +21,6 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -cython==3.2.4 - # via bitshuffle dask==2026.1.2 # via -r base.in defusedxml==0.7.1 @@ -45,7 +41,6 @@ graphviz==0.21 # via -r base.in h5py==3.15.1 # via - # bitshuffle # scippneutron # scippnexus idna==3.11 @@ -73,7 +68,6 @@ networkx==3.6.1 # via cyclebane numpy==2.4.2 # via - # bitshuffle # contourpy # h5py # matplotlib @@ -130,8 +124,10 @@ scipp==26.2.0 # scippneutron # scippnexus # tof -scippneutron==25.11.2 - # via essreduce +scippneutron==26.2.0 + # via + # -r base.in + # essreduce scippnexus==26.1.1 # via # -r base.in @@ -158,6 +154,3 @@ typing-inspection==0.4.2 # via pydantic urllib3==2.6.3 # via requests - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index c139b987..bbf20b65 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -17,6 +17,7 @@ distlib==0.4.0 # via virtualenv filelock==3.24.3 # via + # python-discovery # tox # virtualenv gitdb==4.0.12 @@ -32,12 +33,15 @@ packaging==26.0 # tox platformdirs==4.9.2 # via + # python-discovery # tox # virtualenv pluggy==1.6.0 # via tox pyproject-api==1.10.0 # via tox +python-discovery==1.1.0 + # via virtualenv requests==2.32.5 # via -r ci.in smmap==5.0.2 @@ -46,5 +50,5 @@ tox==4.46.3 # via -r ci.in urllib3==2.6.3 # via requests -virtualenv==20.39.0 +virtualenv==21.1.0 # via tox diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index 49132789..dfbfe06a 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -233,6 +233,3 @@ webencodings==0.5.1 # tinycss2 widgetsnbextension==4.0.15 # via ipywidgets - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index 19fcddf5..f90f757a 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -14,6 +14,3 @@ mypy-extensions==1.1.0 # via mypy pathspec==1.0.4 # via mypy - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index 1e2414a6..2aea7ef5 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -10,7 +10,6 @@ gemmi>=0.6.6 defusedxml>=0.7.1 msgpack>=1.0.8 tof>=25.12.1 -bitshuffle>=0.5.2;os_name == 'posix' pytest>=8.0 scipp --index-url=https://pypi.anaconda.org/scipp-nightly-wheels/simple/ diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 5178cd5a..831488a5 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:4e708d759352a2191073c0fe9616799eae59cf90 +# SHA1:f1c9ffe61d01dcdeb6285fbbf9def291d16747e2 # # This file was generated by pip-compile-multi. # To update, run: @@ -10,8 +10,6 @@ annotated-types==0.7.0 # via pydantic -bitshuffle==0.5.2 ; os_name == "posix" - # via -r nightly.in certifi==2026.2.25 # via requests charset-normalizer==3.4.4 @@ -26,8 +24,6 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib -cython==3.2.4 - # via bitshuffle dask==2026.1.2 # via -r nightly.in defusedxml==0.8.0rc2 @@ -48,7 +44,6 @@ graphviz==0.21 # via -r nightly.in h5py==3.15.1 # via - # bitshuffle # scippneutron # scippnexus idna==3.11 @@ -78,7 +73,6 @@ networkx==3.6.1 # via cyclebane numpy==2.4.2 # via - # bitshuffle # contourpy # h5py # matplotlib @@ -142,7 +136,7 @@ scipp==100.0.0.dev0 # scippneutron # scippnexus # tof -scippneutron==25.11.2 +scippneutron==26.2.0 # via essreduce scippnexus @ git+https://github.com/scipp/scippnexus@main # via @@ -170,6 +164,3 @@ typing-inspection==0.4.2 # via pydantic urllib3==2.6.3 # via requests - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index 4c3fbc6f..96a67092 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -10,16 +10,22 @@ cfgv==3.5.0 distlib==0.4.0 # via virtualenv filelock==3.24.3 - # via virtualenv + # via + # python-discovery + # virtualenv identify==2.6.16 # via pre-commit nodeenv==1.10.0 # via pre-commit platformdirs==4.9.2 - # via virtualenv + # via + # python-discovery + # virtualenv pre-commit==4.5.1 # via -r static.in +python-discovery==1.1.0 + # via virtualenv pyyaml==6.0.3 # via pre-commit -virtualenv==20.39.0 +virtualenv==21.1.0 # via pre-commit diff --git a/packages/essnmx/requirements/test.txt b/packages/essnmx/requirements/test.txt index c900374a..6817392e 100644 --- a/packages/essnmx/requirements/test.txt +++ b/packages/essnmx/requirements/test.txt @@ -7,6 +7,3 @@ # -r base.txt -r basetest.txt - -# The following packages are considered to be unsafe in a requirements file: -# setuptools From dd54e66c585cf61ac03998ad656928d9f97022ea Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Fri, 27 Feb 2026 16:35:53 +0100 Subject: [PATCH 398/403] Add bitshuffle as a test dependency --- packages/essnmx/pyproject.toml | 1 + packages/essnmx/requirements/basetest.in | 1 + packages/essnmx/requirements/basetest.txt | 15 ++++++++++++++- packages/essnmx/requirements/mypy.txt | 3 +++ packages/essnmx/requirements/nightly.in | 1 + packages/essnmx/requirements/nightly.txt | 11 ++++++++++- packages/essnmx/requirements/test.txt | 3 +++ 7 files changed, 33 insertions(+), 2 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 03455125..d1428258 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -55,6 +55,7 @@ essnmx-reduce = "ess.nmx.executables:main" [project.optional-dependencies] test = [ "pytest>=8.0", + "bitshuffle>=0.5.2;os_name == 'posix'", ] [project.urls] diff --git a/packages/essnmx/requirements/basetest.in b/packages/essnmx/requirements/basetest.in index 692bca17..e3fe12d6 100644 --- a/packages/essnmx/requirements/basetest.in +++ b/packages/essnmx/requirements/basetest.in @@ -8,3 +8,4 @@ # --- END OF CUSTOM SECTION --- # The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! pytest>=8.0 +bitshuffle>=0.5.2;os_name == 'posix' diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt index 2509493d..d336a671 100644 --- a/packages/essnmx/requirements/basetest.txt +++ b/packages/essnmx/requirements/basetest.txt @@ -1,12 +1,22 @@ -# SHA1:a183f2aaeff6c4f418995809266f8825553a276e +# SHA1:cf17c7dcdab3a1969e98e1b06b3aafc346f3d586 # # This file was generated by pip-compile-multi. # To update, run: # # requirements upgrade # +bitshuffle==0.5.2 ; os_name == "posix" + # via -r basetest.in +cython==3.2.4 + # via bitshuffle +h5py==3.15.1 + # via bitshuffle iniconfig==2.3.0 # via pytest +numpy==2.4.2 + # via + # bitshuffle + # h5py packaging==26.0 # via pytest pluggy==1.6.0 @@ -15,3 +25,6 @@ pygments==2.19.2 # via pytest pytest==9.0.2 # via -r basetest.in + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt index f90f757a..19fcddf5 100644 --- a/packages/essnmx/requirements/mypy.txt +++ b/packages/essnmx/requirements/mypy.txt @@ -14,3 +14,6 @@ mypy-extensions==1.1.0 # via mypy pathspec==1.0.4 # via mypy + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index 2aea7ef5..1e3e1641 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -11,6 +11,7 @@ defusedxml>=0.7.1 msgpack>=1.0.8 tof>=25.12.1 pytest>=8.0 +bitshuffle>=0.5.2;os_name == 'posix' scipp --index-url=https://pypi.anaconda.org/scipp-nightly-wheels/simple/ --extra-index-url=https://pypi.org/simple diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 831488a5..46ea7879 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:f1c9ffe61d01dcdeb6285fbbf9def291d16747e2 +# SHA1:4e708d759352a2191073c0fe9616799eae59cf90 # # This file was generated by pip-compile-multi. # To update, run: @@ -10,6 +10,8 @@ annotated-types==0.7.0 # via pydantic +bitshuffle==0.5.2 ; os_name == "posix" + # via -r nightly.in certifi==2026.2.25 # via requests charset-normalizer==3.4.4 @@ -24,6 +26,8 @@ cyclebane==24.10.0 # via sciline cycler==0.12.1 # via matplotlib +cython==3.2.4 + # via bitshuffle dask==2026.1.2 # via -r nightly.in defusedxml==0.8.0rc2 @@ -44,6 +48,7 @@ graphviz==0.21 # via -r nightly.in h5py==3.15.1 # via + # bitshuffle # scippneutron # scippnexus idna==3.11 @@ -73,6 +78,7 @@ networkx==3.6.1 # via cyclebane numpy==2.4.2 # via + # bitshuffle # contourpy # h5py # matplotlib @@ -164,3 +170,6 @@ typing-inspection==0.4.2 # via pydantic urllib3==2.6.3 # via requests + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/packages/essnmx/requirements/test.txt b/packages/essnmx/requirements/test.txt index 6817392e..c900374a 100644 --- a/packages/essnmx/requirements/test.txt +++ b/packages/essnmx/requirements/test.txt @@ -7,3 +7,6 @@ # -r base.txt -r basetest.txt + +# The following packages are considered to be unsafe in a requirements file: +# setuptools From 5e67d9aaa3549ff956b6bdbb915de6ea26e5a7ca Mon Sep 17 00:00:00 2001 From: Johannes Kasimir Date: Mon, 2 Mar 2026 09:24:22 +0100 Subject: [PATCH 399/403] deps: lower bound numpy --- packages/essnmx/pyproject.toml | 1 + packages/essnmx/requirements/base.in | 1 + packages/essnmx/requirements/base.txt | 3 ++- packages/essnmx/requirements/ci.txt | 4 ++-- packages/essnmx/requirements/docs.txt | 2 +- packages/essnmx/requirements/nightly.in | 1 + packages/essnmx/requirements/nightly.txt | 3 ++- packages/essnmx/requirements/static.txt | 4 ++-- 8 files changed, 12 insertions(+), 7 deletions(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index d1428258..3ccc3d33 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -44,6 +44,7 @@ dependencies = [ "defusedxml>=0.7.1", "msgpack>=1.0.8", "tof>=25.12.1", + "numpy>=1.20.0", ] dynamic = ["version"] diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in index f3766be7..867d8811 100644 --- a/packages/essnmx/requirements/base.in +++ b/packages/essnmx/requirements/base.in @@ -16,3 +16,4 @@ gemmi>=0.6.6 defusedxml>=0.7.1 msgpack>=1.0.8 tof>=25.12.1 +numpy>=1.20.0 diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt index 66a35e75..0e68f3ad 100644 --- a/packages/essnmx/requirements/base.txt +++ b/packages/essnmx/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:47709a55cff4a7391979e84117af1db9661bcab6 +# SHA1:fadf4d2d86052d63edbc07724456cfb8fc662399 # # This file was generated by pip-compile-multi. # To update, run: @@ -68,6 +68,7 @@ networkx==3.6.1 # via cyclebane numpy==2.4.2 # via + # -r base.in # contourpy # h5py # matplotlib diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt index bbf20b65..157d0bff 100644 --- a/packages/essnmx/requirements/ci.txt +++ b/packages/essnmx/requirements/ci.txt @@ -15,7 +15,7 @@ colorama==0.4.6 # via tox distlib==0.4.0 # via virtualenv -filelock==3.24.3 +filelock==3.25.0 # via # python-discovery # tox @@ -46,7 +46,7 @@ requests==2.32.5 # via -r ci.in smmap==5.0.2 # via gitdb -tox==4.46.3 +tox==4.47.0 # via -r ci.in urllib3==2.6.3 # via requests diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt index dfbfe06a..eb966623 100644 --- a/packages/essnmx/requirements/docs.txt +++ b/packages/essnmx/requirements/docs.txt @@ -149,7 +149,7 @@ pygments==2.19.2 # nbconvert # pydata-sphinx-theme # sphinx -python-dotenv==1.2.1 +python-dotenv==1.2.2 # via pydantic-settings pythreejs==2.4.2 # via -r docs.in diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in index 1e3e1641..81c6979a 100644 --- a/packages/essnmx/requirements/nightly.in +++ b/packages/essnmx/requirements/nightly.in @@ -10,6 +10,7 @@ gemmi>=0.6.6 defusedxml>=0.7.1 msgpack>=1.0.8 tof>=25.12.1 +numpy>=1.20.0 pytest>=8.0 bitshuffle>=0.5.2;os_name == 'posix' scipp diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt index 46ea7879..92bb5f0d 100644 --- a/packages/essnmx/requirements/nightly.txt +++ b/packages/essnmx/requirements/nightly.txt @@ -1,4 +1,4 @@ -# SHA1:4e708d759352a2191073c0fe9616799eae59cf90 +# SHA1:03339aaa977997d3c0ce22dc2aef4b0e25334271 # # This file was generated by pip-compile-multi. # To update, run: @@ -78,6 +78,7 @@ networkx==3.6.1 # via cyclebane numpy==2.4.2 # via + # -r nightly.in # bitshuffle # contourpy # h5py diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt index 96a67092..e80710d7 100644 --- a/packages/essnmx/requirements/static.txt +++ b/packages/essnmx/requirements/static.txt @@ -9,11 +9,11 @@ cfgv==3.5.0 # via pre-commit distlib==0.4.0 # via virtualenv -filelock==3.24.3 +filelock==3.25.0 # via # python-discovery # virtualenv -identify==2.6.16 +identify==2.6.17 # via pre-commit nodeenv==1.10.0 # via pre-commit From c7b530487bc2ed03fec85d0868659500bd396bfe Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo <17974113+YooSunYoung@users.noreply.github.com> Date: Mon, 2 Mar 2026 10:21:55 +0100 Subject: [PATCH 400/403] Numpy lower pin to 2. --- packages/essnmx/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 3ccc3d33..6fd597d7 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ "defusedxml>=0.7.1", "msgpack>=1.0.8", "tof>=25.12.1", - "numpy>=1.20.0", + "numpy>=2.0.0", ] dynamic = ["version"] From 5445de7b69bb47f07b3c37d6ad3b9d16a70e179d Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 5 Mar 2026 10:46:04 +0100 Subject: [PATCH 401/403] Update root configuration. --- .github/workflows/ci.yml | 5 +++++ README.md | 1 + pixi.toml | 12 +++++++++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d4b97e8..dcf32247 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,11 @@ jobs: - 'packages/essreduce/**' - 'pyproject.toml' - 'pixi.lock' + essnmx: + - 'packages/essnmx/**' + - 'packages/essreduce/**' + - 'pyproject.toml' + - 'pixi.lock' formatting: name: Formatting and static analysis diff --git a/README.md b/README.md index d8936b56..63e3c3be 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Monorepo for ESS neutron scattering data reduction packages, managed with [pixi] |---------|-------------| | [essreduce](packages/essreduce/) | Common data reduction tools (core) | | [essimaging](packages/essimaging/) | Neutron imaging (ODIN, TBL, YMIR) | +| [essnmx](packages/essnmx/) | Data reduction for NMX at the European Spallation Source. | ## Dependency graph diff --git a/pixi.toml b/pixi.toml index ec626813..558e0e53 100644 --- a/pixi.toml +++ b/pixi.toml @@ -27,6 +27,10 @@ essreduce = { path = "packages/essreduce", editable = true, extras = ["test"] } [feature.essimaging.pypi-dependencies] essimaging = { path = "packages/essimaging", editable = true, extras = ["test"] } +# essnmx (depends on essreduce) +[feature.essnmx.pypi-dependencies] +essnmx = { path = "packages/essnmx", editable = true, extras = ["test"] } + # ==================== Lint feature ==================== [feature.lint.pypi-dependencies] @@ -50,23 +54,29 @@ essreduce = { path = "packages/essreduce", editable = true, extras = ["test", "d [feature.docs-essimaging.pypi-dependencies] essimaging = { path = "packages/essimaging", editable = true, extras = ["test", "docs"] } +[feature.docs-essnmx.pypi-dependencies] +essnmx = { path = "packages/essnmx", editable = true, extras = ["test", "docs"] } + # ==================== Environments ==================== [environments] # Default: all packages (for full dev setup) -default = { features = ["essreduce", "essimaging"], solve-group = "default" } +default = { features = ["essreduce", "essimaging", "essnmx"], solve-group = "default" } # Per-package test environments (include workspace dep features) essreduce = { features = ["essreduce"], solve-group = "default" } essimaging = { features = ["essimaging", "essreduce"], solve-group = "default" } +essnmx = { features = ["essnmx", "essreduce"], solve-group = "default" } # Lower-bound test environments (separate resolution) lb-essreduce = { features = ["essreduce"], solve-group = "lower-bound" } lb-essimaging = { features = ["essimaging", "essreduce"], solve-group = "lower-bound" } +lb-essnmx = { features = ["essnmx", "essreduce"], solve-group = "lower-bound" } # Docs environments (package with docs extra + pandoc) docs-essreduce = { features = ["docs-essreduce", "docs"], solve-group = "default" } docs-essimaging = { features = ["docs-essimaging", "docs-essreduce", "docs"], solve-group = "default" } +docs-essnmx = { features = ["docs-essnmx", "docs-essreduce", "docs"], solve-group = "default" } # Lint environment (standalone, no package deps) lint = { features = ["lint"], solve-group = "lint" } From b0cc87d009bafb9bce81323b74894d8b2caa14bd Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:09:22 +0100 Subject: [PATCH 402/403] Remove unecessary dev tools. --- .../ISSUE_TEMPLATE/high-level-requirement.yml | 97 -------- packages/essnmx/.github/dependabot.yml | 13 - packages/essnmx/.github/workflows/ci.yml | 58 ----- packages/essnmx/.github/workflows/docs.yml | 79 ------ .../.github/workflows/nightly_at_main.yml | 34 --- .../workflows/nightly_at_main_lower_bound.yml | 37 --- .../.github/workflows/nightly_at_release.yml | 41 --- .../.github/workflows/python-version-ci | 1 - packages/essnmx/.github/workflows/release.yml | 73 ------ packages/essnmx/.github/workflows/test.yml | 79 ------ .../essnmx/.github/workflows/unpinned.yml | 41 --- .../workflows/weekly_windows_macos.yml | 42 ---- packages/essnmx/.pre-commit-config.yaml | 55 ---- packages/essnmx/requirements/base.in | 19 -- packages/essnmx/requirements/base.txt | 157 ------------ packages/essnmx/requirements/basetest.in | 11 - packages/essnmx/requirements/basetest.txt | 30 --- packages/essnmx/requirements/ci.in | 4 - packages/essnmx/requirements/ci.txt | 54 ---- packages/essnmx/requirements/dev.in | 11 - packages/essnmx/requirements/dev.txt | 121 --------- packages/essnmx/requirements/docs.in | 13 - packages/essnmx/requirements/docs.txt | 235 ------------------ packages/essnmx/requirements/make_base.py | 78 ------ packages/essnmx/requirements/mypy.in | 2 - packages/essnmx/requirements/mypy.txt | 19 -- packages/essnmx/requirements/nightly.in | 22 -- packages/essnmx/requirements/nightly.txt | 176 ------------- packages/essnmx/requirements/static.in | 1 - packages/essnmx/requirements/static.txt | 31 --- packages/essnmx/requirements/test.in | 4 - packages/essnmx/requirements/test.txt | 12 - packages/essnmx/requirements/wheels.in | 1 - packages/essnmx/requirements/wheels.txt | 13 - packages/essnmx/tox.ini | 72 ------ 35 files changed, 1736 deletions(-) delete mode 100644 packages/essnmx/.github/ISSUE_TEMPLATE/high-level-requirement.yml delete mode 100644 packages/essnmx/.github/dependabot.yml delete mode 100644 packages/essnmx/.github/workflows/ci.yml delete mode 100644 packages/essnmx/.github/workflows/docs.yml delete mode 100644 packages/essnmx/.github/workflows/nightly_at_main.yml delete mode 100644 packages/essnmx/.github/workflows/nightly_at_main_lower_bound.yml delete mode 100644 packages/essnmx/.github/workflows/nightly_at_release.yml delete mode 100644 packages/essnmx/.github/workflows/python-version-ci delete mode 100644 packages/essnmx/.github/workflows/release.yml delete mode 100644 packages/essnmx/.github/workflows/test.yml delete mode 100644 packages/essnmx/.github/workflows/unpinned.yml delete mode 100644 packages/essnmx/.github/workflows/weekly_windows_macos.yml delete mode 100644 packages/essnmx/.pre-commit-config.yaml delete mode 100644 packages/essnmx/requirements/base.in delete mode 100644 packages/essnmx/requirements/base.txt delete mode 100644 packages/essnmx/requirements/basetest.in delete mode 100644 packages/essnmx/requirements/basetest.txt delete mode 100644 packages/essnmx/requirements/ci.in delete mode 100644 packages/essnmx/requirements/ci.txt delete mode 100644 packages/essnmx/requirements/dev.in delete mode 100644 packages/essnmx/requirements/dev.txt delete mode 100644 packages/essnmx/requirements/docs.in delete mode 100644 packages/essnmx/requirements/docs.txt delete mode 100644 packages/essnmx/requirements/make_base.py delete mode 100644 packages/essnmx/requirements/mypy.in delete mode 100644 packages/essnmx/requirements/mypy.txt delete mode 100644 packages/essnmx/requirements/nightly.in delete mode 100644 packages/essnmx/requirements/nightly.txt delete mode 100644 packages/essnmx/requirements/static.in delete mode 100644 packages/essnmx/requirements/static.txt delete mode 100644 packages/essnmx/requirements/test.in delete mode 100644 packages/essnmx/requirements/test.txt delete mode 100644 packages/essnmx/requirements/wheels.in delete mode 100644 packages/essnmx/requirements/wheels.txt delete mode 100644 packages/essnmx/tox.ini diff --git a/packages/essnmx/.github/ISSUE_TEMPLATE/high-level-requirement.yml b/packages/essnmx/.github/ISSUE_TEMPLATE/high-level-requirement.yml deleted file mode 100644 index 4d87603b..00000000 --- a/packages/essnmx/.github/ISSUE_TEMPLATE/high-level-requirement.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: High-level requirement -description: Describe a high-level requirement -title: "[Requirement] " -labels: ["requirement"] -projects: [] -body: - - type: markdown - attributes: - value: | - Thanks for taking the time to provide as many details as possible for this requirement! - - type: input - id: summary - attributes: - label: Executive summary - description: Provide a short summary of the requirement - placeholder: "Example: We need to correct for X when processing Y." - validations: - required: true - - type: textarea - id: context - attributes: - label: Context and background knowledge - description: | - - What is the context of this requirement? - - What background knowledge is required to understand it? - - Does this depend on previous tasks? Provide links! - - Is there follow-up work? - placeholder: "Example: See summary on Wikipedia, or the following paper." - validations: - required: true - - type: textarea - id: inputs - attributes: - label: Inputs - description: | - Describe in detail all the input data and data properties that are known. - This is not about test data (see below), but about general properties of data that will be used in practice. - placeholder: "Example: A single 1-D spectrum with a known wavelength range." - validations: - required: true - - type: textarea - id: methodology - attributes: - label: Methodology - description: | - Describe, e.g., the computation to be performed. - When linking to references, please refer to the specific section, page, or equation. - placeholder: "Remember you can write equations such as $n\\lambda = 2d\\sin(\\theta)$ using LaTeX syntax, as well as other Markdown formatting." - validations: - required: true - - type: textarea - id: outputs - attributes: - label: Outputs - description: | - Describe in detail all the output data and data properties. - This is not about test data (see below), but about general properties of data that will be used in practice. - placeholder: "Example: The position of the peak in the spectrum." - validations: - required: true - - type: dropdown - id: interfaces - attributes: - label: Which interfaces are required? - multiple: true - options: - - Integrated into reduction workflow - - Python module / function - - Python script - - Jupyter notebook - - Other (please describe in comments) - default: 0 - validations: - required: true - - type: textarea - id: testcases - attributes: - label: Test cases - description: How can we test this requirement? Links to tests data and reference data, or other suggestions. - validations: - required: true - - type: textarea - id: existingimplementations - attributes: - label: Existing implementations - description: Are there any existing implementations or proof-of-concept implementations that we can imitate? This field is specifically for linking to source code. - placeholder: "Example: See this repository ... This script implements the procedure: https://file-storage.server.eu/script.code." - validations: - required: false - - type: textarea - id: comments - attributes: - label: Comments - description: Do you have other comments that do not fall in the above categories? - placeholder: "Example: Depends on issues #1234, blocked by #666." - validations: - required: false diff --git a/packages/essnmx/.github/dependabot.yml b/packages/essnmx/.github/dependabot.yml deleted file mode 100644 index c8076bb1..00000000 --- a/packages/essnmx/.github/dependabot.yml +++ /dev/null @@ -1,13 +0,0 @@ -version: 2 -updates: - # Note: We are not listing package-ecosystem: "github-actions". This causes - # noise in all template instances. Instead dependabot.yml in scipp/copier_template - # triggers updates of github-actions in the *template*. We then use `copier update` - # in template instances. - - package-ecosystem: "pip" - directory: "/requirements" - schedule: - interval: "daily" - allow: - - dependency-name: "scipp" - dependency-type: "direct" diff --git a/packages/essnmx/.github/workflows/ci.yml b/packages/essnmx/.github/workflows/ci.yml deleted file mode 100644 index b273e684..00000000 --- a/packages/essnmx/.github/workflows/ci.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: CI - -on: - push: - branches: - - main - - release - pull_request: - -jobs: - formatting: - name: Formatting and static analysis - runs-on: 'ubuntu-24.04' - outputs: - min_python: ${{ steps.vars.outputs.min_python }} - min_tox_env: ${{ steps.vars.outputs.min_tox_env }} - steps: - - uses: actions/checkout@v4 - - name: Get Python version for other CI jobs - id: vars - run: | - echo "min_python=$(< .github/workflows/python-version-ci)" >> "$GITHUB_OUTPUT" - echo "min_tox_env=py$(sed 's/\.//g' < .github/workflows/python-version-ci)" >> "$GITHUB_OUTPUT" - - uses: actions/setup-python@v5 - with: - python-version-file: '.github/workflows/python-version-ci' - - uses: pre-commit/action@v3.0.1 - with: - extra_args: --all-files - - uses: pre-commit-ci/lite-action@v1.1.0 - if: always() - with: - msg: Apply automatic formatting - - tests: - name: Tests - needs: formatting - strategy: - matrix: - os: ['ubuntu-24.04'] - python: - - version: '${{needs.formatting.outputs.min_python}}' - tox-env: '${{needs.formatting.outputs.min_tox_env}}' - uses: ./.github/workflows/test.yml - with: - os-variant: ${{ matrix.os }} - python-version: ${{ matrix.python.version }} - tox-env: ${{ matrix.python.tox-env }} - secrets: inherit - - docs: - needs: tests - uses: ./.github/workflows/docs.yml - with: - publish: false - linkcheck: ${{ github.ref == 'refs/heads/main' }} - branch: ${{ github.head_ref == '' && github.ref_name || github.head_ref }} - secrets: inherit diff --git a/packages/essnmx/.github/workflows/docs.yml b/packages/essnmx/.github/workflows/docs.yml deleted file mode 100644 index 3a303f84..00000000 --- a/packages/essnmx/.github/workflows/docs.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: Docs - -on: - workflow_dispatch: - inputs: - publish: - default: false - type: boolean - version: - default: '' - required: false - type: string - branch: - description: 'Branch/tag with documentation source. If not set, the current branch will be used.' - default: '' - required: false - type: string - workflow_call: - inputs: - publish: - default: false - type: boolean - version: - default: '' - required: false - type: string - branch: - description: 'Branch/tag with documentation source. If not set, the current branch will be used.' - default: '' - required: false - type: string - linkcheck: - description: 'Run the link checker. If not set the link checker will not be run.' - default: false - required: false - type: boolean - -env: - VERSION: ${{ inputs.version }} - -jobs: - docs: - name: Build documentation - runs-on: 'ubuntu-24.04' - env: - ESS_PROTECTED_FILESTORE_USERNAME: ${{ secrets.ESS_PROTECTED_FILESTORE_USERNAME }} - ESS_PROTECTED_FILESTORE_PASSWORD: ${{ secrets.ESS_PROTECTED_FILESTORE_PASSWORD }} - - steps: - - run: sudo apt install --yes graphviz pandoc - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch == '' && github.ref_name || inputs.branch }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - fetch-depth: 0 # history required so cmake can determine version - - uses: actions/setup-python@v5 - with: - python-version-file: '.github/workflows/python-version-ci' - - run: python -m pip install --upgrade pip - - run: python -m pip install -r requirements/ci.txt - - run: tox -e releasedocs -- "${VERSION}" - if: ${{ inputs.version != '' }} - - run: tox -e docs - if: ${{ inputs.version == '' }} - - run: tox -e linkcheck - if: ${{ inputs.linkcheck }} - - uses: actions/upload-artifact@v4 - id: artifact-upload-step - with: - name: docs_html - path: html/ - - run: echo "::notice::https://remote-unzip.deno.dev/${{ github.repository }}/artifacts/${{ steps.artifact-upload-step.outputs.artifact-id }}" - - - uses: JamesIves/github-pages-deploy-action@v4.8.0 - if: ${{ inputs.publish }} - with: - branch: gh-pages - folder: html - single-commit: true diff --git a/packages/essnmx/.github/workflows/nightly_at_main.yml b/packages/essnmx/.github/workflows/nightly_at_main.yml deleted file mode 100644 index 20e9ee4a..00000000 --- a/packages/essnmx/.github/workflows/nightly_at_main.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Nightly test at main branch - -on: - workflow_dispatch: - schedule: - - cron: '30 1 * * 1-5' - -jobs: - setup: - name: Setup variables - runs-on: 'ubuntu-24.04' - outputs: - min_python: ${{ steps.vars.outputs.min_python }} - steps: - - uses: actions/checkout@v4 - - name: Get Python version for other CI jobs - id: vars - run: echo "min_python=$(< .github/workflows/python-version-ci)" >> "$GITHUB_OUTPUT" - - tests: - name: Tests - needs: setup - strategy: - matrix: - os: ['ubuntu-24.04'] - python: - - version: '${{needs.setup.outputs.min_python}}' - tox-env: 'nightly' - uses: ./.github/workflows/test.yml - with: - os-variant: ${{ matrix.os }} - python-version: ${{ matrix.python.version }} - tox-env: ${{ matrix.python.tox-env }} - secrets: inherit diff --git a/packages/essnmx/.github/workflows/nightly_at_main_lower_bound.yml b/packages/essnmx/.github/workflows/nightly_at_main_lower_bound.yml deleted file mode 100644 index c086e3cc..00000000 --- a/packages/essnmx/.github/workflows/nightly_at_main_lower_bound.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Nightly test using lower bound dependencies - -on: - workflow_dispatch: - schedule: - - cron: '30 1 * * 1-5' - -jobs: - setup: - name: Setup variables - runs-on: 'ubuntu-24.04' - outputs: - min_python: ${{ steps.vars.outputs.min_python }} - steps: - - uses: actions/checkout@v4 - - name: Get Python version for other CI jobs - id: vars - run: echo "min_python=$(< .github/workflows/python-version-ci)" >> "$GITHUB_OUTPUT" - - tests: - name: Tests at lower bound - needs: setup - strategy: - matrix: - os: ['ubuntu-24.04'] - python: - - version: '${{needs.setup.outputs.min_python}}' - runs-on: ${{ matrix.os }} - env: - ESS_PROTECTED_FILESTORE_USERNAME: ${{ secrets.ESS_PROTECTED_FILESTORE_USERNAME }} - ESS_PROTECTED_FILESTORE_PASSWORD: ${{ secrets.ESS_PROTECTED_FILESTORE_PASSWORD }} - steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v7 - with: - python-version: ${{ matrix.python.version }} - - run: uv run --extra=test --resolution=lowest-direct pytest diff --git a/packages/essnmx/.github/workflows/nightly_at_release.yml b/packages/essnmx/.github/workflows/nightly_at_release.yml deleted file mode 100644 index 14b75213..00000000 --- a/packages/essnmx/.github/workflows/nightly_at_release.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Nightly tests at latest release - -on: - workflow_dispatch: - schedule: - - cron: '0 1 * * 1-5' - -jobs: - setup: - name: Setup variables - runs-on: 'ubuntu-24.04' - outputs: - min_python: ${{ steps.vars.outputs.min_python }} - release_tag: ${{ steps.release.outputs.release_tag }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # history required so we can determine latest release tag - - name: Get last release tag from git - id: release - run: echo "release_tag=$(git describe --tags --abbrev=0 --match '[0-9]*.[0-9]*.[0-9]*')" >> "$GITHUB_OUTPUT" - - name: Get Python version for other CI jobs - id: vars - run: echo "min_python=$(cat .github/workflows/python-version-ci)" >> "$GITHUB_OUTPUT" - - tests: - name: Tests - needs: setup - strategy: - matrix: - os: ['ubuntu-24.04'] - python: - - version: '${{needs.setup.outputs.min_python}}' - tox-env: 'nightly' - uses: ./.github/workflows/test.yml - with: - os-variant: ${{ matrix.os }} - python-version: ${{ matrix.python.version }} - tox-env: ${{ matrix.python.tox-env }} - checkout_ref: ${{ needs.setup.outputs.release_tag }} - secrets: inherit diff --git a/packages/essnmx/.github/workflows/python-version-ci b/packages/essnmx/.github/workflows/python-version-ci deleted file mode 100644 index 2c073331..00000000 --- a/packages/essnmx/.github/workflows/python-version-ci +++ /dev/null @@ -1 +0,0 @@ -3.11 diff --git a/packages/essnmx/.github/workflows/release.yml b/packages/essnmx/.github/workflows/release.yml deleted file mode 100644 index 66b8a942..00000000 --- a/packages/essnmx/.github/workflows/release.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: Release - -on: - release: - types: [published] - workflow_dispatch: - -defaults: - run: - shell: bash -l {0} # required for conda env - -jobs: - build_wheels: - name: Wheels - runs-on: 'ubuntu-24.04' - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # history required so setuptools_scm can determine version - - - uses: actions/setup-python@v5 - with: - python-version-file: '.github/workflows/python-version-ci' - - - run: python -m pip install --upgrade pip - - run: python -m pip install -r requirements/wheels.txt - - - name: Build wheels - run: python -m build - - - name: Upload wheels - uses: actions/upload-artifact@v4 - with: - name: dist - path: dist - - upload_pypi: - name: Deploy PyPI - needs: [build_wheels] - runs-on: 'ubuntu-24.04' - environment: release - permissions: - id-token: write - if: github.event_name == 'release' && github.event.action == 'published' - steps: - - uses: actions/download-artifact@v4 - - uses: pypa/gh-action-pypi-publish@v1.12.4 - - docs: - needs: [upload_pypi] - uses: ./.github/workflows/docs.yml - with: - publish: ${{ github.event_name == 'release' && github.event.action == 'published' }} - secrets: inherit - - assets: - name: Upload docs - needs: docs - runs-on: 'ubuntu-24.04' - permissions: - contents: write # This is needed so that the action can upload the asset - steps: - - uses: actions/download-artifact@v4 - - name: Zip documentation - run: | - mv docs_html documentation-${{ github.ref_name }} - zip -r documentation-${{ github.ref_name }}.zip documentation-${{ github.ref_name }} - - name: Upload release assets - uses: svenstaro/upload-release-action@v2 - with: - file: ./documentation-${{ github.ref_name }}.zip - overwrite: false diff --git a/packages/essnmx/.github/workflows/test.yml b/packages/essnmx/.github/workflows/test.yml deleted file mode 100644 index ca40b253..00000000 --- a/packages/essnmx/.github/workflows/test.yml +++ /dev/null @@ -1,79 +0,0 @@ -name: Test - -on: - workflow_dispatch: - inputs: - os-variant: - default: 'ubuntu-24.04' - type: string - python-version: - type: string - tox-env: - default: 'test' - type: string - pip-recipe: - default: 'requirements/ci.txt' - type: string - coverage-report: - default: false - type: boolean - checkout_ref: - default: '' - type: string - workflow_call: - inputs: - os-variant: - default: 'ubuntu-24.04' - type: string - python-version: - type: string - tox-env: - default: 'test' - type: string - pip-recipe: - default: 'requirements/ci.txt' - type: string - coverage-report: - default: false - type: boolean - checkout_ref: - default: '' - type: string - -jobs: - package-test: - runs-on: ${{ inputs.os-variant }} - - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.checkout_ref }} - - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python-version }} - - run: python -m pip install --upgrade pip - - run: python -m pip install . - - run: python tests/package_test.py - name: Run package tests - - test: - runs-on: ${{ inputs.os-variant }} - env: - ESS_PROTECTED_FILESTORE_USERNAME: ${{ secrets.ESS_PROTECTED_FILESTORE_USERNAME }} - ESS_PROTECTED_FILESTORE_PASSWORD: ${{ secrets.ESS_PROTECTED_FILESTORE_PASSWORD }} - - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.checkout_ref }} - - uses: actions/setup-python@v5 - with: - python-version: ${{ inputs.python-version }} - - run: python -m pip install --upgrade pip - - run: python -m pip install -r ${{ inputs.pip-recipe }} - - run: tox -e ${{ inputs.tox-env }} - - uses: actions/upload-artifact@v4 - if: ${{ inputs.coverage-report }} - with: - name: CoverageReport - path: coverage_html/ diff --git a/packages/essnmx/.github/workflows/unpinned.yml b/packages/essnmx/.github/workflows/unpinned.yml deleted file mode 100644 index ff03faa1..00000000 --- a/packages/essnmx/.github/workflows/unpinned.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Unpinned tests at latest release - -on: - workflow_dispatch: - schedule: - - cron: '0 2 * * 1' - -jobs: - setup: - name: Setup variables - runs-on: 'ubuntu-24.04' - outputs: - min_python: ${{ steps.vars.outputs.min_python }} - release_tag: ${{ steps.release.outputs.release_tag }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # history required so we can determine latest release tag - - name: Get last release tag from git - id: release - run: echo "release_tag=$(git describe --tags --abbrev=0 --match '[0-9]*.[0-9]*.[0-9]*')" >> "$GITHUB_OUTPUT" - - name: Get Python version for other CI jobs - id: vars - run: echo "min_python=$(cat .github/workflows/python-version-ci)" >> "$GITHUB_OUTPUT" - - tests: - name: Tests - needs: setup - strategy: - matrix: - os: ['ubuntu-24.04'] - python: - - version: '${{needs.setup.outputs.min_python}}' - tox-env: 'unpinned' - uses: ./.github/workflows/test.yml - with: - os-variant: ${{ matrix.os }} - python-version: ${{ matrix.python.version }} - tox-env: ${{ matrix.python.tox-env }} - checkout_ref: ${{ needs.setup.outputs.release_tag }} - secrets: inherit diff --git a/packages/essnmx/.github/workflows/weekly_windows_macos.yml b/packages/essnmx/.github/workflows/weekly_windows_macos.yml deleted file mode 100644 index 1544d7f9..00000000 --- a/packages/essnmx/.github/workflows/weekly_windows_macos.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Windows and MacOS weekly tests - -on: - workflow_dispatch: - schedule: - - cron: '0 2 * * 1' - -jobs: - pytox: - name: Python and Tox env - runs-on: 'ubuntu-24.04' - outputs: - min_python: ${{ steps.vars.outputs.min_python }} - min_tox_env: ${{ steps.vars.outputs.min_tox_env }} - steps: - - uses: actions/checkout@v4 - - name: Get Python version for other CI jobs - id: vars - run: | - echo "min_python=$(cat .github/workflows/python-version-ci)" >> $GITHUB_OUTPUT - echo "min_tox_env=py$(cat .github/workflows/python-version-ci | sed 's/\.//g')" >> $GITHUB_OUTPUT - tests: - name: Tests - needs: pytox - strategy: - matrix: - os: ['macos-latest', 'windows-latest'] - python: - - version: '${{needs.pytox.outputs.min_python}}' - tox-env: '${{needs.pytox.outputs.min_tox_env}}' - uses: ./.github/workflows/test.yml - with: - os-variant: ${{ matrix.os }} - python-version: ${{ matrix.python.version }} - tox-env: ${{ matrix.python.tox-env }} - - docs: - needs: tests - uses: ./.github/workflows/docs.yml - with: - publish: false - branch: ${{ github.head_ref == '' && github.ref_name || github.head_ref }} diff --git a/packages/essnmx/.pre-commit-config.yaml b/packages/essnmx/.pre-commit-config.yaml deleted file mode 100644 index 683b94d7..00000000 --- a/packages/essnmx/.pre-commit-config.yaml +++ /dev/null @@ -1,55 +0,0 @@ -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v6.0.0 - hooks: - - id: check-added-large-files - - id: check-case-conflict - - id: check-illegal-windows-names - - id: check-json - exclude: asv.conf.json - - id: check-merge-conflict - - id: check-toml - - id: check-yaml - - id: detect-private-key - - id: trailing-whitespace - args: [ --markdown-linebreak-ext=md ] - exclude: '\.svg' - - repo: https://github.com/kynan/nbstripout - rev: 0.8.2 - hooks: - - id: nbstripout - types: [ "jupyter" ] - args: [ "--drop-empty-cells", - "--extra-keys 'metadata.language_info.version cell.metadata.jp-MarkdownHeadingCollapsed cell.metadata.pycharm'" ] - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.6 - hooks: - - id: ruff - args: [ --fix ] - types_or: [ python, pyi, jupyter ] - - id: ruff-format - types_or: [ python, pyi ] - - repo: https://github.com/codespell-project/codespell - rev: v2.4.1 - hooks: - - id: codespell - additional_dependencies: - - tomli - exclude_types: - - svg - - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.10.0 - hooks: - - id: python-no-eval - - id: python-no-log-warn - - id: python-use-type-annotations - - id: rst-backticks - - id: rst-directive-colons - - id: rst-inline-touching-normal - - id: text-unicode-replacement-char - - repo: https://github.com/rhysd/actionlint - rev: v1.7.9 - hooks: - - id: actionlint - # Disable because of false-positive SC2046 - args: ["-shellcheck="] diff --git a/packages/essnmx/requirements/base.in b/packages/essnmx/requirements/base.in deleted file mode 100644 index 867d8811..00000000 --- a/packages/essnmx/requirements/base.in +++ /dev/null @@ -1,19 +0,0 @@ -# Anything above "--- END OF CUSTOM SECTION ---" -# will not be touched by ``make_base.py`` -# --- END OF CUSTOM SECTION --- -# The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! -dask>=2022.1.0 -essreduce>=26.2.1 -graphviz -plopp>=24.7.0 -sciline>=24.06.0 -scipp>=25.3.0 -scippnexus>=23.12.0 -scippneutron>=26.02.0 -pooch>=1.5 -pandas>=2.1.2 -gemmi>=0.6.6 -defusedxml>=0.7.1 -msgpack>=1.0.8 -tof>=25.12.1 -numpy>=1.20.0 diff --git a/packages/essnmx/requirements/base.txt b/packages/essnmx/requirements/base.txt deleted file mode 100644 index 0e68f3ad..00000000 --- a/packages/essnmx/requirements/base.txt +++ /dev/null @@ -1,157 +0,0 @@ -# SHA1:fadf4d2d86052d63edbc07724456cfb8fc662399 -# -# This file was generated by pip-compile-multi. -# To update, run: -# -# requirements upgrade -# -annotated-types==0.7.0 - # via pydantic -certifi==2026.2.25 - # via requests -charset-normalizer==3.4.4 - # via requests -click==8.3.1 - # via dask -cloudpickle==3.1.2 - # via dask -contourpy==1.3.3 - # via matplotlib -cyclebane==24.10.0 - # via sciline -cycler==0.12.1 - # via matplotlib -dask==2026.1.2 - # via -r base.in -defusedxml==0.7.1 - # via -r base.in -dnspython==2.8.0 - # via email-validator -email-validator==2.3.0 - # via scippneutron -essreduce==26.2.2 - # via -r base.in -fonttools==4.61.1 - # via matplotlib -fsspec==2026.2.0 - # via dask -gemmi==0.7.4 - # via -r base.in -graphviz==0.21 - # via -r base.in -h5py==3.15.1 - # via - # scippneutron - # scippnexus -idna==3.11 - # via - # email-validator - # requests -kiwisolver==1.4.9 - # via matplotlib -lazy-loader==0.4 - # via - # plopp - # scippneutron - # tof -locket==1.0.0 - # via partd -matplotlib==3.10.8 - # via - # mpltoolbox - # plopp -mpltoolbox==26.2.0 - # via scippneutron -msgpack==1.1.2 - # via -r base.in -networkx==3.6.1 - # via cyclebane -numpy==2.4.2 - # via - # -r base.in - # contourpy - # h5py - # matplotlib - # pandas - # scipp - # scippneutron - # scipy -packaging==26.0 - # via - # dask - # lazy-loader - # matplotlib - # pooch -pandas==3.0.1 - # via -r base.in -partd==1.4.2 - # via dask -pillow==12.1.1 - # via matplotlib -platformdirs==4.9.2 - # via pooch -plopp==26.2.1 - # via - # -r base.in - # scippneutron - # tof -pooch==1.9.0 - # via - # -r base.in - # tof -pydantic==2.12.5 - # via scippneutron -pydantic-core==2.41.5 - # via pydantic -pyparsing==3.3.2 - # via matplotlib -python-dateutil==2.9.0.post0 - # via - # matplotlib - # pandas - # scippneutron -pyyaml==6.0.3 - # via dask -requests==2.32.5 - # via pooch -sciline==25.11.1 - # via - # -r base.in - # essreduce -scipp==26.2.0 - # via - # -r base.in - # essreduce - # scippneutron - # scippnexus - # tof -scippneutron==26.2.0 - # via - # -r base.in - # essreduce -scippnexus==26.1.1 - # via - # -r base.in - # essreduce - # scippneutron -scipy==1.17.1 - # via - # scippneutron - # scippnexus -six==1.17.0 - # via python-dateutil -tof==26.1.0 - # via -r base.in -toolz==1.1.0 - # via - # dask - # partd -typing-extensions==4.15.0 - # via - # pydantic - # pydantic-core - # typing-inspection -typing-inspection==0.4.2 - # via pydantic -urllib3==2.6.3 - # via requests diff --git a/packages/essnmx/requirements/basetest.in b/packages/essnmx/requirements/basetest.in deleted file mode 100644 index e3fe12d6..00000000 --- a/packages/essnmx/requirements/basetest.in +++ /dev/null @@ -1,11 +0,0 @@ -# Dependencies that are only used by tests. -# Do not make an environment from this file, use test.txt instead! -# Add more dependencies in the ``test`` list -# under ``[project.optional-dependencies]`` section, in ``pyproject.toml`` - -# Anything above "--- END OF CUSTOM SECTION ---" -# will not be touched by ``make_base.py`` -# --- END OF CUSTOM SECTION --- -# The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! -pytest>=8.0 -bitshuffle>=0.5.2;os_name == 'posix' diff --git a/packages/essnmx/requirements/basetest.txt b/packages/essnmx/requirements/basetest.txt deleted file mode 100644 index d336a671..00000000 --- a/packages/essnmx/requirements/basetest.txt +++ /dev/null @@ -1,30 +0,0 @@ -# SHA1:cf17c7dcdab3a1969e98e1b06b3aafc346f3d586 -# -# This file was generated by pip-compile-multi. -# To update, run: -# -# requirements upgrade -# -bitshuffle==0.5.2 ; os_name == "posix" - # via -r basetest.in -cython==3.2.4 - # via bitshuffle -h5py==3.15.1 - # via bitshuffle -iniconfig==2.3.0 - # via pytest -numpy==2.4.2 - # via - # bitshuffle - # h5py -packaging==26.0 - # via pytest -pluggy==1.6.0 - # via pytest -pygments==2.19.2 - # via pytest -pytest==9.0.2 - # via -r basetest.in - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/packages/essnmx/requirements/ci.in b/packages/essnmx/requirements/ci.in deleted file mode 100644 index e5c3075a..00000000 --- a/packages/essnmx/requirements/ci.in +++ /dev/null @@ -1,4 +0,0 @@ -gitpython -packaging -requests -tox diff --git a/packages/essnmx/requirements/ci.txt b/packages/essnmx/requirements/ci.txt deleted file mode 100644 index 157d0bff..00000000 --- a/packages/essnmx/requirements/ci.txt +++ /dev/null @@ -1,54 +0,0 @@ -# SHA1:6344d52635ea11dca331a3bc6eb1833c4c64d585 -# -# This file was generated by pip-compile-multi. -# To update, run: -# -# requirements upgrade -# -cachetools==7.0.1 - # via tox -certifi==2026.2.25 - # via requests -charset-normalizer==3.4.4 - # via requests -colorama==0.4.6 - # via tox -distlib==0.4.0 - # via virtualenv -filelock==3.25.0 - # via - # python-discovery - # tox - # virtualenv -gitdb==4.0.12 - # via gitpython -gitpython==3.1.46 - # via -r ci.in -idna==3.11 - # via requests -packaging==26.0 - # via - # -r ci.in - # pyproject-api - # tox -platformdirs==4.9.2 - # via - # python-discovery - # tox - # virtualenv -pluggy==1.6.0 - # via tox -pyproject-api==1.10.0 - # via tox -python-discovery==1.1.0 - # via virtualenv -requests==2.32.5 - # via -r ci.in -smmap==5.0.2 - # via gitdb -tox==4.47.0 - # via -r ci.in -urllib3==2.6.3 - # via requests -virtualenv==21.1.0 - # via tox diff --git a/packages/essnmx/requirements/dev.in b/packages/essnmx/requirements/dev.in deleted file mode 100644 index 53ddf47e..00000000 --- a/packages/essnmx/requirements/dev.in +++ /dev/null @@ -1,11 +0,0 @@ --r base.in --r ci.in --r docs.in --r mypy.in --r static.in --r test.in --r wheels.in -copier -jupyterlab -pip-compile-multi -pre-commit diff --git a/packages/essnmx/requirements/dev.txt b/packages/essnmx/requirements/dev.txt deleted file mode 100644 index 7f1435b3..00000000 --- a/packages/essnmx/requirements/dev.txt +++ /dev/null @@ -1,121 +0,0 @@ -# SHA1:efd19a3a98c69fc3d6d6233ed855de7e4a208f74 -# -# This file was generated by pip-compile-multi. -# To update, run: -# -# requirements upgrade -# --r base.txt --r ci.txt --r docs.txt --r mypy.txt --r static.txt --r test.txt --r wheels.txt -anyio==4.12.1 - # via - # httpx - # jupyter-server -argon2-cffi==25.1.0 - # via jupyter-server -argon2-cffi-bindings==25.1.0 - # via argon2-cffi -arrow==1.4.0 - # via isoduration -async-lru==2.2.0 - # via jupyterlab -cffi==2.0.0 - # via argon2-cffi-bindings -copier==9.12.0 - # via -r dev.in -dunamai==1.26.0 - # via copier -fqdn==1.5.1 - # via jsonschema -funcy==2.0 - # via copier -h11==0.16.0 - # via httpcore -httpcore==1.0.9 - # via httpx -httpx==0.28.1 - # via jupyterlab -isoduration==20.11.0 - # via jsonschema -jinja2-ansible-filters==1.3.2 - # via copier -json5==0.13.0 - # via jupyterlab-server -jsonpointer==3.0.0 - # via jsonschema -jsonschema[format-nongpl]==4.26.0 - # via - # jupyter-events - # jupyterlab-server - # nbformat -jupyter-events==0.12.0 - # via jupyter-server -jupyter-lsp==2.3.0 - # via jupyterlab -jupyter-server==2.17.0 - # via - # jupyter-lsp - # jupyterlab - # jupyterlab-server - # notebook-shim -jupyter-server-terminals==0.5.4 - # via jupyter-server -jupyterlab==4.5.5 - # via -r dev.in -jupyterlab-server==2.28.0 - # via jupyterlab -lark==1.3.1 - # via rfc3987-syntax -notebook-shim==0.2.4 - # via jupyterlab -pip-compile-multi==3.2.2 - # via -r dev.in -pip-tools==7.5.3 - # via pip-compile-multi -plumbum==1.10.0 - # via copier -prometheus-client==0.24.1 - # via jupyter-server -pycparser==3.0 - # via cffi -python-json-logger==4.0.0 - # via jupyter-events -questionary==2.1.1 - # via copier -rfc3339-validator==0.1.4 - # via - # jsonschema - # jupyter-events -rfc3986-validator==0.1.1 - # via - # jsonschema - # jupyter-events -rfc3987-syntax==1.1.0 - # via jsonschema -send2trash==2.1.0 - # via jupyter-server -terminado==0.18.1 - # via - # jupyter-server - # jupyter-server-terminals -toposort==1.10 - # via pip-compile-multi -tzdata==2025.3 - # via arrow -uri-template==1.3.0 - # via jsonschema -webcolors==25.10.0 - # via jsonschema -websocket-client==1.9.0 - # via jupyter-server -wheel==0.46.3 - # via pip-tools - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/packages/essnmx/requirements/docs.in b/packages/essnmx/requirements/docs.in deleted file mode 100644 index 9873d260..00000000 --- a/packages/essnmx/requirements/docs.in +++ /dev/null @@ -1,13 +0,0 @@ --r base.in -autodoc-pydantic -ipykernel -ipython!=8.7.0 # Breaks syntax highlighting in Jupyter code cells. -myst-parser -nbsphinx -pydata-sphinx-theme>=0.14 -sphinx<9.0.0 -sphinx-autodoc-typehints -sphinx-copybutton -sphinx-design -pythreejs # For instrument view. -scippneutron diff --git a/packages/essnmx/requirements/docs.txt b/packages/essnmx/requirements/docs.txt deleted file mode 100644 index eb966623..00000000 --- a/packages/essnmx/requirements/docs.txt +++ /dev/null @@ -1,235 +0,0 @@ -# SHA1:e784872c1be8b2208cae1e80c661a0ec223964fc -# -# This file was generated by pip-compile-multi. -# To update, run: -# -# requirements upgrade -# --r base.txt -accessible-pygments==0.0.5 - # via pydata-sphinx-theme -alabaster==1.0.0 - # via sphinx -asttokens==3.0.1 - # via stack-data -attrs==25.4.0 - # via - # jsonschema - # referencing -autodoc-pydantic==2.2.0 - # via -r docs.in -babel==2.18.0 - # via - # pydata-sphinx-theme - # sphinx -beautifulsoup4==4.14.3 - # via - # nbconvert - # pydata-sphinx-theme -bleach[css]==6.3.0 - # via nbconvert -comm==0.2.3 - # via - # ipykernel - # ipywidgets -debugpy==1.8.20 - # via ipykernel -decorator==5.2.1 - # via ipython -docutils==0.21.2 - # via - # myst-parser - # nbsphinx - # pydata-sphinx-theme - # sphinx -executing==2.2.1 - # via stack-data -fastjsonschema==2.21.2 - # via nbformat -imagesize==1.4.1 - # via sphinx -ipydatawidgets==4.3.5 - # via pythreejs -ipykernel==7.2.0 - # via -r docs.in -ipython==9.10.0 - # via - # -r docs.in - # ipykernel - # ipywidgets -ipython-pygments-lexers==1.1.1 - # via ipython -ipywidgets==8.1.8 - # via - # ipydatawidgets - # pythreejs -jedi==0.19.2 - # via ipython -jinja2==3.1.6 - # via - # myst-parser - # nbconvert - # nbsphinx - # sphinx -jsonschema==4.26.0 - # via nbformat -jsonschema-specifications==2025.9.1 - # via jsonschema -jupyter-client==8.8.0 - # via - # ipykernel - # nbclient -jupyter-core==5.9.1 - # via - # ipykernel - # jupyter-client - # nbclient - # nbconvert - # nbformat -jupyterlab-pygments==0.3.0 - # via nbconvert -jupyterlab-widgets==3.0.16 - # via ipywidgets -markdown-it-py==4.0.0 - # via - # mdit-py-plugins - # myst-parser -markupsafe==3.0.3 - # via - # jinja2 - # nbconvert -matplotlib-inline==0.2.1 - # via - # ipykernel - # ipython -mdit-py-plugins==0.5.0 - # via myst-parser -mdurl==0.1.2 - # via markdown-it-py -mistune==3.2.0 - # via nbconvert -myst-parser==5.0.0 - # via -r docs.in -nbclient==0.10.4 - # via nbconvert -nbconvert==7.17.0 - # via nbsphinx -nbformat==5.10.4 - # via - # nbclient - # nbconvert - # nbsphinx -nbsphinx==0.9.8 - # via -r docs.in -nest-asyncio==1.6.0 - # via ipykernel -pandocfilters==1.5.1 - # via nbconvert -parso==0.8.6 - # via jedi -pexpect==4.9.0 - # via ipython -prompt-toolkit==3.0.52 - # via ipython -psutil==7.2.2 - # via ipykernel -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.3 - # via stack-data -pydantic-settings==2.13.1 - # via autodoc-pydantic -pydata-sphinx-theme==0.16.1 - # via -r docs.in -pygments==2.19.2 - # via - # accessible-pygments - # ipython - # ipython-pygments-lexers - # nbconvert - # pydata-sphinx-theme - # sphinx -python-dotenv==1.2.2 - # via pydantic-settings -pythreejs==2.4.2 - # via -r docs.in -pyzmq==27.1.0 - # via - # ipykernel - # jupyter-client -referencing==0.37.0 - # via - # jsonschema - # jsonschema-specifications -roman-numerals==4.1.0 - # via roman-numerals-py -roman-numerals-py==4.1.0 - # via sphinx -rpds-py==0.30.0 - # via - # jsonschema - # referencing -snowballstemmer==3.0.1 - # via sphinx -soupsieve==2.8.3 - # via beautifulsoup4 -sphinx==8.2.3 - # via - # -r docs.in - # autodoc-pydantic - # myst-parser - # nbsphinx - # pydata-sphinx-theme - # sphinx-autodoc-typehints - # sphinx-copybutton - # sphinx-design -sphinx-autodoc-typehints==3.5.2 - # via -r docs.in -sphinx-copybutton==0.5.2 - # via -r docs.in -sphinx-design==0.7.0 - # via -r docs.in -sphinxcontrib-applehelp==2.0.0 - # via sphinx -sphinxcontrib-devhelp==2.0.0 - # via sphinx -sphinxcontrib-htmlhelp==2.1.0 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==2.0.0 - # via sphinx -sphinxcontrib-serializinghtml==2.0.0 - # via sphinx -stack-data==0.6.3 - # via ipython -tinycss2==1.4.0 - # via bleach -tornado==6.5.4 - # via - # ipykernel - # jupyter-client -traitlets==5.14.3 - # via - # ipykernel - # ipython - # ipywidgets - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbclient - # nbconvert - # nbformat - # nbsphinx - # pythreejs - # traittypes -traittypes==0.2.3 - # via ipydatawidgets -wcwidth==0.6.0 - # via prompt-toolkit -webencodings==0.5.1 - # via - # bleach - # tinycss2 -widgetsnbextension==4.0.15 - # via ipywidgets diff --git a/packages/essnmx/requirements/make_base.py b/packages/essnmx/requirements/make_base.py deleted file mode 100644 index 2cda547f..00000000 --- a/packages/essnmx/requirements/make_base.py +++ /dev/null @@ -1,78 +0,0 @@ -from argparse import ArgumentParser -from pathlib import Path - -import tomli - -parser = ArgumentParser() -parser.add_argument( - "--nightly", - default="", - help="List of dependencies to install from main branch for nightly tests, " - "separated by commas.", -) -args = parser.parse_args() - -CUSTOM_AUTO_SEPARATOR = """ -# --- END OF CUSTOM SECTION --- -# The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! -""" - - -def write_dependencies(dependency_name: str, dependencies: list[str]) -> None: - path = Path(f"{dependency_name}.in") - if path.exists(): - sections = path.read_text().split(CUSTOM_AUTO_SEPARATOR) - if len(sections) > 1: - custom = sections[0] - else: - custom = "" - else: - custom = "" - with path.open("w") as f: - f.write(custom) - f.write(CUSTOM_AUTO_SEPARATOR) - f.write("\n".join(dependencies)) - f.write("\n") - - -with open("../pyproject.toml", "rb") as toml_file: - pyproject = tomli.load(toml_file) - dependencies = pyproject["project"].get("dependencies") - if dependencies is None: - raise RuntimeError("No dependencies found in pyproject.toml") - dependencies = [dep.strip().strip('"') for dep in dependencies] - test_dependencies = ( - pyproject["project"].get("optional-dependencies", {}).get("test", []) - ) - test_dependencies = [dep.strip().strip('"') for dep in test_dependencies] - - -write_dependencies("base", dependencies) -write_dependencies("basetest", test_dependencies) - - -def as_nightly(repo: str) -> str: - if "/" in repo: - org, repo = repo.split("/") - else: - org = "scipp" - if repo == "scipp": - # With the standard pip resolver index-url takes precedence over - # extra-index-url but with uv it's reversed, so if we move to tox-uv - # this needs to be reversed. - return ( - "scipp\n" - "--index-url=https://pypi.anaconda.org/scipp-nightly-wheels/simple/\n" - "--extra-index-url=https://pypi.org/simple\n" - "--pre" - ) - return f"{repo} @ git+https://github.com/{org}/{repo}@main" - - -nightly = tuple(args.nightly.split(",") if args.nightly else []) -nightly_dependencies = [ - dep for dep in dependencies + test_dependencies if not dep.startswith(nightly) -] -nightly_dependencies += [as_nightly(arg) for arg in nightly] - -write_dependencies("nightly", nightly_dependencies) diff --git a/packages/essnmx/requirements/mypy.in b/packages/essnmx/requirements/mypy.in deleted file mode 100644 index 5027d8c3..00000000 --- a/packages/essnmx/requirements/mypy.in +++ /dev/null @@ -1,2 +0,0 @@ --r test.in -mypy diff --git a/packages/essnmx/requirements/mypy.txt b/packages/essnmx/requirements/mypy.txt deleted file mode 100644 index 19fcddf5..00000000 --- a/packages/essnmx/requirements/mypy.txt +++ /dev/null @@ -1,19 +0,0 @@ -# SHA1:859ef9c15e5e57c6c91510133c01f5751feee941 -# -# This file was generated by pip-compile-multi. -# To update, run: -# -# requirements upgrade -# --r test.txt -librt==0.8.1 - # via mypy -mypy==1.19.1 - # via -r mypy.in -mypy-extensions==1.1.0 - # via mypy -pathspec==1.0.4 - # via mypy - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/packages/essnmx/requirements/nightly.in b/packages/essnmx/requirements/nightly.in deleted file mode 100644 index 81c6979a..00000000 --- a/packages/essnmx/requirements/nightly.in +++ /dev/null @@ -1,22 +0,0 @@ - -# --- END OF CUSTOM SECTION --- -# The following was generated by 'tox -e deps', DO NOT EDIT MANUALLY! -dask>=2022.1.0 -essreduce>=26.2.1 -graphviz -pooch>=1.5 -pandas>=2.1.2 -gemmi>=0.6.6 -defusedxml>=0.7.1 -msgpack>=1.0.8 -tof>=25.12.1 -numpy>=1.20.0 -pytest>=8.0 -bitshuffle>=0.5.2;os_name == 'posix' -scipp ---index-url=https://pypi.anaconda.org/scipp-nightly-wheels/simple/ ---extra-index-url=https://pypi.org/simple ---pre -sciline @ git+https://github.com/scipp/sciline@main -scippnexus @ git+https://github.com/scipp/scippnexus@main -plopp @ git+https://github.com/scipp/plopp@main diff --git a/packages/essnmx/requirements/nightly.txt b/packages/essnmx/requirements/nightly.txt deleted file mode 100644 index 92bb5f0d..00000000 --- a/packages/essnmx/requirements/nightly.txt +++ /dev/null @@ -1,176 +0,0 @@ -# SHA1:03339aaa977997d3c0ce22dc2aef4b0e25334271 -# -# This file was generated by pip-compile-multi. -# To update, run: -# -# requirements upgrade -# ---index-url https://pypi.anaconda.org/scipp-nightly-wheels/simple/ ---extra-index-url https://pypi.org/simple - -annotated-types==0.7.0 - # via pydantic -bitshuffle==0.5.2 ; os_name == "posix" - # via -r nightly.in -certifi==2026.2.25 - # via requests -charset-normalizer==3.4.4 - # via requests -click==8.3.1 - # via dask -cloudpickle==3.1.2 - # via dask -contourpy==1.3.3 - # via matplotlib -cyclebane==24.10.0 - # via sciline -cycler==0.12.1 - # via matplotlib -cython==3.2.4 - # via bitshuffle -dask==2026.1.2 - # via -r nightly.in -defusedxml==0.8.0rc2 - # via -r nightly.in -dnspython==2.8.0 - # via email-validator -email-validator==2.3.0 - # via scippneutron -essreduce==26.2.2 - # via -r nightly.in -fonttools==4.61.1 - # via matplotlib -fsspec==2026.2.0 - # via dask -gemmi==0.7.4 - # via -r nightly.in -graphviz==0.21 - # via -r nightly.in -h5py==3.15.1 - # via - # bitshuffle - # scippneutron - # scippnexus -idna==3.11 - # via - # email-validator - # requests -iniconfig==2.3.0 - # via pytest -kiwisolver==1.4.10rc0 - # via matplotlib -lazy-loader==0.4 - # via - # plopp - # scippneutron - # tof -locket==1.0.0 - # via partd -matplotlib==3.10.8 - # via - # mpltoolbox - # plopp -mpltoolbox==26.2.0 - # via scippneutron -msgpack==1.1.2 - # via -r nightly.in -networkx==3.6.1 - # via cyclebane -numpy==2.4.2 - # via - # -r nightly.in - # bitshuffle - # contourpy - # h5py - # matplotlib - # pandas - # scipp - # scippneutron - # scipy -packaging==26.0 - # via - # dask - # lazy-loader - # matplotlib - # pooch - # pytest -pandas==3.0.1 - # via -r nightly.in -partd==1.4.2 - # via dask -pillow==12.1.1 - # via matplotlib -platformdirs==4.9.2 - # via pooch -plopp @ git+https://github.com/scipp/plopp@main - # via - # -r nightly.in - # scippneutron - # tof -pluggy==1.6.0 - # via pytest -pooch==1.9.0 - # via - # -r nightly.in - # tof -pydantic==2.13.0b2 - # via scippneutron -pydantic-core==2.42.0 - # via pydantic -pygments==2.19.2 - # via pytest -pyparsing==3.3.2 - # via matplotlib -pytest==9.0.2 - # via -r nightly.in -python-dateutil==2.9.0.post0 - # via - # matplotlib - # pandas - # scippneutron -pyyaml==6.0.3 - # via dask -requests==2.32.5 - # via pooch -sciline @ git+https://github.com/scipp/sciline@main - # via - # -r nightly.in - # essreduce -scipp==100.0.0.dev0 - # via - # -r nightly.in - # essreduce - # scippneutron - # scippnexus - # tof -scippneutron==26.2.0 - # via essreduce -scippnexus @ git+https://github.com/scipp/scippnexus@main - # via - # -r nightly.in - # essreduce - # scippneutron -scipy==1.17.1 - # via - # scippneutron - # scippnexus -six==1.17.0 - # via python-dateutil -tof==26.1.0 - # via -r nightly.in -toolz==1.1.0 - # via - # dask - # partd -typing-extensions==4.15.0 - # via - # pydantic - # pydantic-core - # typing-inspection -typing-inspection==0.4.2 - # via pydantic -urllib3==2.6.3 - # via requests - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/packages/essnmx/requirements/static.in b/packages/essnmx/requirements/static.in deleted file mode 100644 index 416634f5..00000000 --- a/packages/essnmx/requirements/static.in +++ /dev/null @@ -1 +0,0 @@ -pre-commit diff --git a/packages/essnmx/requirements/static.txt b/packages/essnmx/requirements/static.txt deleted file mode 100644 index e80710d7..00000000 --- a/packages/essnmx/requirements/static.txt +++ /dev/null @@ -1,31 +0,0 @@ -# SHA1:5a0b1bb22ae805d8aebba0f3bf05ab91aceae0d8 -# -# This file was generated by pip-compile-multi. -# To update, run: -# -# requirements upgrade -# -cfgv==3.5.0 - # via pre-commit -distlib==0.4.0 - # via virtualenv -filelock==3.25.0 - # via - # python-discovery - # virtualenv -identify==2.6.17 - # via pre-commit -nodeenv==1.10.0 - # via pre-commit -platformdirs==4.9.2 - # via - # python-discovery - # virtualenv -pre-commit==4.5.1 - # via -r static.in -python-discovery==1.1.0 - # via virtualenv -pyyaml==6.0.3 - # via pre-commit -virtualenv==21.1.0 - # via pre-commit diff --git a/packages/essnmx/requirements/test.in b/packages/essnmx/requirements/test.in deleted file mode 100644 index 7b409792..00000000 --- a/packages/essnmx/requirements/test.in +++ /dev/null @@ -1,4 +0,0 @@ -# Add test dependencies in basetest.in - --r base.in --r basetest.in diff --git a/packages/essnmx/requirements/test.txt b/packages/essnmx/requirements/test.txt deleted file mode 100644 index c900374a..00000000 --- a/packages/essnmx/requirements/test.txt +++ /dev/null @@ -1,12 +0,0 @@ -# SHA1:ef2ee9576d8a9e65b44e2865a26887eed3fc49d1 -# -# This file was generated by pip-compile-multi. -# To update, run: -# -# requirements upgrade -# --r base.txt --r basetest.txt - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/packages/essnmx/requirements/wheels.in b/packages/essnmx/requirements/wheels.in deleted file mode 100644 index 378eac25..00000000 --- a/packages/essnmx/requirements/wheels.in +++ /dev/null @@ -1 +0,0 @@ -build diff --git a/packages/essnmx/requirements/wheels.txt b/packages/essnmx/requirements/wheels.txt deleted file mode 100644 index 0d70d60c..00000000 --- a/packages/essnmx/requirements/wheels.txt +++ /dev/null @@ -1,13 +0,0 @@ -# SHA1:80754af91bfb6d1073585b046fe0a474ce868509 -# -# This file was generated by pip-compile-multi. -# To update, run: -# -# requirements upgrade -# -build==1.4.0 - # via -r wheels.in -packaging==26.0 - # via build -pyproject-hooks==1.2.0 - # via build diff --git a/packages/essnmx/tox.ini b/packages/essnmx/tox.ini deleted file mode 100644 index 3fbfcaed..00000000 --- a/packages/essnmx/tox.ini +++ /dev/null @@ -1,72 +0,0 @@ -[tox] -envlist = py311 -isolated_build = true - -[testenv] -deps = -r requirements/test.txt -setenv = - JUPYTER_PLATFORM_DIRS = 1 -commands = pytest {posargs} - -[testenv:nightly] -deps = -r requirements/nightly.txt -setenv = - PIP_INDEX_URL = https://pypi.anaconda.org/scipp-nightly-wheels/simple - PIP_EXTRA_INDEX_URL = https://pypi.org/simple -commands = pytest {posargs} - -[testenv:unpinned] -description = Test with unpinned dependencies, as a user would install now. -deps = - -r requirements/basetest.txt - essnmx -commands = pytest {posargs} - -[testenv:docs] -description = invoke sphinx-build to build the HTML docs -deps = -r requirements/docs.txt -allowlist_externals=find -commands = python -m sphinx -W -j2 -v -b html -d {toxworkdir}/docs_doctrees docs html - python -m sphinx -W -j2 -v -b doctest -d {toxworkdir}/docs_doctrees docs html - find html -type f -name "*.ipynb" -not -path "html/_sources/*" -delete - -[testenv:releasedocs] -description = invoke sphinx-build to build the HTML docs from a released version -skip_install = true -deps = - essnmx=={posargs} - {[testenv:docs]deps} -allowlist_externals={[testenv:docs]allowlist_externals} -commands = {[testenv:docs]commands} - -[testenv:linkcheck] -description = Run Sphinx linkcheck -deps = -r requirements/docs.txt -commands = python -m sphinx -j2 -v -b linkcheck -d {toxworkdir}/docs_doctrees docs html - -[testenv:static] -description = Code formatting and static analysis -skip_install = true -deps = -r requirements/static.txt -allowlist_externals = sh -# The first run of pre-commit may reformat files. If this happens, it returns 1 but this -# should not fail the job. So just run again if it fails. A second failure means that -# either the different formatters can't agree on a format or that static analysis failed. -commands = sh -c 'pre-commit run -a || (echo "" && pre-commit run -a)' - -[testenv:mypy] -description = Type checking (mypy) -deps = -r requirements/mypy.txt -commands = python -m mypy . - -[testenv:deps] -description = Update dependencies by running pip-compile-multi -deps = - pip-compile-multi - tomli - # Avoid https://github.com/jazzband/pip-tools/issues/2131 - pip==24.2 -skip_install = true -changedir = requirements -commands = python ./make_base.py --nightly scipp,sciline,scippnexus,plopp - pip-compile-multi -d . --backtracking --annotate-index From a48ecb40c6d7528fe2fb1ba0eacc7ac0477a7ebc Mon Sep 17 00:00:00 2001 From: YooSunYoung <17974113+YooSunYoung@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:13:09 +0100 Subject: [PATCH 403/403] Add setup tool configuration for subpackage deployment. --- packages/essnmx/pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/essnmx/pyproject.toml b/packages/essnmx/pyproject.toml index 6fd597d7..a225a968 100644 --- a/packages/essnmx/pyproject.toml +++ b/packages/essnmx/pyproject.toml @@ -10,7 +10,7 @@ name = "essnmx" description = "Data reduction for NMX at the European Spallation Source." authors = [{ name = "Scipp contributors" }] license = "BSD-3-Clause" -license-files = ["LICENSE"] +license-files = ["../../LICENSE"] readme = "README.md" classifiers = [ "Intended Audience :: Science/Research", @@ -65,6 +65,9 @@ test = [ "Source" = "https://github.com/scipp/essnmx" [tool.setuptools_scm] +root = "../.." +tag_regex = "^essnmx/(?P[vV]?\\d+(?:\\.\\d+)*(?:[._-]?\\w+)*)$" +git_describe_command = [ "git", "describe", "--dirty", "--long", "--match", "essnmx/*[0-9]*"] [tool.pytest.ini_options] minversion = "7.0"