diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 0000000..a5ba01a --- /dev/null +++ b/.codespellrc @@ -0,0 +1,3 @@ +[codespell] +skip = CHANGELOG.md +ignore-words-list = nin, notIn diff --git a/.github/workflows/test_and_release.yml b/.github/workflows/test_and_release.yml new file mode 100644 index 0000000..38e8b4b --- /dev/null +++ b/.github/workflows/test_and_release.yml @@ -0,0 +1,92 @@ +name: Test and Release + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + + # Install and run pre-commit + - run: | + pip install pre-commit + pre-commit install + pre-commit run --all-files + + pytest: + name: Pytest ${{ matrix.config.name }} ${{ matrix.python-version }} + runs-on: ${{ matrix.config.os }} + + services: + mongo: + image: mongo:7 + ports: + - 27017:27017 + + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + config: + - { name: "Linux", os: ubuntu-latest } + + defaults: + run: + shell: bash + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + + - name: Install system dependencies + if: matrix.config.os == 'ubuntu-latest' + run: | + sudo apt-get update + sudo apt-get install -y \ + libsdl2-ttf-2.0-0 \ + libsdl2-2.0-0 + + - name: Install and Run Tests + run: | + python -m pip install --upgrade pip + python -m pip install ".[dev]" + python -m pytest -s tests + + release: + needs: [pre-commit, pytest] + runs-on: ubuntu-latest + if: github.event_name == 'push' + environment: release + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + contents: write # IMPORTANT: mandatory for making GitHub Releases + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Python Semantic Release + id: release + uses: python-semantic-release/python-semantic-release@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..9c41816 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,81 @@ +ci: + autoupdate_commit_msg: "chore: update pre-commit hooks" + autofix_commit_msg: "style: pre-commit fixes" + +exclude: ^.cruft.json|.copier-answers.yml$ + +repos: + - repo: https://github.com/adamchainz/blacken-docs + rev: "1.19.1" + hooks: + - id: blacken-docs + additional_dependencies: [black==24.*] + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: "v5.0.0" + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-merge-conflict + - id: check-symlinks + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: mixed-line-ending + - id: name-tests-test + args: ["--pytest-test-first"] + - id: requirements-txt-fixer + - id: trailing-whitespace + + - repo: https://github.com/pre-commit/pygrep-hooks + rev: "v1.10.0" + hooks: + - id: rst-backticks + - id: rst-directive-colons + - id: rst-inline-touching-normal + + - repo: https://github.com/rbubley/mirrors-prettier + rev: "v3.4.2" + hooks: + - id: prettier + types_or: [yaml, markdown, html, css, scss, javascript, json] + args: [--prose-wrap=always] + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.11.9" + hooks: + - id: ruff + args: ["--fix", "--show-fixes"] + - id: ruff-format + args: ["--line-length", "120"] + + - repo: https://github.com/codespell-project/codespell + rev: "v2.3.0" + hooks: + - id: codespell + + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: "v0.10.0.1" + hooks: + - id: shellcheck + + - repo: local + hooks: + - id: disallow-caps + name: Disallow improper capitalization + language: pygrep + entry: PyBind|Numpy|Cmake|CCache|Github|PyTest + exclude: .pre-commit-config.yaml + + - repo: https://github.com/abravalheri/validate-pyproject + rev: "v0.23" + hooks: + - id: validate-pyproject + additional_dependencies: ["validate-pyproject-schema-store[all]"] + + - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook + rev: v9.20.0 + hooks: + - id: commitlint + stages: [commit-msg] + additional_dependencies: ["@commitlint/config-conventional"] diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..1b763b1 --- /dev/null +++ b/.prettierignore @@ -0,0 +1 @@ +CHANGELOG.md diff --git a/README.md b/README.md index 00df540..156e06e 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,36 @@ # GirderBIDS -Girder plugin to import a BIDS database +A Girder plugin providing support for working with datasets in the Brain Imaging +Data Structure (BIDS) format. The project includes: + +- A Girder plugin for managing and handling BIDS datasets within Girder. +- A command-line interface (CLI) for validating and importing BIDS datasets into + a Girder instance (with or without the plugin). ## 1. Setup ### Ubuntu 22.04: + ```bash -pip install -r requirements.txt +pip install ".[cli]" ``` ### MacOS + ```bash curl -fsSL https://deno.land/install.sh | sh deno compile -ERWN -o bids-validator jsr:@bids/validator -pip install girder girder-client fire +pip install ".[cli]" ``` ### Windows -Not currently supported. + +BIDS Validation is not supported but you can still use the plugin and the +importer (without validation) + +```bash +pip install ".[cli]" +``` ## 2. Build girder front @@ -28,11 +41,15 @@ girder build ``` ### MacOS -Install npm using Homebrew : + +Install npm using Homebrew : + ```bash brew install node ``` + Add the following environment variable: + ```bash export NODE_OPTIONS=--openssl-legacy-provider ``` @@ -44,6 +61,7 @@ girder build ## 3. Install mongodb ### Ubuntu 22.04 + To set up MongoDB 4.4 on Ubuntu 22.04 execute the following commands : ```bash @@ -61,7 +79,9 @@ chown -R mongodb:mongodb /var/lib/mongodb/* ``` ### MacOS + Add MongoDB 4.4: + ```bash brew tap mongodb/brew brew install mongodb-community@4.4 @@ -74,7 +94,8 @@ brew services start mongodb-community@4.4 girder serve ``` -You can specify a database other than "girder" by creating a girder.cfg with following content: +You can specify a database other than "girder" by creating a girder.cfg with +following content: ``` [database] @@ -88,14 +109,23 @@ GIRDER_CONFIG=./girder.cfg girder serve ``` ## 4. Create admin account with "Register" on localhost:8080 + ## 5. Login and create API key + ## 6. Create assetstore on localhost:8080 -## 7. Create a Collection and a Folder in the collection, copy the ID of the created Folder + +## 7. Create a Collection copy the ID of the created collection ## 8. Import BIDS database ```bash -python tools/bids-importer.py --bids_dir ... --girder_api_url http://localhost:8080/api/v1 --girder_api_key ... --girder_folder_id ... +bids-importer --bids_dir ... --api_url http://localhost:8080/api/v1 --api_key ... --location_id ... ``` -If you want to ignore the validation step, pass `--ignore_validation` on the command-line. +If you want to ignore the validation step, pass `--ignore_validation` on the +command-line. If you need to upload the dataset directly under a collection or a +user, pass `--location_type {collection / user / folder}` (default is `folder`). +If you want to import the BIDS Dataset using the Girder Plugin, pass +`--use_plugin` on the command-line. If you want to copy BIDS JSON to Girder +metadata, pass `--extract_metadata` on the command-line (works only for +`dataset_description.json` and JSON sidecars). diff --git a/bids_plugin/__init__.py b/bids_plugin/__init__.py new file mode 100644 index 0000000..e9969d4 --- /dev/null +++ b/bids_plugin/__init__.py @@ -0,0 +1,17 @@ +from typing import Any + +from girder import plugin +from girder.utility.model_importer import ModelImporter + +from .api import BIDSDatasetResource, BIDSFolderResource, BIDSItemResource +from .models import BIDSDatasetModel, BIDSFolderModel, BIDSItemModel + + +class BIDSPlugin(plugin.GirderPlugin): + def load(self, info: dict[str, Any]) -> None: + ModelImporter.registerModel("bids_dataset", BIDSDatasetModel, plugin="bids_plugin") + info["apiRoot"].bids_dataset = BIDSDatasetResource() + ModelImporter.registerModel("bids_folder", BIDSFolderModel, plugin="bids_plugin") + info["apiRoot"].bids_folder = BIDSFolderResource() + ModelImporter.registerModel("bids_item", BIDSItemModel, plugin="bids_plugin") + info["apiRoot"].bids_item = BIDSItemResource() diff --git a/bids_plugin/api/__init__.py b/bids_plugin/api/__init__.py new file mode 100644 index 0000000..43d9867 --- /dev/null +++ b/bids_plugin/api/__init__.py @@ -0,0 +1,5 @@ +from .bids_dataset import BIDSDatasetResource +from .bids_folder import BIDSFolderResource +from .bids_item import BIDSItemResource + +__all__ = ["BIDSDatasetResource", "BIDSFolderResource", "BIDSItemResource"] diff --git a/bids_plugin/api/bids_dataset.py b/bids_plugin/api/bids_dataset.py new file mode 100644 index 0000000..7785f45 --- /dev/null +++ b/bids_plugin/api/bids_dataset.py @@ -0,0 +1,133 @@ +from typing import Any + +from girder.api import access +from girder.api.describe import Description, autoDescribeRoute +from girder.api.rest import Resource, filtermodel +from girder.constants import AccessType, SortDir, TokenScope +from girder.models.collection import Collection +from girder.utility.model_importer import ModelImporter +from pydantic import TypeAdapter +from pymongo.cursor import Cursor + +from bids_plugin.models import BIDSDatasetModel +from bids_plugin.utility import BIDSDescription, GirderModel, MongoOperators + + +class BIDSDatasetResource(Resource): + """RESTful BIDS dataset resource""" + + def __init__(self) -> None: + super().__init__() + self.resourceName = "bids_dataset" + self._model = BIDSDatasetModel() + self.route("GET", (), self.list_datasets) + self.route("POST", (), self.create_dataset) + + @access.user(TokenScope.DATA_READ) + @filtermodel(model=BIDSDatasetModel) + @autoDescribeRoute( + Description("List bids dataset in the database user has access to.") + .responseClass("BIDSDataset", array=True) + .modelParam( + "collection_id", + "The ID of the root collection.", + model=Collection, + level=AccessType.WRITE, + paramType="query", + destName="collection", + required=False, + ) + .modelParam( + "parent_id", + "The ID of the parent.", + model=BIDSDatasetModel, + level=AccessType.WRITE, + paramType="query", + destName="parent", + required=False, + ) + .param( + "is_derivative", + "Whether to search for derivative datasets", + dataType="boolean", + required=False, + ) + .pagingParams(defaultSort="created", defaultSortDir=SortDir.DESCENDING) + ) + def list_datasets( + self, + collection: GirderModel | None, + parent: GirderModel | None, + is_derivative: bool, + limit: int, + offset: int, + sort: str, + ) -> Cursor | Any: + user = self.getCurrentUser() + query = { + "dataset_description": {MongoOperators.exists: True, MongoOperators.notEqual: None}, + } + if collection is not None: + query.update({"baseParentId": collection["_id"]}) + + if parent is not None: + query.update({"parentId": parent["_id"]}) + + if is_derivative is not None: + query.update({"dataset_description.DatasetType": "derivative" if is_derivative else "raw"}) + + return self._model.find( + query=query, + offset=offset, + limit=limit, + sort=sort, + user=user, + ) + + @access.user(scope=TokenScope.DATA_WRITE) + @filtermodel(model=BIDSDatasetModel) + @autoDescribeRoute( + Description("Create a new BIDS dataset.") + .responseClass("BIDSDataset") + .param("parent_id", "The ID of the dataset's parent.") + .param( + "parent_type", + "Type of the dataset's parent", + required=False, + enum=["folder", "user", "collection"], + default="folder", + ) + .param("name", "Name of the BIDS Dataset.", strip=True) + .jsonParam( + "dataset_description", + "A JSON object containing the dataset description keys to add", + paramType="form", + schema=TypeAdapter(BIDSDescription).json_schema(), + ) + .param( + "reuse_existing", + "Return existing BIDS dataset if it exists rather than creating a new one.", + dataType="boolean", + required=False, + default=False, + ) + .errorResponse() + .errorResponse("Write access was denied on the parent.", 403) + ) + def create_dataset( + self, parent_type: str, parent_id: str, name: str, dataset_description: dict[str, Any], reuse_existing: bool + ) -> GirderModel: + user = self.getCurrentUser() + parent = ModelImporter.model(parent_type).load(id=parent_id, user=user, level=AccessType.WRITE, exc=True) + + dataset_description["Name"] = name + TypeAdapter(BIDSDescription).validate_python(dataset_description) + + return self._model.create_bids_dataset( + user, + name, + parent, + BIDSDescription(**dataset_description), + parent_type=parent_type, + reuse_existing=reuse_existing, + ) diff --git a/bids_plugin/api/bids_folder.py b/bids_plugin/api/bids_folder.py new file mode 100644 index 0000000..aba0142 --- /dev/null +++ b/bids_plugin/api/bids_folder.py @@ -0,0 +1,87 @@ +from typing import Any + +from girder.api import access +from girder.api.describe import Description, autoDescribeRoute +from girder.api.rest import Resource, filtermodel +from girder.constants import AccessType, SortDir, TokenScope +from girder.models.folder import Folder +from pymongo.cursor import Cursor + +from bids_plugin.models import BIDSDatasetModel, BIDSFolderModel +from bids_plugin.utility import GirderModel + + +class BIDSFolderResource(Resource): + """RESTful BIDS Folder resource""" + + def __init__(self) -> None: + super().__init__() + self.resourceName = "bids_folder" + self._model = BIDSFolderModel() + self.route("GET", (), self.list_folders) + self.route("POST", (), self.create_folder) + + @access.user(TokenScope.DATA_READ) + @filtermodel(model=BIDSFolderModel) + @autoDescribeRoute( + Description("List bids folder in a BIDS dataset user has access to.") + .responseClass("BIDSFolder", array=True) + .modelParam( + "dataset_id", + "The ID of the root BIDS dataset", + model=BIDSDatasetModel, + level=AccessType.READ, + paramType="query", + destName="dataset", + ) + .pagingParams(defaultSort="created", defaultSortDir=SortDir.DESCENDING) + ) + def list_folders(self, dataset: GirderModel, limit: int, offset: int, sort: str) -> Cursor | Any: + user = self.getCurrentUser() + query = {"dataset_id": dataset["_id"]} + + return self._model.find( + query=query, + limit=limit, + offset=offset, + sort=sort, + user=user, + ) + + @access.user(scope=TokenScope.DATA_WRITE) + @filtermodel(model=BIDSFolderModel) + @autoDescribeRoute( + Description("Create a new BIDS dataset.") + .responseClass("BIDSFolder") + .modelParam( + "folder_id", + "The ID of the parent folder.", + model=Folder, + level=AccessType.WRITE, + paramType="query", + destName="folder", + ) + .param("name", "Name of the BIDS Folder.", strip=True) + .param( + "reuse_existing", + "Return existing BIDS folder if it exists rather than creating a new one.", + dataType="boolean", + required=False, + default=False, + ) + .errorResponse() + .errorResponse("Write access was denied on the parent.", 403) + ) + def create_folder( + self, + folder: GirderModel, + name: str, + reuse_existing: bool, + ) -> GirderModel: + user = self.getCurrentUser() + return self._model.create_bids_folder( + user, + name, + folder, + reuse_existing=reuse_existing, + ) diff --git a/bids_plugin/api/bids_item.py b/bids_plugin/api/bids_item.py new file mode 100644 index 0000000..2f998f8 --- /dev/null +++ b/bids_plugin/api/bids_item.py @@ -0,0 +1,148 @@ +from typing import Any + +from girder.api import access +from girder.api.describe import Description, autoDescribeRoute +from girder.api.rest import Resource, filtermodel +from girder.constants import AccessType, SortDir, TokenScope +from pymongo.cursor import Cursor + +from bids_plugin.models import BIDSDatasetModel, BIDSFolderModel, BIDSItemModel +from bids_plugin.utility import GirderModel + + +class BIDSItemResource(Resource): + """RESTful BIDS Item resource""" + + def __init__(self) -> None: + super().__init__() + self.resourceName = "bids_item" + self._model = BIDSItemModel() + self.route("GET", (), self.list_items) + self.route("GET", (":id", "path"), self.get_dataset_path) + self.route("POST", (), self.create_item) + + @access.user(TokenScope.DATA_READ) + @filtermodel(model=BIDSItemModel) + @autoDescribeRoute( + Description("List bids item in the database user has access to.") + .responseClass("BIDSItem", array=True) + .modelParam( + "dataset_id", + "The ID of the root BIDS dataset", + model=BIDSDatasetModel, + level=AccessType.WRITE, + paramType="query", + destName="dataset", + ) + .modelParam( + "source_id", + "The ID of the source BIDS item", + model=BIDSItemModel, + level=AccessType.READ, + paramType="query", + destName="source", + required=False, + ) + .param( + "name", + "The name of the BIDS item to search for", + dataType="boolean", + required=False, + strip=True, + ) + .param("suffix", "Pass this to search BIDS item by suffix", required=False) + .param("extension", "Pass this to search BIDS item by extension", required=False) + .pagingParams(defaultSort="name", defaultSortDir=SortDir.ASCENDING) + ) + def list_items( + self, + dataset: GirderModel, + source: GirderModel | None, + name: str | None, + suffix: str | None, + extension: str | None, + limit: int, + offset: int, + sort: Any, + ) -> Cursor | Any: + user = self.getCurrentUser() + query = {"dataset_id": dataset["_id"]} + if source is not None: + query.update({"source_id": source["_id"]}) + + if name is not None: + query.update({"name": name}) + + if suffix is not None: + query.update({"suffix": suffix}) + + if extension is not None: + query.update({"extension": extension}) + + return self._model.find( + query=query, + limit=limit, + offset=offset, + sort=sort, + user=user, + ) + + @access.user(scope=TokenScope.DATA_WRITE) + @filtermodel(model=BIDSItemModel) + @autoDescribeRoute( + Description("Create a new BIDS item.") + .responseClass("BIDSItem") + .param("name", "Name of the BIDS item.", strip=True) + .modelParam( + "folder_id", + "The ID of the parent BIDS folder.", + model=BIDSFolderModel, + level=AccessType.WRITE, + paramType="query", + destName="folder", + ) + .modelParam( + "source_id", + "The ID of the optional source item.", + model=BIDSItemModel, + level=AccessType.WRITE, + paramType="query", + destName="source", + required=False, + ) + .param( + "reuse_existing", + "Return existing BIDS folder if it exists rather than creating a new one.", + dataType="boolean", + required=False, + default=False, + ) + .errorResponse() + .errorResponse("Write access was denied on the parent.", 403) + ) + def create_item( + self, + name: str, + folder: GirderModel, + source: GirderModel | None, + reuse_existing: bool, + ) -> GirderModel: + user = self.getCurrentUser() + + return self._model.create_bids_item( + user, + name, + folder, + source, + reuse_existing, + ) + + @access.user(scope=TokenScope.DATA_READ) + @autoDescribeRoute( + Description("Get the path to BIDS dataset of the item.") + .modelParam("id", model=BIDSItemModel, level=AccessType.READ, destName="item") + .errorResponse("ID was invalid.") + .errorResponse("Read access was denied for the item.", 403) + ) + def get_dataset_path(self, item: GirderModel) -> list[GirderModel]: + return self._model.parents_to_dataset(item, self.getCurrentUser()) diff --git a/bids_plugin/models/__init__.py b/bids_plugin/models/__init__.py new file mode 100644 index 0000000..f54d6f3 --- /dev/null +++ b/bids_plugin/models/__init__.py @@ -0,0 +1,5 @@ +from .bids_dataset import BIDSDatasetModel +from .bids_folder import BIDSFolderModel +from .bids_item import BIDSItemModel + +__all__ = ["BIDSDatasetModel", "BIDSFolderModel", "BIDSItemModel"] diff --git a/bids_plugin/models/bids_dataset.py b/bids_plugin/models/bids_dataset.py new file mode 100644 index 0000000..41e6603 --- /dev/null +++ b/bids_plugin/models/bids_dataset.py @@ -0,0 +1,70 @@ +from typing import Any + +from girder.constants import AccessType +from girder.exceptions import ValidationException +from girder.models.folder import Folder + +from bids_plugin.utility import ( + BIDSDataset, + BIDSDescription, + GirderModel, +) + + +class BIDSDatasetModel(Folder): + def initialize(self) -> None: + super().initialize() + self.exposeFields( + level=AccessType.READ, + fields=BIDSDataset.fields(), + ) + + def create_bids_dataset( + self, + user: GirderModel, + name: str, + parent: GirderModel, + dataset_description: BIDSDescription, + parent_type: str = "folder", + reuse_existing: bool = False, + ) -> GirderModel | Any: + if reuse_existing: + name.strip("") + existing = self.findOne({"parentId": parent["_id"], "name": name}) + if existing: + return existing + + bids_dataset_folder = self.createFolder(parent, name, parentType=parent_type, creator=user, allowRename=True) + + if dataset_description.DatasetType == "raw": + derivatives_folder_id = self.createFolder(bids_dataset_folder, name="derivatives")["_id"] + else: + derivatives_folder_id = None + + bids_dataset = BIDSDataset( + name=name, + dataset_description=dataset_description, + derivatives_folder_id=derivatives_folder_id, + ).as_dict() + + bids_dataset_folder.update(bids_dataset) + return self.save_bids(bids_dataset_folder) + + def save_bids(self, dataset: GirderModel) -> None: + try: + self.validate_bids(dataset) + return self.save(dataset) + + except ValidationException as e: + self.remove(dataset) + raise e + + def validate_bids(self, dataset: GirderModel) -> None: + if "dataset_description" not in dataset: + raise ValidationException("Invalid BIDS Dataset: missing 'dataset_description' field") + + if "derivatives_folder_id" not in dataset: + raise ValidationException("Invalid BIDS Dataset: missing 'derivatives_folder_id' field") + + if not dataset["dataset_description"].get("BIDSVersion"): + raise ValidationException("Invalid BIDS Dataset: missing 'BIDSVersion' field in dataset description") diff --git a/bids_plugin/models/bids_folder.py b/bids_plugin/models/bids_folder.py new file mode 100644 index 0000000..0769c99 --- /dev/null +++ b/bids_plugin/models/bids_folder.py @@ -0,0 +1,99 @@ +from typing import Any + +from girder.constants import AccessType +from girder.exceptions import ValidationException +from girder.models.folder import Folder + +from bids_plugin.models import BIDSDatasetModel +from bids_plugin.utility import ( + BIDSDatatype, + BIDSFolder, + GirderModel, +) + + +class BIDSFolderModel(Folder): + def initialize(self) -> None: + super().initialize() + self.exposeFields( + level=AccessType.READ, + fields=BIDSFolder.fields(), + ) + + def _check_bids_hierarchy(self, folder_name: GirderModel, parent_folder: GirderModel) -> None: + parent_name = parent_folder["name"] + if folder_name.startswith("sub-"): + if parent_name.startswith("sub-"): + raise ValidationException("Invalid BIDS Hierarchy: Subject folder must be at dataset level.") + return + + if folder_name.startswith("ses-"): + if not parent_name.startswith("sub-"): + raise ValidationException("Invalid BIDS Hierarchy: Session folder must be at subject level.") + return + + if folder_name in BIDSDatatype: + if not parent_name.startswith(("sub-", "ses-")): + raise ValidationException( + "Invalid BIDS Hierarchy: Datatype folder must be at subject or session level." + ) + return + + raise ValidationException("Invalid BIDS Folder name: Unconventional BIDS folder name") + + def parents_to_dataset( + self, folder: GirderModel, user: GirderModel | None = None, path: list[GirderModel] | None = None + ) -> list[GirderModel]: + force = user is None + path = path or [] + parent_id = folder["parentId"] + if parent_id == folder["dataset_id"]: + return path + + parent_folder = self.load(parent_id, level=AccessType.READ, user=user, force=force) + path = [self.filter(parent_folder, user), *path] + return self.parents_to_dataset(parent_folder, user, path) + + def create_bids_folder( + self, + user: GirderModel, + name: str, + parent_folder: GirderModel, + reuse_existing: bool = False, + ) -> GirderModel | Any: + if reuse_existing: + existing = self.findOne({"parentId": parent_folder["_id"], "name": name}) + if existing: + return existing + + if parent_folder.get("dataset_description"): + BIDSDatasetModel().validate_bids(parent_folder) + dataset_id = parent_folder["_id"] + else: + BIDSFolderModel().validate_bids(parent_folder) + dataset_id = parent_folder["dataset_id"] + + self._check_bids_hierarchy(name, parent_folder) + + bids_folder = self.createFolder(parent_folder, name, creator=user) + bids_folder.update( + BIDSFolder( + name=name, + dataset_id=dataset_id, + ).as_dict() + ) + + return self.save_bids(bids_folder) + + def save_bids(self, folder: GirderModel) -> None: + try: + self.validate_bids(folder) + return self.save(folder) + + except ValidationException as e: + self.remove(folder) + raise e + + def validate_bids(self, folder: GirderModel) -> None: + if not folder.get("dataset_id"): + raise ValidationException("Invalid BIDS Folder: missing 'dataset_id' field") diff --git a/bids_plugin/models/bids_item.py b/bids_plugin/models/bids_item.py new file mode 100644 index 0000000..f3806e9 --- /dev/null +++ b/bids_plugin/models/bids_item.py @@ -0,0 +1,113 @@ +import re +from typing import Any + +from girder.constants import AccessType +from girder.exceptions import ValidationException +from girder.models.item import Item + +from bids_plugin.models import BIDSDatasetModel, BIDSFolderModel +from bids_plugin.utility import BIDSDatatype, BIDSItem, GirderModel + + +class BIDSItemModel(Item): + def initialize(self) -> None: + super().initialize() + self.exposeFields( + level=AccessType.READ, + fields=BIDSItem.fields(), + ) + + def _check_bids_hierarchy( + self, item_name: GirderModel, item_extension: str | None, parent_folder: GirderModel + ) -> None: + parent_name = parent_folder["name"] + if parent_name not in BIDSDatatype and not ( + item_extension is None or item_extension.startswith(("json", "tsv")) + ): + raise ValidationException("Invalid BIDS Hierarchy: data items must be at datatype level.") + + if "dataset_description" in parent_folder: + return + + hierarchy = BIDSFolderModel().parents_to_dataset(parent_folder) + prefix = "_".join(folder["name"] for folder in hierarchy) + if not item_name.startswith(prefix): + raise ValidationException(f"Invalid BIDS Item name: item name must start with '{prefix}'.") + + def _extract_bids_suffix_and_extension(self, item_name: str) -> tuple[str | None, str | None]: + name_parts = item_name.split(".") + base_name = name_parts[0] + extension = ".".join(name_parts[1:]) if len(name_parts) > 1 else None + + if "-" not in base_name: + return None, extension + + match = re.search(r"_([a-zA-Z0-9]+)$", base_name) + if match: + return match.group(1), extension + + return None, extension + + def parents_to_dataset(self, item: GirderModel, user: GirderModel | None = None) -> list[GirderModel]: + force = user is None + folder_model = BIDSFolderModel() + parent_folder = folder_model.load(item["folderId"], level=AccessType.READ, user=user, force=force) + return folder_model.parents_to_dataset(parent_folder, user, [folder_model.filter(parent_folder, user)]) + + def create_bids_item( + self, + user: GirderModel, + name: str, + parent_folder: GirderModel, + source: GirderModel | None = None, + reuse_existing: bool = False, + ) -> GirderModel | Any: + if reuse_existing: + existing = self.findOne({"folderId": parent_folder["_id"], "name": name}) + if existing: + return existing + + if parent_folder.get("dataset_description"): + BIDSDatasetModel().validate_bids(parent_folder) + dataset_id = parent_folder["_id"] + else: + BIDSFolderModel().validate_bids(parent_folder) + dataset_id = parent_folder["dataset_id"] + + suffix, extension = self._extract_bids_suffix_and_extension(name) + + self._check_bids_hierarchy(name, extension, parent_folder) + + item = self.createItem(name, user, parent_folder) + item.update( + BIDSItem( + name=name, + dataset_id=dataset_id, + source_id=source["_id"] if source else None, + suffix=suffix, + extension=extension, + ).as_dict() + ) + + return self.save_bids(item) + + def save_bids(self, item: GirderModel) -> None: + try: + self.validate_bids(item) + return self.save(item) + except ValidationException as e: + self.remove(item) + raise e + + def validate_bids(self, item: GirderModel) -> None: + if not item.get("dataset_id"): + raise ValidationException("Invalid BIDS Item: missing 'dataset_id' field") + + if "source_id" not in item: + raise ValidationException("Invalid BIDS Item: missing 'source_id' field") + + if "suffix" not in item: + raise ValidationException("Invalid BIDS Item: missing 'suffix' field") + + if "extension" not in item: + raise ValidationException("Invalid BIDS Item: missing 'extension' field") diff --git a/bids_plugin/utility/__init__.py b/bids_plugin/utility/__init__.py new file mode 100644 index 0000000..d20aca6 --- /dev/null +++ b/bids_plugin/utility/__init__.py @@ -0,0 +1,19 @@ +from .models import ( + BIDSDataset, + BIDSDatatype, + BIDSDescription, + BIDSFolder, + BIDSItem, + GirderModel, +) +from .mongo_utilities import MongoOperators + +__all__ = [ + "BIDSDataset", + "BIDSDatatype", + "BIDSDescription", + "BIDSFolder", + "BIDSItem", + "GirderModel", + "MongoOperators", +] diff --git a/bids_plugin/utility/models.py b/bids_plugin/utility/models.py new file mode 100644 index 0000000..ef346f5 --- /dev/null +++ b/bids_plugin/utility/models.py @@ -0,0 +1,77 @@ +from dataclasses import asdict, dataclass, field, fields +from enum import Enum +from typing import Any + +GirderModel = dict[str, Any] + + +class BIDSDatatype(Enum): + UNDEFINED = None + ANAT = "anat" + FUNC = "func" + FMAP = "fmap" + DWI = "dwi" + PERF = "perf" + EEG = "eeg" + MEG = "meg" + IEEG = "ieeg" + BEH = "beh" + PET = "pet" + MICR = "micr" + NIRS = "nirs" + MOTION = "motion" + MRS = "mrs" + + +@dataclass +class Model: + name: str | None = None + + @classmethod + def fields(cls) -> list[str]: + return [f.name for f in fields(cls)] + + def as_dict(self, extra_fields: dict[str, Any] | None = None) -> dict[str, Any]: + model_dict = asdict(self) + + if isinstance(extra_fields, dict): + model_dict.update(extra_fields) + + return model_dict + + +@dataclass +class BIDSDescription: + Name: str = "" + BIDSVersion: str = "" + HEDVersion: str = "" + DatasetType: str = "raw" + License: str = "" + Authors: list = field(default_factory=list) + Acknowledgements: str = "" + HowToAcknowledge: str = "" + Funding: list[Any] = field(default_factory=list) + EthicsApprovals: list[Any] = field(default_factory=list) + ReferencesAndLinks: list[Any] = field(default_factory=list) + DatasetDOI: str = "doi:" + GeneratedBy: str = "" + SourceDatasets: list[Any] = field(default_factory=list) + + +@dataclass +class BIDSItem(Model): + dataset_id: str | None = None + extension: str | None = None + suffix: str | None = None + source_id: str | None = None + + +@dataclass +class BIDSFolder(Model): + dataset_id: str | None = None + + +@dataclass +class BIDSDataset(Model): + dataset_description: BIDSDescription = field(default_factory=BIDSDescription) + derivatives_folder_id: str | None = None diff --git a/bids_plugin/utility/mongo_utilities.py b/bids_plugin/utility/mongo_utilities.py new file mode 100644 index 0000000..82dba47 --- /dev/null +++ b/bids_plugin/utility/mongo_utilities.py @@ -0,0 +1,42 @@ +class MongoOperators: + addFields = "$addFields" + addToSet = "$addToSet" + and_ = "$and" + as_ = "as" + equal = "$eq" + exists = "$exists" + expr = "$expr" + first = "$first" + from_ = "from" + greater = "$gt" + greaterEqual = "$gte" + group = "$group" + in_ = "$in" + input = "input" + indexOfArray = "$indexOfArray" + last = "$last" + let = "let" + limit = "$limit" + literal = "$literal" + lookup = "$lookup" + lowerEqual = "$lte" + map = "$map" + match = "$match" + max = "$max" + min = "$min" + newRoot = "newRoot" + notEqual = "$ne" + notIn = "$nin" + options = "$options" + or_ = "$or" + pipeline = "pipeline" + project = "$project" + push = "$push" + regex = "$regex" + replaceRoot = "$replaceRoot" + reverseArray = "$reverseArray" + set = "$set" + skip = "$skip" + sort = "$sort" + sum = "$sum" + unwind = "$unwind" diff --git a/cli/__init__.py b/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cli/bids_importer.py b/cli/bids_importer.py new file mode 100644 index 0000000..4f8db94 --- /dev/null +++ b/cli/bids_importer.py @@ -0,0 +1,269 @@ +import io +import json +import logging +import subprocess +import sys +from pathlib import Path + +import fire +import girder_client + +from bids_plugin.utility.models import GirderModel + +logging.basicConfig(level=logging.DEBUG) +logging.getLogger("urllib3").setLevel(logging.WARNING) + +logger = logging.getLogger(__name__) + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +BIDS_COMPOUND_EXTENSIONS = { + ".nii.gz", + ".tsv.gz", +} + + +def bids_stem(filename: str) -> str: + """Return the filename without its BIDS extension.""" + name = Path(filename).name + + for ext in BIDS_COMPOUND_EXTENSIONS: + if name.endswith(ext): + return name[: -len(ext)] + + return Path(name).stem + + +def validate_bids(directory: str) -> bool: + """ + Runs the BIDS Validator on the given directory. + + :param directory: Path to the BIDS dataset directory. + :return: Boolean indicating whether the dataset is valid. + """ + try: + result = subprocess.run( + ["bids-validator-deno", "--json", directory], check=False, capture_output=True, text=True + ) + output = result.stdout + errors = result.stderr + if errors: + logger.error(f"Validation errors: {errors}") + return False + # Check if there are no errors + return '"errors": []' in output or '"severity": "error"' not in output + except FileNotFoundError: + logger.error("bids-validator not found. Make sure you installed bids-validator-deno") + return False + + +class BIDSImporter: + def __init__(self, bids_dir: str, api_url: str, api_key: str, location_id: str, location_type: str) -> None: + """ + :param bids_dir: Path to the BIDS dataset directory. + :param api_url: The API URL of the Girder instance. + :param api_key: API key for authentication. + :param location_id: The ID of the root folder in Girder where the data will be uploaded. + :param location_type: The type of the root folder: "folder" or "collection". + """ + self.bids_dir = Path(bids_dir) + self.root_folder_id = location_id + self.root_folder_type = location_type + self.dataset_name = self.bids_dir.resolve().name + + self.girder_client = girder_client.GirderClient(apiUrl=api_url) + self.girder_client.authenticate(apiKey=api_key) + + def _get_item_metadata(self, item: GirderModel) -> str: + """Extract metadata from item files""" + metadata_file = next(self.girder_client.listFile(item["_id"], limit=1)) + file_obj = io.BytesIO() + for chunk in self.girder_client.downloadFileAsIterator(metadata_file["_id"]): + if chunk: + file_obj.write(chunk) + file_obj.seek(0, 0) + return json.load(file_obj) + + def _get_associated_item(self, folder_id: str, metadata_item: GirderModel) -> GirderModel | None: + """ + Retrieves the data item associated to a metadata item. + + :param folder_id: ID of the parent folder in Girder. + :param item: The BIDS item to find an associated ID for. + :return: A tuple containing the associated ID and its type (folder or item), or None if not found. + """ + file_name = bids_stem(metadata_item["name"]) + for item in self.girder_client.listItem(folder_id): + if item["_id"] == metadata_item["_id"]: + continue + + if item["name"].startswith(file_name): + return item + return None + + def _extract_metadata(self, location_id: str, location_type: str = "folder") -> None: + """ + Extracts metadata from JSON files and adds it to Girder item's metadata. + + :param folder_id: ID of the Girder folder containing BIDS data. + """ + if location_type == "folder": + for item in self.girder_client.listItem(location_id): + if item["name"] == "dataset_description.json": + dataset_desc = self._get_item_metadata(item) + self.girder_client.addMetadataToFolder(location_id, dataset_desc) + + elif item["name"].endswith(".json"): + associated_item = self._get_associated_item(location_id, item) + if associated_item is None: + continue + + metadata = self._get_item_metadata(item) + self.girder_client.addMetadataToItem(associated_item["_id"], metadata) + + for child_folder_id in self.girder_client.listFolder(location_id, location_type): + self._extract_metadata(child_folder_id["_id"]) + + def extract_metadata(self) -> None: + self._extract_metadata(self.root_folder_id, self.root_folder_type) + + def upload_dataset(self, use_plugin: bool = False) -> None: + if len(list(self.girder_client.listFolder(self.root_folder_id, self.root_folder_type, self.dataset_name))) > 0: + raise Exception(f"A folder named {self.dataset_name} already exists in the Girder database") + + if use_plugin: + self._plugin_upload_dataset(self.bids_dir, self.root_folder_id, self.root_folder_type) + else: + self.girder_client.upload( + str(self.bids_dir), self.root_folder_id, self.root_folder_type, leafFoldersAsItems=False + ) + + def _plugin_upload_dataset(self, file_pattern: Path, parent_id: str, parent_type: str = "folder") -> None: + logger.info(f"Creating BIDS Dataset from {file_pattern.name}") + # Create dataset + dataset_desc_file = file_pattern / "dataset_description.json" + with dataset_desc_file.open("r") as f: + dataset_desc = f.read() + + dataset_folder = self.girder_client.post( + "bids_dataset", + parameters={ + "name": file_pattern.name, + "parent_id": parent_id, + "parent_type": parent_type, + "dataset_description": dataset_desc, + }, + ) + + # Parse dataset recursively + for element_path in file_pattern.iterdir(): + if element_path.is_file(): + self._plugin_upload_item(element_path, dataset_folder["_id"]) + + elif element_path.name == "derivatives": + derivative_folder = self.girder_client.loadOrCreateFolder( + element_path.name, dataset_folder["_id"], "folder" + ) + for derivative_element_path in element_path.iterdir(): + if derivative_element_path.is_dir(): + self._plugin_upload_dataset(derivative_element_path, derivative_folder["_id"]) + + else: + self._plugin_upload_folder(element_path, dataset_folder["_id"]) + + def _plugin_upload_folder(self, folder_path: Path, parent_id: str) -> None: + logger.info(f"Creating BIDS Folder from {folder_path.name}") + folder = self.girder_client.post( + "bids_folder", + parameters={ + "folder_id": parent_id, + "name": folder_path.name, + }, + ) + + for element_path in folder_path.iterdir(): + if element_path.is_dir(): + self._plugin_upload_folder(element_path, folder["_id"]) + else: + self._plugin_upload_item( + element_path, + folder["_id"], + ) + + def _plugin_upload_item(self, item_path: Path, folder_id: str) -> None: + logger.info(f"Creating BIDS Item from {item_path.name}") + item = self.girder_client.post( + "bids_item", + parameters={ + "folder_id": folder_id, + "name": item_path.name, + }, + ) + self.girder_client.uploadFileToItem(item["_id"], str(item_path)) + + def clean(self) -> None: + dataset_folder = next( + self.girder_client.listFolder(self.root_folder_id, self.root_folder_type, self.dataset_name), None + ) + if dataset_folder: + logger.info(f"Deleting {self.dataset_name}") + self.girder_client.delete( + "resource", parameters={"resources": json.dumps({"folder": [dataset_folder["_id"]]})} + ) + + +def main( + bids_dir: str, + api_url: str, + api_key: str, + location_id: str, + location_type: str = "folder", + ignore_validation: bool = False, + use_plugin: bool = False, + extract_metadata: bool = False, +) -> None: + """ + Main function for validating and uploading BIDS datasets. + + :param bids_dir: Path to the BIDS dataset directory. + :param api_url: The API URL of the Girder instance. + :param api_key: API key for authentication. + :param location_id: The ID of the root folder in Girder where the data will be uploaded. + :param location_type: The type of the root folder: "folder" or "collection". + :param ignore_validation: Whether to skip BIDS validation before upload. + :param plugin: whether to use BIDS Girder plugin + :param extract_metadata: whether to extract metadata of files to enrich items metadata + """ + if not ignore_validation: + logger.info("Validating BIDS dataset...") + if validate_bids(bids_dir): + logger.info("BIDS dataset is valid") + else: + logger.error("BIDS dataset validation failed. Aborting upload.") + sys.exit(1) + logger.info("Uploading to Girder...") + + importer = BIDSImporter(bids_dir, api_url, api_key, location_id, location_type) + + try: + logger.debug("Upload dataset") + importer.upload_dataset(use_plugin) + except girder_client.HttpError as e: + logger.error(f"Could not upload dataset: {e}") + importer.clean() + else: + if extract_metadata: + logger.debug("Extract metadata") + importer.extract_metadata() + logger.info("Successful upload of dataset") + + +def cli() -> None: + fire.Fire(main) + + +if __name__ == "__main__": + cli() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8013fd3 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,104 @@ +[project] +name = "GirderBIDS" +version = "0.1.0" +description = "Girder BIDS project containing a Girder Plugin to handle BIDS and a CLI BIDS importer" +authors = [] +dependencies = [ + "girder>=5.0.0", + "pydantic", +] + +[project.optional-dependencies] +dev = [ + "pytest", + "pytest-girder>=5.0.0", +] +cli = [ + "girder-client>=5.0.0", + "fire", + "bids-validator-deno==2.4.1; sys_platform == 'linux'", +] + +[project.entry-points."girder.plugin"] +bids_plugin = "bids_plugin:BIDSPlugin" + +[project.scripts] +bids-importer = "cli.bids_importer:cli" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.build.targets.wheel] +packages = ["bids_plugin", "cli"] + +[tool.ruff] + +[tool.ruff.lint] +extend-select = [ + "ANN", # flake8-type-hint + "ARG", # flake8-unused-arguments + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "EM", # flake8-errmsg + "EXE", # flake8-executable + "G", # flake8-logging-format + "I", # isort + "ICN", # flake8-import-conventions + "NPY", # NumPy specific rules + "PD", # pandas-vet + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PT", # flake8-pytest-style + "PTH", # flake8-use-pathlib + "RET", # flake8-return + "RUF", # Ruff-specific + "SIM", # flake8-simplify + "T20", # flake8-print + "UP", # pyupgrade + "YTT", # flake8-2020 +] +ignore = [ + "EM101", + "EM102", + "G004", # f-string in logging statement + "PLR09", # Too many <...> + "PLR2004", # Magic value used in comparison + "ISC001", # Conflicts with formatter + "ANN401", # Any + "ANN002", # *args + "ANN003", # **kwargs +] +isort.required-imports = [] + +[tool.ruff.lint.per-file-ignores] +"**/tests/*.py" = ["ARG001"] + +[tool.semantic_release] +version_toml = [ + "pyproject.toml:project.version", +] +version_variables = [ + "__init__.py:__version__", +] + +[tool.semantic_release.publish] +dist_glob_patterns = ["dist/*"] +upload_to_vcs_release = true + +[tool.semantic_release.branches.main] +match = "(main|master)" + +[tool.coverage.report] +omit = ["*/tests/*"] +exclude_lines = [ + "if TYPE_CHECKING:", + "@abstractmethod", +] + +[tool.pytest.ini_options] +asyncio_default_fixture_loop_scope = "function" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 3a88a4c..0000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -bids-validator-deno==2.4.1 -girder -girder-client -fire diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..adbbd50 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,95 @@ +from typing import Any + +import pytest +from girder.constants import AccessType +from girder.models.collection import Collection + +from bids_plugin.models import BIDSDatasetModel, BIDSFolderModel, BIDSItemModel +from bids_plugin.utility import BIDSDataset, BIDSDescription + + +def pytest_collection_modifyitems(items: Any) -> None: + for item in items: + item.add_marker(pytest.mark.plugin("bids_plugin")) + + +@pytest.fixture +def collection(user: Any) -> Any: + return Collection().createCollection("Test collection", creator=user) + + +@pytest.fixture +def folder(collection: Any, user: Any) -> Any: + return BIDSFolderModel().createFolder(collection, "Random folder", parentType="collection") + + +@pytest.fixture +def raw_dataset_description() -> Any: + return BIDSDescription(BIDSVersion="1.8.0") + + +@pytest.fixture +def derivative_dataset_description() -> Any: + return BIDSDescription(BIDSVersion="1.8.0", DatasetType="derivative") + + +@pytest.fixture +def derivative_dataset(dataset: BIDSDataset, user: Any, derivative_dataset_description: Any) -> Any: + derivatives_folder = BIDSFolderModel.load(dataset["derivatives_folder_id"], AccessType.WRITE, user) + return BIDSDatasetModel().create_bids_dataset( + user, "Derivative Dataset", parent=derivatives_folder, dataset_description=derivative_dataset_description + ) + + +@pytest.fixture +def dataset(collection: Any, user: Any, raw_dataset_description: Any) -> Any: + return BIDSDatasetModel().create_bids_dataset( + user, "Dataset 1", parent=collection, parent_type="collection", dataset_description=raw_dataset_description + ) + + +@pytest.fixture +def subject_folder(dataset: Any, user: Any) -> Any: + return BIDSFolderModel().create_bids_folder( + user, + "sub-01", + dataset, + ) + + +@pytest.fixture +def datatype_folder(subject_folder: Any, user: Any) -> Any: + return BIDSFolderModel().create_bids_folder( + user, + "anat", + subject_folder, + ) + + +@pytest.fixture +def dataset_list( + collection: Any, user: Any, raw_dataset_description: Any, derivative_dataset_description: Any +) -> list[Any]: + dataset1 = BIDSDatasetModel().create_bids_dataset( + user, "Dataset 1", collection, raw_dataset_description, "collection" + ) + dataset2 = BIDSDatasetModel().create_bids_dataset( + user, "Dataset 2", collection, raw_dataset_description, "collection" + ) + dataset3 = BIDSDatasetModel().create_bids_dataset( + user, "Dataset 3", collection, derivative_dataset_description, "collection" + ) + return [dataset1, dataset2, dataset3] + + +@pytest.fixture +def subject_folder_list(dataset: Any, user: Any) -> Any: + return [BIDSFolderModel().create_bids_folder(user, f"sub-0{i + 1}", dataset) for i in range(2)] + + +@pytest.fixture +def item_list(datatype_folder: Any, user: Any) -> Any: + return [ + BIDSItemModel().create_bids_item(user, f"sub-01_task-rest_analysis{i + 1}.nii.gz", datatype_folder) + for i in range(2) + ] diff --git a/tests/test_dataset_model.py b/tests/test_dataset_model.py new file mode 100644 index 0000000..8c32039 --- /dev/null +++ b/tests/test_dataset_model.py @@ -0,0 +1,50 @@ +from typing import Any + +import pytest +from bson.objectid import ObjectId +from girder.constants import AccessType +from girder.exceptions import ValidationException + +from bids_plugin.models import BIDSDatasetModel +from bids_plugin.utility import BIDSDescription, GirderModel + + +def test_create_dataset_without_dataset_description_raises_error( + db: Any, collection: GirderModel, user: GirderModel +) -> None: + dataset_name = "Test Dataset" + with pytest.raises(ValidationException) as exc_info: + BIDSDatasetModel().create_bids_dataset( + user, + dataset_name, + collection, + BIDSDescription(), + "collection", + ) + + assert "Invalid BIDS Dataset" in str(exc_info.value) + + saved_dataset = list(BIDSDatasetModel().find(query={"collection_id": collection["_id"], "name": dataset_name})) + + assert len(saved_dataset) == 0 + + +def test_create_dataset_with_dataset_description(db: Any, collection: GirderModel, user: GirderModel) -> None: + dataset_name = "Test Dataset" + created_dataset = BIDSDatasetModel().create_bids_dataset( + user, + dataset_name, + collection, + parent_type="collection", + dataset_description=BIDSDescription(Name=dataset_name, BIDSVersion="1.10.0"), + ) + + assert created_dataset["name"] == dataset_name + assert ObjectId(created_dataset["creatorId"]) == user["_id"] + assert created_dataset["baseParentId"] == collection["_id"] + assert created_dataset.get("dataset_description") + assert created_dataset.get("derivatives_folder_id") + + saved_dataset = BIDSDatasetModel().load(created_dataset["_id"], user=user, level=AccessType.WRITE) + + assert saved_dataset diff --git a/tests/test_dataset_resource.py b/tests/test_dataset_resource.py new file mode 100644 index 0000000..6735570 --- /dev/null +++ b/tests/test_dataset_resource.py @@ -0,0 +1,97 @@ +from typing import Any + +from bson.objectid import ObjectId +from girder.constants import AccessType +from pydantic import TypeAdapter +from pytest_girder.assertions import assertStatusOk + +from bids_plugin.models import BIDSDatasetModel +from bids_plugin.utility import BIDSDescription, GirderModel + + +def test_list_all_datasets( + db: Any, collection: GirderModel, dataset_list: list[GirderModel], server: Any, user: Any +) -> None: + resp = server.request( + method="GET", + path="/bids_dataset", + params={"collection_id": collection["_id"]}, + user=user, + ) + + assertStatusOk(resp) + + resp_dataset_list = resp.json + + assert len(resp_dataset_list) == 3 + assert any(ds["name"] == dataset_list[0]["name"] for ds in resp_dataset_list) + assert any(ds["name"] == dataset_list[1]["name"] for ds in resp_dataset_list) + assert any(ds["name"] == dataset_list[2]["name"] for ds in resp_dataset_list) + + +def test_list_raw_datasets( + db: Any, collection: GirderModel, dataset_list: list[GirderModel], server: Any, user: GirderModel +) -> None: + resp = server.request( + method="GET", + path="/bids_dataset", + params={"collection_id": collection["_id"], "is_derivative": False}, + user=user, + ) + + assertStatusOk(resp) + + resp_dataset_list = resp.json + + assert len(resp_dataset_list) == 2 + assert any(ds["name"] == dataset_list[0]["name"] for ds in resp_dataset_list) + assert any(ds["name"] == dataset_list[1]["name"] for ds in resp_dataset_list) + assert not any(ds["name"] == dataset_list[2]["name"] for ds in resp_dataset_list) + + +def test_list_derivative_datasets( + db: Any, collection: GirderModel, dataset_list: list[GirderModel], server: Any, user: GirderModel +) -> None: + resp = server.request( + method="GET", + path="/bids_dataset", + params={"collection_id": collection["_id"], "is_derivative": True}, + user=user, + ) + + assertStatusOk(resp) + + resp_dataset_list = resp.json + + assert len(resp_dataset_list) == 1 + assert not any(ds["name"] == dataset_list[0]["name"] for ds in resp_dataset_list) + assert not any(ds["name"] == dataset_list[1]["name"] for ds in resp_dataset_list) + assert any(ds["name"] == dataset_list[2]["name"] for ds in resp_dataset_list) + + +def test_create_dataset( + db: Any, collection: GirderModel, raw_dataset_description: BIDSDescription, server: Any, user: GirderModel +) -> None: + dataset_name = "Test Dataset" + raw_dataset_description.Name = dataset_name + resp = server.request( + method="POST", + path="/bids_dataset", + params={ + "name": dataset_name, + "parent_id": collection["_id"], + "parent_type": "collection", + "dataset_description": TypeAdapter(BIDSDescription).dump_json(raw_dataset_description), + }, + user=user, + ) + resp_dataset = resp.json + + assertStatusOk(resp) + assert resp_dataset["name"] == dataset_name + assert ObjectId(resp_dataset["creatorId"]) == user["_id"] + assert "dataset_description" in resp_dataset + + saved_dataset = BIDSDatasetModel().load(resp_dataset["_id"], user=user, level=AccessType.WRITE) + + assert saved_dataset diff --git a/tests/test_folder_model.py b/tests/test_folder_model.py new file mode 100644 index 0000000..8cc34b1 --- /dev/null +++ b/tests/test_folder_model.py @@ -0,0 +1,74 @@ +from typing import Any + +import pytest +from bson.objectid import ObjectId +from girder.constants import AccessType +from girder.exceptions import ValidationException + +from bids_plugin.models import BIDSFolderModel +from bids_plugin.utility import GirderModel + + +def check_bids_folder_does_not_exist(parent: GirderModel, folder_name: str) -> None: + folder_list = list(BIDSFolderModel().find(query={"parentId": parent["_id"], "name": folder_name})) + assert len(folder_list) == 0 + + +def test_create_subject_folder_in_dataset(db: Any, dataset: GirderModel, user: GirderModel) -> None: + subject_folder_name = "sub-01" + created_subject_folder = BIDSFolderModel().create_bids_folder( + user, + subject_folder_name, + dataset, + ) + + assert created_subject_folder["name"] == subject_folder_name + assert ObjectId(created_subject_folder["creatorId"]) == user["_id"] + assert ObjectId(created_subject_folder["parentId"]) == dataset["_id"] + assert ObjectId(created_subject_folder.get("dataset_id")) == dataset["_id"] + + saved_subject = BIDSFolderModel().load(created_subject_folder["_id"], user=user, level=AccessType.WRITE) + + assert saved_subject + + +def test_create_subject_in_folder_raises_error(db: Any, folder: GirderModel, user: GirderModel) -> None: + subject_folder_name = "sub-01" + with pytest.raises(ValidationException) as exc_info: + BIDSFolderModel().create_bids_folder( + user, + subject_folder_name, + folder, + ) + + assert "Invalid BIDS Folder" in str(exc_info.value) + + check_bids_folder_does_not_exist(folder, subject_folder_name) + + +def test_create_subject_with_wrong_name_raises_error(db: Any, dataset: GirderModel, user: GirderModel) -> None: + subject_folder_name = "subject1" + with pytest.raises(ValidationException) as exc_info: + BIDSFolderModel().create_bids_folder( + user, + subject_folder_name, + dataset, + ) + + assert "Invalid BIDS Folder name" in str(exc_info.value) + + check_bids_folder_does_not_exist(dataset, subject_folder_name) + + +def test_create_session_in_dataset_raises_error(db: Any, dataset: GirderModel, user: GirderModel) -> None: + session_name = "ses-01" + with pytest.raises(ValidationException) as exc_info: + BIDSFolderModel().create_bids_folder( + user, + session_name, + dataset, + ) + + assert "Invalid BIDS Hierarchy" in str(exc_info.value) + + check_bids_folder_does_not_exist(dataset, session_name) diff --git a/tests/test_folder_resource.py b/tests/test_folder_resource.py new file mode 100644 index 0000000..141dbdd --- /dev/null +++ b/tests/test_folder_resource.py @@ -0,0 +1,51 @@ +from typing import Any + +from bson.objectid import ObjectId +from girder.constants import AccessType +from pytest_girder.assertions import assertStatusOk + +from bids_plugin.models import BIDSFolderModel +from bids_plugin.utility import GirderModel + + +def test_list_folders( + db: Any, dataset: GirderModel, subject_folder_list: list[GirderModel], server: Any, user: Any +) -> None: + resp = server.request( + method="GET", + path="/bids_folder", + params={"dataset_id": dataset["_id"]}, + user=user, + ) + + assertStatusOk(resp) + + resp_subject_folder_list = resp.json + + assert len(resp_subject_folder_list) == 2 + assert any(ds["name"] == subject_folder_list[0]["name"] for ds in resp_subject_folder_list) + assert any(ds["name"] == subject_folder_list[1]["name"] for ds in resp_subject_folder_list) + + +def test_create_subject_folder(db: Any, dataset: GirderModel, server: Any, user: GirderModel) -> None: + subject_folder_name = "sub-01" + resp = server.request( + method="POST", + path="/bids_folder", + params={ + "folder_id": dataset["_id"], + "name": subject_folder_name, + }, + user=user, + ) + resp_subject_folder = resp.json + + assertStatusOk(resp) + assert resp_subject_folder["name"] == subject_folder_name + assert ObjectId(resp_subject_folder["creatorId"]) == user["_id"] + assert ObjectId(resp_subject_folder["parentId"]) == dataset["_id"] + assert ObjectId(resp_subject_folder.get("dataset_id")) == dataset["_id"] + + saved_subject_folder = BIDSFolderModel().load(resp_subject_folder["_id"], user=user, level=AccessType.WRITE) + + assert saved_subject_folder diff --git a/tests/test_item_model.py b/tests/test_item_model.py new file mode 100644 index 0000000..c088791 --- /dev/null +++ b/tests/test_item_model.py @@ -0,0 +1,101 @@ +from typing import Any + +import pytest +from bson.objectid import ObjectId +from girder.constants import AccessType +from girder.exceptions import ValidationException + +from bids_plugin.models import BIDSItemModel +from bids_plugin.utility import GirderModel + + +def check_bids_item_does_not_exist(folder: GirderModel, item_name: str) -> None: + item_list = list(BIDSItemModel().find(query={"name": item_name, "folderId": folder["_id"]})) + assert len(item_list) == 0 + + +def test_create_item(db: Any, dataset: GirderModel, datatype_folder: GirderModel, user: GirderModel) -> None: + item_name = "sub-01_task-rest_analysis.nii.gz" + created_item = BIDSItemModel().create_bids_item( + user, + item_name, + datatype_folder, + ) + + assert created_item["name"] == item_name + assert ObjectId(created_item["creatorId"]) == user["_id"] + assert ObjectId(created_item["folderId"]) == datatype_folder["_id"] + assert ObjectId(created_item.get("dataset_id")) == dataset["_id"] + assert "source_id" in created_item + assert created_item["suffix"] == "analysis" + assert created_item["extension"] == "nii.gz" + + saved_item = BIDSItemModel().load(created_item["_id"], user=user, level=AccessType.WRITE) + + assert saved_item + + +def test_create_item_in_folder_raises_error(db: Any, folder: GirderModel, user: GirderModel) -> None: + item_name = "sub-01_task-rest_analysis.nii.gz" + with pytest.raises(ValidationException) as exc_info: + BIDSItemModel().create_bids_item( + user, + item_name, + folder, + ) + + assert "Invalid BIDS Folder" in str(exc_info.value) + + check_bids_item_does_not_exist(folder, item_name) + + +def test_create_metadata_item_in_dataset(db: Any, dataset: GirderModel, user: GirderModel) -> None: + item_name = "sub-01_task-rest_analysis.json" + created_item = BIDSItemModel().create_bids_item( + user, + item_name, + dataset, + ) + + assert created_item["name"] == item_name + assert ObjectId(created_item["creatorId"]) == user["_id"] + assert ObjectId(created_item["folderId"]) == dataset["_id"] + assert ObjectId(created_item.get("dataset_id")) == dataset["_id"] + assert "source_id" in created_item + assert created_item["suffix"] == "analysis" + assert created_item["extension"] == "json" + + saved_item = BIDSItemModel().load(created_item["_id"], user=user, level=AccessType.WRITE) + + assert saved_item + + +def test_create_data_item_in_dataset_raises_error(db: Any, dataset: GirderModel, user: GirderModel) -> None: + item_name = "sub-01_task-rest_analysis.nii.gz" + with pytest.raises(ValidationException) as exc_info: + BIDSItemModel().create_bids_item( + user, + item_name, + dataset, + dataset, + ) + + assert "Invalid BIDS Hierarchy" in str(exc_info.value) + + check_bids_item_does_not_exist(dataset, item_name) + + +def test_create_item_with_wrong_name_raises_error( + db: Any, dataset: GirderModel, datatype_folder: GirderModel, user: GirderModel +) -> None: + item_name = "analysis.nii.gz" + with pytest.raises(ValidationException) as exc_info: + BIDSItemModel().create_bids_item( + user, + item_name, + datatype_folder, + ) + + assert "Invalid BIDS Item name" in str(exc_info.value) + + check_bids_item_does_not_exist(datatype_folder, item_name) diff --git a/tests/test_item_resource.py b/tests/test_item_resource.py new file mode 100644 index 0000000..665e993 --- /dev/null +++ b/tests/test_item_resource.py @@ -0,0 +1,73 @@ +from typing import Any + +from bson.objectid import ObjectId +from girder.constants import AccessType +from pytest_girder.assertions import assertStatusOk + +from bids_plugin.models import BIDSItemModel +from bids_plugin.utility import GirderModel + + +def test_list_items(db: Any, dataset: GirderModel, item_list: list[GirderModel], server: Any, user: Any) -> None: + resp = server.request( + method="GET", + path="/bids_item", + params={"dataset_id": dataset["_id"]}, + user=user, + ) + + assertStatusOk(resp) + + resp_item_list = resp.json + + assert len(resp_item_list) == 2 + assert any(it["name"] == item_list[0]["name"] for it in resp_item_list) + assert any(it["name"] == item_list[1]["name"] for it in resp_item_list) + + +def test_list_items_matches_suffix( + db: Any, dataset: GirderModel, item_list: list[GirderModel], server: Any, user: Any +) -> None: + resp = server.request( + method="GET", + path="/bids_item", + params={"dataset_id": dataset["_id"], "suffix": "analysis1"}, + user=user, + ) + + assertStatusOk(resp) + + resp_item_list = resp.json + + assert len(resp_item_list) == 1 + assert any(it["name"] == item_list[0]["name"] for it in resp_item_list) + assert resp_item_list[0]["suffix"] == "analysis1" + + +def test_create_item( + db: Any, dataset: GirderModel, datatype_folder: GirderModel, server: Any, user: GirderModel +) -> None: + item_name = "sub-01_task-rest_analysis.nii.gz" + resp = server.request( + method="POST", + path="/bids_item", + params={ + "folder_id": datatype_folder["_id"], + "name": item_name, + }, + user=user, + ) + resp_item = resp.json + + assertStatusOk(resp) + assert resp_item["name"] == item_name + assert ObjectId(resp_item["creatorId"]) == user["_id"] + assert ObjectId(resp_item["folderId"]) == datatype_folder["_id"] + assert ObjectId(resp_item.get("dataset_id")) == dataset["_id"] + assert "source_id" in resp_item + assert resp_item["suffix"] == "analysis" + assert resp_item["extension"] == "nii.gz" + + saved_item = BIDSItemModel().load(resp_item["_id"], user=user, level=AccessType.WRITE) + + assert saved_item diff --git a/tools/bids-importer.py b/tools/bids-importer.py deleted file mode 100644 index a8f12e4..0000000 --- a/tools/bids-importer.py +++ /dev/null @@ -1,260 +0,0 @@ -""" -BIDS Importer Script - -This script validates and uploads BIDS (Brain Imaging Data Structure) datasets to a Girder instance. -It ensures proper validation using the BIDS Validator before uploading and preserves the folder -hierarchy while adding metadata. - -Functions: - - validate_bids(directory): Validates the BIDS dataset using bids-validator. - - get_or_create_folder(gc, parent_id, folder_name): Retrieves or creates a folder in Girder. - - delete_folder_contents(gc, folder_id): Deletes all items and subfolders within a given folder. - - get_file_size(f): Retrieves the size of a file in bytes. - - get_file_metadata(f): Extracts metadata from a JSON file. - - get_file_path_metadata(file_path): Reads a JSON file and returns its contents as a dictionary. - - is_bids_item(item): Determines whether a file is a BIDS metadata file. - - get_associated_id(gc, parent_id, bids_item): Retrieves the associated ID for a BIDS item. - - extract_bids_metadata(gc, folder_id, recursive): Extracts and assigns metadata to items in Girder. - - upload_to_girder(api_url, api_key, root_folder_id, bids_root, import_mode): Uploads BIDS data to Girder. - - main(bids_dir, girder_api_url, girder_api_key, girder_folder_id, import_mode, ignore_validation): - Main function for validating and uploading BIDS data. - -Classes: - - ImportMode: Enum class defining different import modes. - -:param bids_dir: Path to the BIDS dataset directory. -:param girder_api_url: The API URL of the Girder instance. -:param girder_api_key: API key for authentication with Girder. -:param girder_folder_id: The ID of the root folder in Girder where the data will be uploaded. -:param import_mode: The mode for handling existing data in Girder (default: OVERWRITE_ON_SAME_NAME). -:param ignore_validation: Whether to skip BIDS validation before upload (default: False). - -Usage: - python bids-importer.py --bids_dir --girder_api_url --girder_api_key --girder_folder_id - --import_mode --ignore_validation -""" - -from enum import Enum -import fire -import girder_client -import io -import json -import os -# from bids_validator import BIDSValidator -import subprocess -import sys - -import logging - -logging.basicConfig(level=logging.DEBUG) -logging.getLogger('urllib3').setLevel(logging.WARNING) - -logger = logging.getLogger() -# logger.setLevel(logging.DEBUG) - - -def validate_bids(directory): - """ - Runs the BIDS Validator on the given directory. - - :param directory: Path to the BIDS dataset directory. - :return: Boolean indicating whether the dataset is valid. - """ - try: - result = subprocess.run(['bids-validator-deno', '--json', directory], - capture_output=True, text=True) - output = result.stdout - errors = result.stderr - if errors: - logger.error(f"Validation errors: {errors}") - return False - # Check if there are no errors - return ('"errors": []' in output or - '"severity": "error"' not in output) - except FileNotFoundError: - logger.error("bids-validator not found. Make sure you installed bids-validator-deno") - return False - - -def get_or_create_folder(gc, parent_id, folder_name): - """ - Retrieves or creates a folder in Girder. - - :param gc: Girder client instance. - :param parent_id: ID of the parent folder. - :param folder_name: Name of the folder to retrieve or create. - :return: Folder ID. - """ - existing_folders = list(gc.listFolder(parent_id, name=folder_name)) - if existing_folders: - return existing_folders[0]['_id'] - new_folder = gc.createFolder(parent_id, folder_name, parentType='folder') - return new_folder['_id'] - - -def delete_folder_contents(gc, folder_id): - """ - Deletes all items and subfolders within a given folder. - - :param gc: Girder client instance. - :param folder_id: ID of the folder to clear. - """ - # Delete all items in the folder - for item in gc.listItem(folder_id): - gc.delete(f"/item/{item['_id']}") # Delete each item - - # Delete all subfolders - for folder in gc.listFolder(folder_id): - delete_folder_contents(gc, folder["_id"]) # Recursively delete contents - gc.delete(f"/folder/{folder['_id']}") # Delete the folder itself - - -def get_file_size(f): - """ - Retrieves the size of a file in bytes. - - :param f: File object. - :return: Size of the file in bytes. - """ - f.seek(0, 2) # Move to the end of the file - file_size = f.tell() # Get the position (size in bytes) - f.seek(0, 0) # Move back to the beginning of the file - return file_size - - -def get_file_metadata(f): - """Convert a file object that contains a JSON file into a dictionnary""" - f.seek(0, 0) - return json.load(f) - - -def get_file_path_metadata(file_path): - """ - Reads a JSON file and returns its contents as a dictionary. - - :param file_path: Path to the JSON file. - :return: Dictionary containing the file's metadata. - """ - with open(file_path, 'rb') as f: - return get_file_metadata(f) - - -def is_bids_item(item): - """ - Determines whether a given file is a BIDS metadata file. - - :param item: The item (file) to check. - :return: True if the item is a BIDS metadata file, False otherwise. - """ - return item['name'].endswith('.json') - - -def get_associated_id(gc, parent_id, bids_item): - """ - Retrieves the associated ID for a BIDS item. - - :param gc: Girder client instance. - :param parent_id: ID of the parent folder in Girder. - :param bids_item: The BIDS item to find an associated ID for. - :return: A tuple containing the associated ID and its type (folder or item), or None if not found. - """ - file_name = bids_item['name'] - if file_name == 'dataset_description.json': - return parent_id, 'folder' - file_base, extension = os.path.splitext(file_name) - for item in gc.listItem(parent_id): - if item['name'].startswith(file_base): - return item['_id'], 'item' - return None - - -class ImportMode(Enum): - """ - Enum class defining different import modes for handling existing data in Girder. - - :param RESET_DATABASE: Deletes all existing data before uploading new data. - :param OVERWRITE_ON_SAME_NAME: Overwrites existing datasets with the same name. - """ - RESET_DATABASE = 'RESET_DATABASE' - OVERWRITE_ON_SAME_NAME = 'OVERWRITE_ON_SAME_NAME' - - -def extract_bids_metadata(gc, folder_id, recursive=True): - """ - Extracts metadata from BIDS-related JSON files and adds it to Girder. - - :param gc: Girder client instance. - :param folder_id: ID of the Girder folder containing BIDS data. - :param recursive: Whether to process subfolders recursively (default: True). - """ - for item in gc.listItem(folder_id): - if is_bids_item(item): - associated_id, type = get_associated_id(gc, folder_id, item) - bids_file = next(gc.listFile(item['_id'], limit=1)) - file_obj = io.BytesIO() - for chunk in gc.downloadFileAsIterator(bids_file['_id']): - if chunk: - file_obj.write(chunk) - metadata = get_file_metadata(file_obj) - if type == 'item': - gc.addMetadataToItem(associated_id, metadata) - elif type == 'folder': - gc.addMetadataToFolder(associated_id, metadata) - - if recursive: - for child_folder_id in gc.listFolder(folder_id): - extract_bids_metadata(gc, child_folder_id['_id']) - - -def upload_to_girder(api_url, api_key, root_folder_id, bids_root, - import_mode): - """ - Uploads valid BIDS files to Girder, preserving folder hierarchy. - - :param api_url: Girder API URL. - :param api_key: API key for authentication. - :param root_folder_id: ID of the root folder in Girder where files will be uploaded. - :param bids_root: Root directory of the BIDS dataset. - :param import_mode: Mode for handling existing data. - """ - gc = girder_client.GirderClient(apiUrl=api_url) - gc.authenticate(apiKey=api_key) - - if import_mode == ImportMode.RESET_DATABASE.name: - logger.info(f"Deleting folder {root_folder_id}") - delete_folder_contents(gc, root_folder_id) - - gc.upload(bids_root, root_folder_id, 'folder', leafFoldersAsItems=False, - reuseExisting=True) - extract_bids_metadata(gc, root_folder_id) - - logger.info("Upload complete!") - - -def main(bids_dir, girder_api_url, girder_api_key, girder_folder_id, - import_mode: ImportMode = ImportMode.OVERWRITE_ON_SAME_NAME, - ignore_validation: bool = False): - """ - Main function for validating and uploading BIDS datasets. - - :param bids_dir: Path to the BIDS dataset directory. - :param girder_api_url: The API URL of the Girder instance. - :param girder_api_key: API key for authentication. - :param girder_folder_id: The ID of the root folder in Girder where the data will be uploaded. - :param import_mode: The mode for handling existing data in Girder. - :param ignore_validation: Whether to skip BIDS validation before upload. - """ - if not ignore_validation: - logger.info("Validating BIDS dataset...") - if validate_bids(bids_dir): - logger.info("BIDS dataset is valid") - else: - logger.error("BIDS dataset validation failed. Aborting upload.") - sys.exit(1) - logger.info("Uploading to Girder...") - upload_to_girder(girder_api_url, girder_api_key, girder_folder_id, - bids_dir, import_mode) - - -if __name__ == "__main__": - fire.Fire(main)