diff --git a/.copier-answers.resonant.yml b/.copier-answers.resonant.yml index d4d08164a..e12dbcc2a 100644 --- a/.copier-answers.resonant.yml +++ b/.copier-answers.resonant.yml @@ -1,4 +1,4 @@ -_commit: v0.41.0 +_commit: v0.47.1 _src_path: gh:kitware-resonant/cookiecutter-resonant core_app_name: api include_example_code: false @@ -6,3 +6,4 @@ project_name: DANDI Archive project_slug: dandiapi python_package_name: dandiapi site_domain: api.dandiarchive.org +use_asgi: false diff --git a/.editorconfig b/.editorconfig index a0cb1cc0e..b6494e09d 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,21 +7,28 @@ insert_final_newline = true trim_trailing_whitespace = true charset = utf-8 -[*.py] -indent_size = 4 -max_line_length = 100 +[*.{css,pcss}] +indent_size = 2 [*.html] indent_size = 2 -[*.css] +[*.ini] +indent_size = 4 + +[*.{js,ts,vue}] indent_size = 2 +max_line_length = 100 -[{*.yml,*.yaml}] +[*.{json,jsonc,json5}] indent_size = 2 -[*.ini] +[*.py] indent_size = 4 +max_line_length = 100 + +[*.toml] +indent_size = 2 -[*.sh] +[*.{yml,yaml}] indent_size = 2 diff --git a/.github/workflows/cli-integration.yml b/.github/workflows/cli-integration.yml index a64487547..40bf1d635 100644 --- a/.github/workflows/cli-integration.yml +++ b/.github/workflows/cli-integration.yml @@ -68,9 +68,7 @@ jobs: if: matrix.dandi-version == 'release' run: > uvx --with dandi[test] - pytest --pyargs -v --dandi-api --deselect=tests/test_keyring.py::test_dandi_authenticate_no_env_var dandi - # TODO: Revert back to - # pytest --pyargs -v --dandi-api dandi + pytest --pyargs -v -s --dandi-api dandi env: DANDI_TESTS_PERSIST_DOCKER_COMPOSE: "1" @@ -78,9 +76,7 @@ jobs: if: matrix.dandi-version == 'prod' run: > uvx --with "dandi[test] @ git+https://github.com/dandi/dandi-cli" - pytest --pyargs -v --dandi-api --deselect=tests/test_keyring.py::test_dandi_authenticate_no_env_var dandi - # TODO: Revert back to - # pytest --pyargs -v --dandi-api dandi + pytest --pyargs -v -s --dandi-api dandi env: DANDI_TESTS_PERSIST_DOCKER_COMPOSE: "1" diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d0263b59..86d4dcd9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,101 @@ +# v0.21.8 (Fri Mar 06 2026) + +#### ๐Ÿ› Bug Fix + +- Upgrade to Resonant v0.47 [#2692](https://github.com/dandi/dandi-archive/pull/2692) ([@brianhelba](https://github.com/brianhelba)) + +#### Authors: 1 + +- Brian Helba ([@brianhelba](https://github.com/brianhelba)) + +--- + +# v0.21.7 (Fri Mar 06 2026) + +#### ๐Ÿ› Bug Fix + +- Add scrollbar to meditor component for overflowing content [#2724](https://github.com/dandi/dandi-archive/pull/2724) ([@kabilar](https://github.com/kabilar)) +- Don't return `null` names or usernames from user search endpoint [#2681](https://github.com/dandi/dandi-archive/pull/2681) ([@jjnesbitt](https://github.com/jjnesbitt)) + +#### Authors: 2 + +- Jacob Nesbitt ([@jjnesbitt](https://github.com/jjnesbitt)) +- Kabilar Gunalan ([@kabilar](https://github.com/kabilar)) + +--- + +# v0.21.6 (Wed Feb 18 2026) + +#### ๐Ÿ› Bug Fix + +- Properly unembargo blobs when asset is updated [#2713](https://github.com/dandi/dandi-archive/pull/2713) ([@jjnesbitt](https://github.com/jjnesbitt)) + +#### Authors: 1 + +- Jacob Nesbitt ([@jjnesbitt](https://github.com/jjnesbitt)) + +--- + +# v0.21.5 (Tue Feb 17 2026) + +:tada: This release contains work from a new contributor! :tada: + +Thank you, Heberto Mayorquin ([@h-mayorquin](https://github.com/h-mayorquin)), for all your work! + +#### ๐Ÿ› Bug Fix + +- Fix propagation of embargoed date from dandisets to assets [#2698](https://github.com/dandi/dandi-archive/pull/2698) ([@h-mayorquin](https://github.com/h-mayorquin) [@jjnesbitt](https://github.com/jjnesbitt)) +- bf: fix GarbageCollectionEvent.__str__ referencing nonexistent field [#2710](https://github.com/dandi/dandi-archive/pull/2710) ([@yarikoptic](https://github.com/yarikoptic) [@jjnesbitt](https://github.com/jjnesbitt)) +- Add "How to Cite" Tab to Dandiset Landing Page [#2671](https://github.com/dandi/dandi-archive/pull/2671) ([@bendichter](https://github.com/bendichter) [@jjnesbitt](https://github.com/jjnesbitt)) + +#### ๐Ÿ“ Documentation + +- Fixes of minor inconsistencies in docs claude identified [#2709](https://github.com/dandi/dandi-archive/pull/2709) ([@yarikoptic](https://github.com/yarikoptic)) + +#### ๐Ÿงช Tests + +- Revert "bf(test): test_nwb2asset_remote_asset which might be stalling here" [#2708](https://github.com/dandi/dandi-archive/pull/2708) ([@yarikoptic](https://github.com/yarikoptic)) +- ci(tests): pass -s to pytest and skip hanging test_nwb2asset_remote_asset [#2662](https://github.com/dandi/dandi-archive/pull/2662) ([@yarikoptic](https://github.com/yarikoptic)) + +#### Authors: 4 + +- Ben Dichter ([@bendichter](https://github.com/bendichter)) +- Heberto Mayorquin ([@h-mayorquin](https://github.com/h-mayorquin)) +- Jacob Nesbitt ([@jjnesbitt](https://github.com/jjnesbitt)) +- Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic)) + +--- + +# v0.21.4 (Thu Feb 05 2026) + +#### ๐Ÿ› Bug Fix + +- Update copyright year in footer [#2701](https://github.com/dandi/dandi-archive/pull/2701) ([@kabilar](https://github.com/kabilar)) + +#### Authors: 1 + +- Kabilar Gunalan ([@kabilar](https://github.com/kabilar)) + +--- + +# v0.21.3 (Mon Feb 02 2026) + +:tada: This release contains work from a new contributor! :tada: + +Thank you, William Allen ([@williamjallen](https://github.com/williamjallen)), for all your work! + +#### ๐Ÿ› Bug Fix + +- Update URL for user support [#2688](https://github.com/dandi/dandi-archive/pull/2688) ([@kabilar](https://github.com/kabilar)) +- Fix dandiset ID parsing during upload initialization [#2696](https://github.com/dandi/dandi-archive/pull/2696) ([@williamjallen](https://github.com/williamjallen)) + +#### Authors: 2 + +- Kabilar Gunalan ([@kabilar](https://github.com/kabilar)) +- William Allen ([@williamjallen](https://github.com/williamjallen)) + +--- + # v0.21.2 (Tue Jan 13 2026) #### ๐Ÿ› Bug Fix diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 65349f538..eced9615e 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -147,7 +147,7 @@ For frequent deployment administration tasks, `django-extensions` provides a con ### create_dev_dandiset ``` -python manage.py create_dev_dandiset --owner $(git config user.email) --name My Dummy Dandiset +./manage.py create_dev_dandiset --owner $(git config user.email) --name My Dummy Dandiset ``` This creates a dummy dandiset with valid metadata and a single dummy asset. diff --git a/dandiapi/api/management/__init__.py b/dandiapi/api/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dandiapi/api/management/commands/__init__.py b/dandiapi/api/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dandiapi/api/management/commands/cleanup_blobs.py b/dandiapi/api/management/commands/cleanup_blobs.py index 4051616f8..c573aa0d9 100644 --- a/dandiapi/api/management/commands/cleanup_blobs.py +++ b/dandiapi/api/management/commands/cleanup_blobs.py @@ -1,35 +1,28 @@ from __future__ import annotations -from django.conf import settings +from django.core.files.storage import default_storage import djclick as click -from storages.backends.s3 import S3Storage from dandiapi.api.models.upload import AssetBlob -BUCKET = settings.DANDI_DANDISETS_BUCKET_NAME - - -def s3_client(): - storage = S3Storage(bucket_name=BUCKET) - return storage.connection.meta.client - @click.command() @click.option('--delete', is_flag=True, default=False) def cleanup_blobs(*, delete: bool): - client = s3_client() + client = default_storage.s3_client + bucket_name = default_storage.bucket_name # Ignore pagination for now, hopefully there aren't enough objects to matter - objs = client.list_object_versions(Bucket=BUCKET, Prefix='dev/') + objs = client.list_object_versions(Bucket=bucket_name, Prefix='dev/') for version in objs['Versions']: if not AssetBlob.objects.filter(etag=version['ETag'][1:-1]).exists(): click.echo(f'Deleting version {version["Key"]}') if delete: client.delete_object( - Bucket=BUCKET, Key=version['Key'], VersionId=version['VersionId'] + Bucket=bucket_name, Key=version['Key'], VersionId=version['VersionId'] ) for delete_marker in objs['DeleteMarkers']: click.echo(f'Deleting delete marker {delete_marker["Key"]}') if delete: client.delete_object( - Bucket=BUCKET, Key=delete_marker['Key'], VersionId=delete_marker['VersionId'] + Bucket=bucket_name, Key=delete_marker['Key'], VersionId=delete_marker['VersionId'] ) diff --git a/dandiapi/api/management/commands/createsuperuser.py b/dandiapi/api/management/commands/createsuperuser.py index 2bf5d644b..31320013c 100644 --- a/dandiapi/api/management/commands/createsuperuser.py +++ b/dandiapi/api/management/commands/createsuperuser.py @@ -3,10 +3,11 @@ from typing import TYPE_CHECKING from django.db.models.signals import post_init, post_save -from resonant_settings.allauth_support.management.commands import createsuperuser from dandiapi.api.models.user import UserMetadata +from resonant_settings.allauth_support.management.commands import createsuperuser + if TYPE_CHECKING: from resonant_settings.allauth_support.createsuperuser import EmailAsUsernameProxyUser diff --git a/dandiapi/api/migrations/0001_default_site.py b/dandiapi/api/migrations/0001_default_site.py index f5da225e1..19679e0fb 100644 --- a/dandiapi/api/migrations/0001_default_site.py +++ b/dandiapi/api/migrations/0001_default_site.py @@ -10,7 +10,7 @@ from django.db.migrations.state import StateApps -def update_default_site(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor): +def update_default_site(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None: Site = apps.get_model('sites', 'Site') # A default site object may or may not exist. @@ -27,7 +27,7 @@ def update_default_site(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor ) -def rollback_default_site(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor): +def rollback_default_site(apps: StateApps, schema_editor: BaseDatabaseSchemaEditor) -> None: Site = apps.get_model('sites', 'Site') # This is the initial value of the default site object, as populated by the sites app. @@ -42,5 +42,5 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(update_default_site, rollback_default_site), + migrations.RunPython(update_default_site, rollback_default_site, elidable=False), ] diff --git a/dandiapi/api/models/asset.py b/dandiapi/api/models/asset.py index ef3756f94..d8c2c3646 100644 --- a/dandiapi/api/models/asset.py +++ b/dandiapi/api/models/asset.py @@ -13,14 +13,21 @@ from django.core.exceptions import ValidationError from django.core.validators import RegexValidator from django.db import models -from django.db.models import Q +from django.db.models import CharField, DateField, Min, Q +from django.db.models.functions import Cast from django.urls import reverse from django_extensions.db.models import TimeStampedModel +from dandiapi.api.models.dandiset import Dandiset from dandiapi.api.models.metadata import PublishableMetadataMixin from .version import Version + +class EmbargoedAssetWithinOpenDandisetError(Exception): + """Raised when an embargoed asset exists in an open dandiset.""" + + ASSET_CHARS_REGEX = r'[A-z0-9(),&\s#+~_=-]' ASSET_PATH_REGEX = rf'^({ASSET_CHARS_REGEX}?\/?\.?{ASSET_CHARS_REGEX})+$' ASSET_COMPUTED_FIELDS = [ @@ -236,23 +243,72 @@ def is_different_from( def dandi_asset_id(asset_id: str | uuid.UUID): return f'dandiasset:{asset_id}' + def access_metadata(self): + # Default to open access + embargoed = self.is_embargoed + access = { + 'schemaKey': 'AccessRequirements', + 'status': AccessType.EmbargoedAccess.value + if embargoed + else AccessType.OpenAccess.value, + } + + # Nothing else to do if not embargoed + if not embargoed: + return access + + # For zarr assets, is_embargoed is based on the dandiset directly, and a zarr can + # only be associated with one dandiset, so we know this dandiset is embargoed + if self.zarr is not None: + draft_version = self.zarr.dandiset.draft_version + + # EmbargoedUntil isn't guaranteed to be set + embargo_end_date: str | None = draft_version.metadata['access'][0].get('embargoedUntil') + if embargo_end_date is not None: + access['embargoedUntil'] = embargo_end_date + + return access + + # In the blob case, we need to consider all dandisets this blob might be associated with, + # and take the minimum embargo end date + + # This blob should only be associated with embargoed dandisets + open_dandisets = self.blob.assets.filter( + versions__dandiset__embargo_status=Dandiset.EmbargoStatus.OPEN + ) + if open_dandisets.exists(): + raise EmbargoedAssetWithinOpenDandisetError( + 'Embargoed asset contained within OPEN dandiset' + ) + + # Retrieve the minimum embargo end date, across all dandisets + embargo_end_date: datetime.date | None = ( + self.blob.assets.filter(versions__isnull=False) + .annotate( + embargo_end_date=Cast( + Cast('versions__metadata__access__0__embargoedUntil', output_field=CharField()), + output_field=DateField(), + ) + ) + .aggregate(min_embargo_end_date=Min('embargo_end_date'))['min_embargo_end_date'] + ) + + if embargo_end_date is not None: + access['embargoedUntil'] = embargo_end_date.isoformat() + + return access + @property def full_metadata(self): download_url = settings.DANDI_API_URL + reverse( 'asset-download', kwargs={'asset_id': str(self.asset_id)}, ) + metadata = { **self.metadata, 'id': self.dandi_asset_id(self.asset_id), - 'access': [ - { - 'schemaKey': 'AccessRequirements', - 'status': AccessType.EmbargoedAccess.value - if self.is_embargoed - else AccessType.OpenAccess.value, - } - ], + 'access': [self.access_metadata()], 'path': self.path, 'identifier': str(self.asset_id), 'contentUrl': [download_url, self.s3_url], diff --git a/dandiapi/api/models/garbage_collection.py b/dandiapi/api/models/garbage_collection.py index 2583ddc2d..945a778f2 100644 --- a/dandiapi/api/models/garbage_collection.py +++ b/dandiapi/api/models/garbage_collection.py @@ -10,7 +10,7 @@ class GarbageCollectionEvent(models.Model): ) def __str__(self) -> str: - return f'{self.type} ({self.created})' + return f'{self.type} ({self.timestamp})' class GarbageCollectionEventRecord(models.Model): diff --git a/dandiapi/api/services/asset/__init__.py b/dandiapi/api/services/asset/__init__.py index b50dfa6c1..a597e626d 100644 --- a/dandiapi/api/services/asset/__init__.py +++ b/dandiapi/api/services/asset/__init__.py @@ -46,6 +46,9 @@ def _create_asset( return asset +# NOTE: There is intentionally no transaction.atomic decorator on this function, as it's meant +# to be called from the high level service functions within this file, which do wrap this in a +# transaction. In production code, this function should never be called outside of a transaction. def _add_asset_to_version( *, version: Version, @@ -53,6 +56,22 @@ def _add_asset_to_version( zarr_archive: ZarrArchive | None, metadata: dict, ) -> Asset: + # Creating an asset in an OPEN dandiset that points to an + # embargoed blob results in that blob being unembargoed. + # NOTE: This only applies to asset blobs, as zarrs cannot belong to + # multiple dandisets at once. + if ( + asset_blob is not None + and asset_blob.embargoed + and version.dandiset.embargo_status == Dandiset.EmbargoStatus.OPEN + ): + asset_blob.embargoed = False + asset_blob.save() + + transaction.on_commit( + lambda: remove_asset_blob_embargoed_tag_task.delay(blob_id=asset_blob.blob_id) + ) + path = metadata['path'] asset = _create_asset( path=path, asset_blob=asset_blob, zarr_archive=zarr_archive, metadata=metadata @@ -169,20 +188,6 @@ def add_asset_to_version( raise ZarrArchiveBelongsToDifferentDandisetError with transaction.atomic(): - # Creating an asset in an OPEN dandiset that points to an - # embargoed blob results in that blob being unembargoed. - # NOTE: This only applies to asset blobs, as zarrs cannot belong to - # multiple dandisets at once. - if ( - asset_blob is not None - and asset_blob.embargoed - and version.dandiset.embargo_status == Dandiset.EmbargoStatus.OPEN - ): - AssetBlob.objects.filter(blob_id=asset_blob.blob_id).update(embargoed=False) - transaction.on_commit( - lambda: remove_asset_blob_embargoed_tag_task.delay(blob_id=asset_blob.blob_id) - ) - asset = _add_asset_to_version( version=version, asset_blob=asset_blob, diff --git a/dandiapi/api/services/embargo/utils.py b/dandiapi/api/services/embargo/utils.py index 64c3117d8..dda5c09de 100644 --- a/dandiapi/api/services/embargo/utils.py +++ b/dandiapi/api/services/embargo/utils.py @@ -4,7 +4,6 @@ import logging from typing import TYPE_CHECKING -from django.conf import settings from django.core.files.storage import default_storage from django.db.models import Q from more_itertools import chunked @@ -32,7 +31,7 @@ def _delete_object_tags(blob: str): def _delete_zarr_object_tags(zarr: str): paginator = ZarrArchive.storage.s3_client.get_paginator('list_objects_v2') pages = paginator.paginate( - Bucket=settings.DANDI_DANDISETS_BUCKET_NAME, Prefix=zarr_s3_path(zarr_id=zarr) + Bucket=default_storage.bucket_name, Prefix=zarr_s3_path(zarr_id=zarr) ) # Constant low thread number to limit memory usage, as each thread diff --git a/dandiapi/api/templates/api/mail/approved_user_message.txt b/dandiapi/api/templates/api/mail/approved_user_message.txt index cdd8caff4..a414b2271 100644 --- a/dandiapi/api/templates/api/mail/approved_user_message.txt +++ b/dandiapi/api/templates/api/mail/approved_user_message.txt @@ -6,8 +6,7 @@ Your EMBER-DANDI account has been approved. You can go to {{ dandi_web_app_url } Please use the following links to post any questions or issues. DANDI Docs: https://docs.dandiarchive.org -Discussions: https://github.com/dandi/helpdesk/discussions -Issues: https://github.com/dandi/helpdesk/issues/new/choose +Support: https://docs.dandiarchive.org/support/ YouTube: https://www.youtube.com/@dandiarchive (please Subscribe) diff --git a/dandiapi/api/templates/api/mail/registered_message.txt b/dandiapi/api/templates/api/mail/registered_message.txt index f656a3719..54dc6a86b 100644 --- a/dandiapi/api/templates/api/mail/registered_message.txt +++ b/dandiapi/api/templates/api/mail/registered_message.txt @@ -18,9 +18,8 @@ Getting started: https://emberarchive.org/getting-started Various aspects might relate to the DANDI Archive infrastructure, for which there are DANDI Docs: https://docs.dandiarchive.org -Discussions: https://github.com/dandi/helpdesk/discussions -Issues: https://github.com/dandi/helpdesk/issues/new/choose -DANDI YouTube: https://www.youtube.com/@dandiarchive +Support: https://docs.dandiarchive.org/support/ +YouTube: https://www.youtube.com/@dandiarchive (please Subscribe) Thank you for choosing EMBER-DANDI for your neurophysiology data needs. diff --git a/dandiapi/api/tests/test_asset.py b/dandiapi/api/tests/test_asset.py index ac0428888..a23b7565a 100644 --- a/dandiapi/api/tests/test_asset.py +++ b/dandiapi/api/tests/test_asset.py @@ -235,44 +235,6 @@ def test_asset_full_metadata_zarr(draft_asset_factory): } -@pytest.mark.django_db -def test_asset_full_metadata_access( - draft_asset_factory, asset_blob_factory, zarr_archive_factory, embargoed_zarr_archive_factory -): - raw_metadata = { - 'foo': 'bar', - 'schemaVersion': DANDI_SCHEMA_VERSION, - } - embargoed_zarr_asset: Asset = draft_asset_factory( - metadata=raw_metadata, blob=None, zarr=embargoed_zarr_archive_factory() - ) - open_zarr_asset: Asset = draft_asset_factory( - metadata=raw_metadata, blob=None, zarr=zarr_archive_factory() - ) - - embargoed_blob_asset: Asset = draft_asset_factory( - metadata=raw_metadata, blob=asset_blob_factory(embargoed=True), zarr=None - ) - open_blob_asset: Asset = draft_asset_factory( - metadata=raw_metadata, blob=asset_blob_factory(embargoed=False), zarr=None - ) - - # Test that access is correctly inferred from embargo status - assert embargoed_zarr_asset.full_metadata['access'] == [ - {'schemaKey': 'AccessRequirements', 'status': AccessType.EmbargoedAccess.value} - ] - assert embargoed_blob_asset.full_metadata['access'] == [ - {'schemaKey': 'AccessRequirements', 'status': AccessType.EmbargoedAccess.value} - ] - - assert open_zarr_asset.full_metadata['access'] == [ - {'schemaKey': 'AccessRequirements', 'status': AccessType.OpenAccess.value} - ] - assert open_blob_asset.full_metadata['access'] == [ - {'schemaKey': 'AccessRequirements', 'status': AccessType.OpenAccess.value} - ] - - # API Tests @@ -910,6 +872,7 @@ def test_asset_create_unembargo_in_progress(api_client, embargoed_asset_blob): @pytest.mark.django_db(transaction=True) def test_asset_create_embargoed_asset_blob_open_dandiset(api_client, embargoed_asset_blob, mocker): + """Test that creating an asset in an open dandiset unembargoes the asset blob.""" user = UserFactory.create() draft_version = DraftVersionFactory.create(dandiset__owners=[user]) api_client.force_authenticate(user=user) @@ -949,6 +912,54 @@ def test_asset_create_embargoed_asset_blob_open_dandiset(api_client, embargoed_a assert draft_version.status == Version.Status.PENDING +@pytest.mark.django_db(transaction=True) +def test_asset_update_embargoed_asset_blob_open_dandiset( + api_client, draft_asset_factory, asset_blob, embargoed_asset_blob, mocker +): + """Test that updating an asset in an open dandiset unembargoes the asset blob.""" + # First, create asset pointing to a different blob in an open dandiset + user = UserFactory.create() + draft_version = DraftVersionFactory.create(dandiset__owners=[user]) + asset = draft_asset_factory(blob=asset_blob) + draft_version.assets.add(asset) + + # Now, update asset to point to an embargoed blob + assert draft_version.dandiset.embargo_status == Dandiset.EmbargoStatus.OPEN + assert embargoed_asset_blob.embargoed + + # Mock this so we can check that it's been called later + mocked_func = mocker.patch('dandiapi.api.services.embargo.remove_asset_blob_embargoed_tag') + + api_client.force_authenticate(user=user) + resp = api_client.put( + f'/api/dandisets/{draft_version.dandiset.identifier}' + f'/versions/{draft_version.version}/assets/{asset.asset_id}/', + { + 'metadata': { + 'encodingFormat': 'application/x-nwb', + 'path': 'test/create/asset.txt', + 'meta': 'data', + 'foo': ['bar', 'baz'], + '1': 2, + }, + 'blob_id': embargoed_asset_blob.blob_id, + }, + ) + assert resp.status_code == 200 + new_asset = Asset.objects.get(asset_id=resp.json()['asset_id']) + + # Ensure the blob is now OPEN + assert new_asset.blob == embargoed_asset_blob + assert not new_asset.blob.embargoed + + # We can't test that the tags were correctly removed in a testing env, but we can test that the + # function which removes the tags was correctly invoked + mocked_func.assert_called_once() + + # Adding an Asset should trigger a revalidation + assert draft_version.status == Version.Status.PENDING + + @pytest.mark.django_db def test_asset_create_zarr(api_client): user = UserFactory.create() @@ -1335,7 +1346,9 @@ def test_asset_rest_update(api_client, asset, asset_blob): @pytest.mark.django_db def test_asset_rest_update_embargo(api_client, asset, embargoed_asset_blob): user = UserFactory.create() - draft_version = DraftVersionFactory.create(dandiset__owners=[user]) + draft_version = DraftVersionFactory.create( + dandiset__embargo_status=Dandiset.EmbargoStatus.EMBARGOED, dandiset__owners=[user] + ) api_client.force_authenticate(user=user) draft_version.assets.add(asset) add_asset_paths(asset=asset, version=draft_version) diff --git a/dandiapi/api/tests/test_asset_access_metadata.py b/dandiapi/api/tests/test_asset_access_metadata.py new file mode 100644 index 000000000..cdecd7da9 --- /dev/null +++ b/dandiapi/api/tests/test_asset_access_metadata.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from dandischema.consts import DANDI_SCHEMA_VERSION +from dandischema.models import AccessType +import pytest + +from dandiapi.api.models.asset import EmbargoedAssetWithinOpenDandisetError +from dandiapi.api.models.dandiset import Dandiset +from dandiapi.api.tests.factories import ( + DraftVersionFactory, +) + +if TYPE_CHECKING: + from dandiapi.api.models import Asset + + +@pytest.mark.django_db +def test_asset_full_metadata_access( + draft_asset_factory, asset_blob_factory, zarr_archive_factory, embargoed_zarr_archive_factory +): + raw_metadata = { + 'foo': 'bar', + 'schemaVersion': DANDI_SCHEMA_VERSION, + } + embargoed_zarr_asset: Asset = draft_asset_factory( + metadata=raw_metadata, blob=None, zarr=embargoed_zarr_archive_factory() + ) + open_zarr_asset: Asset = draft_asset_factory( + metadata=raw_metadata, blob=None, zarr=zarr_archive_factory() + ) + + embargoed_blob_asset: Asset = draft_asset_factory( + metadata=raw_metadata, blob=asset_blob_factory(embargoed=True), zarr=None + ) + open_blob_asset: Asset = draft_asset_factory( + metadata=raw_metadata, blob=asset_blob_factory(embargoed=False), zarr=None + ) + + # Test that access is correctly inferred from embargo status + assert embargoed_zarr_asset.full_metadata['access'] == [ + {'schemaKey': 'AccessRequirements', 'status': AccessType.EmbargoedAccess.value} + ] + assert embargoed_blob_asset.full_metadata['access'] == [ + {'schemaKey': 'AccessRequirements', 'status': AccessType.EmbargoedAccess.value} + ] + + assert open_zarr_asset.full_metadata['access'] == [ + {'schemaKey': 'AccessRequirements', 'status': AccessType.OpenAccess.value} + ] + assert open_blob_asset.full_metadata['access'] == [ + {'schemaKey': 'AccessRequirements', 'status': AccessType.OpenAccess.value} + ] + + +@pytest.mark.django_db +def test_access_metadata_open_blob_asset(asset_blob_factory, draft_asset_factory): + """Open blob asset returns no embargoedUntil.""" + asset = draft_asset_factory(blob=asset_blob_factory(embargoed=False)) + assert asset.access_metadata() == { + 'schemaKey': 'AccessRequirements', + 'status': AccessType.OpenAccess.value, + } + + +@pytest.mark.django_db +def test_access_metadata_open_zarr_asset(zarr_archive_factory, draft_asset_factory): + """Open zarr asset returns no embargoedUntil.""" + zarr = zarr_archive_factory(dandiset__embargo_status=Dandiset.EmbargoStatus.OPEN) + asset = draft_asset_factory(zarr=zarr, blob=None) + assert asset.access_metadata() == { + 'schemaKey': 'AccessRequirements', + 'status': AccessType.OpenAccess.value, + } + + +@pytest.mark.django_db +def test_access_metadata_embargoed_zarr_with_embargoed_until( + embargoed_zarr_archive_factory, draft_asset_factory +): + """Embargoed zarr asset returns embargoedUntil from draft version.""" + zarr = embargoed_zarr_archive_factory() + draft_version = zarr.dandiset.draft_version + draft_version.metadata['access'][0]['embargoedUntil'] = '2026-06-15' + draft_version.save() + + asset = draft_asset_factory(zarr=zarr, blob=None) + draft_version.assets.add(asset) + + assert asset.access_metadata() == { + 'schemaKey': 'AccessRequirements', + 'status': AccessType.EmbargoedAccess.value, + 'embargoedUntil': '2026-06-15', + } + + +@pytest.mark.django_db +def test_access_metadata_embargoed_zarr_without_embargoed_until( + embargoed_zarr_archive_factory, draft_asset_factory +): + """Embargoed zarr asset without embargoedUntil on version has no embargoedUntil in access.""" + zarr = embargoed_zarr_archive_factory() + asset = draft_asset_factory(zarr=zarr, blob=None) + zarr.dandiset.draft_version.assets.add(asset) + + assert 'embargoedUntil' not in asset.access_metadata() + + +@pytest.mark.django_db +def test_access_metadata_embargoed_blob_with_embargoed_until( + embargoed_asset_blob, draft_asset_factory +): + """Embargoed blob asset returns embargoedUntil from version.""" + draft_version = DraftVersionFactory.create( + dandiset__embargo_status=Dandiset.EmbargoStatus.EMBARGOED + ) + draft_version.metadata['access'][0]['embargoedUntil'] = '2026-06-15' + draft_version.save() + + asset = draft_asset_factory(blob=embargoed_asset_blob) + draft_version.assets.add(asset) + + assert asset.access_metadata() == { + 'schemaKey': 'AccessRequirements', + 'status': AccessType.EmbargoedAccess.value, + 'embargoedUntil': '2026-06-15', + } + + +@pytest.mark.django_db +def test_access_metadata_embargoed_blob_shared_across_embargoed_dandisets( + embargoed_asset_blob, draft_asset_factory +): + """Blob shared by multiple embargoed dandisets returns minimum embargo end date.""" + version_a = DraftVersionFactory.create( + dandiset__embargo_status=Dandiset.EmbargoStatus.EMBARGOED + ) + version_a.metadata['access'][0]['embargoedUntil'] = '2026-08-01' + version_a.save() + asset_a = draft_asset_factory(blob=embargoed_asset_blob) + version_a.assets.add(asset_a) + + version_b = DraftVersionFactory.create( + dandiset__embargo_status=Dandiset.EmbargoStatus.EMBARGOED + ) + version_b.metadata['access'][0]['embargoedUntil'] = '2018-10-25' + version_b.save() + asset_b = draft_asset_factory(blob=embargoed_asset_blob) + version_b.assets.add(asset_b) + + # Both assets should report the minimum embargo end date + assert asset_a.access_metadata()['embargoedUntil'] == '2018-10-25' + assert asset_b.access_metadata()['embargoedUntil'] == '2018-10-25' + + +@pytest.mark.django_db +def test_access_metadata_embargoed_blob_in_open_dandiset_raises( + embargoed_asset_blob, draft_asset_factory +): + """Embargoed blob in an open dandiset raises EmbargoedAssetWithinOpenDandisetError.""" + draft_version = DraftVersionFactory.create(dandiset__embargo_status=Dandiset.EmbargoStatus.OPEN) + asset = draft_asset_factory(blob=embargoed_asset_blob, zarr=None) + draft_version.assets.add(asset) + + with pytest.raises(EmbargoedAssetWithinOpenDandisetError): + asset.access_metadata() + + +@pytest.mark.django_db +def test_access_metadata_embargoed_blob_no_embargoed_until( + embargoed_asset_blob, draft_asset_factory +): + """Embargoed blob with no embargoedUntil on any version has no embargoedUntil in access.""" + assets = [] + for _ in range(5): + draft_version = DraftVersionFactory.create( + dandiset__embargo_status=Dandiset.EmbargoStatus.EMBARGOED + ) + asset = draft_asset_factory(blob=embargoed_asset_blob) + draft_version.assets.add(asset) + assets.append(asset) + + for asset in assets: + assert 'embargoedUntil' not in asset.access_metadata() diff --git a/dandiapi/api/tests/test_upload.py b/dandiapi/api/tests/test_upload.py index 842011d31..79ace220a 100644 --- a/dandiapi/api/tests/test_upload.py +++ b/dandiapi/api/tests/test_upload.py @@ -101,6 +101,26 @@ def test_upload_initialize(api_client, embargoed): assert upload.blob.name == f'blobs/{upload_id[:3]}/{upload_id[3:6]}/{upload_id}' +@pytest.mark.django_db +@pytest.mark.parametrize('dandiset_id', ['DANDI:abc', 'DANDI:123456', '001']) +def test_upload_initialize_invalid_dandiset(api_client, dandiset_id): + user = UserFactory.create() + DandisetFactory.create(embargo_status=Dandiset.EmbargoStatus.UNEMBARGOING, owners=[user]) + api_client.force_authenticate(user=user) + + content_size = 123 + resp = api_client.post( + '/api/uploads/initialize/', + { + 'contentSize': content_size, + 'digest': {'algorithm': 'dandi:dandi-etag', 'value': 'f' * 32 + '-1'}, + 'dandiset': dandiset_id, + }, + ) + + assert resp.status_code == 400 + + @pytest.mark.django_db def test_upload_initialize_unembargo_in_progress(api_client): user = UserFactory.create() diff --git a/dandiapi/api/tests/test_users.py b/dandiapi/api/tests/test_users.py index c39598ece..fe48d062f 100644 --- a/dandiapi/api/tests/test_users.py +++ b/dandiapi/api/tests/test_users.py @@ -157,6 +157,30 @@ def test_user_search_extra_data(api_client): assert api_client.get('/api/users/search/?', {'username': 'odysseus'}).data == [] +@pytest.mark.django_db +def test_user_search_extra_data_no_name(api_client): + """Test that a `null` name field in extra_data doesn't result in a `null` in the output.""" + user = UserFactory.create(social_account__extra_data__name=None) + + api_client.force_authenticate(user=user) + + resp = api_client.get('/api/users/search/?', {'username': user.username}) + assert len(resp.data) == 1 + assert resp.data[0]['name'] == user.get_full_name() + + +@pytest.mark.django_db +def test_user_search_extra_data_no_username(api_client): + """Test that a `null` username field in extra_data doesn't result in a `null` in the output.""" + user = UserFactory.create(social_account__extra_data__login=None) + + api_client.force_authenticate(user=user) + + resp = api_client.get('/api/users/search/?', {'username': user.username}) + assert len(resp.data) == 1 + assert resp.data[0]['username'] == user.username + + @pytest.mark.parametrize( ('status', 'expected_status_code', 'expected_search_results_value'), [ diff --git a/dandiapi/api/views/upload.py b/dandiapi/api/views/upload.py index 5da16a0d8..a0dadec23 100644 --- a/dandiapi/api/views/upload.py +++ b/dandiapi/api/views/upload.py @@ -39,7 +39,7 @@ class DigestSerializer(serializers.Serializer): class UploadInitializationRequestSerializer(serializers.Serializer): contentSize = serializers.IntegerField(min_value=1) # noqa: N815 digest = DigestSerializer() - dandiset = serializers.RegexField(Dandiset.IDENTIFIER_REGEX) + dandiset = serializers.RegexField(f'^{Dandiset.IDENTIFIER_REGEX}$') class PartInitializationResponseSerializer(serializers.Serializer): @@ -131,7 +131,7 @@ def upload_initialize_view(request: AuthenticatedRequest) -> HttpResponseBase: if digest['algorithm'] != 'dandi:dandi-etag': return Response('Unsupported Digest Type', status=400) etag = digest['value'] - dandiset_id = request_serializer.validated_data['dandiset'] + dandiset_id = int(request_serializer.validated_data['dandiset']) dandiset = get_object_or_404( get_visible_dandisets(request.user), id=dandiset_id, diff --git a/dandiapi/api/views/users.py b/dandiapi/api/views/users.py index 7ee076e18..9937ab61c 100644 --- a/dandiapi/api/views/users.py +++ b/dandiapi/api/views/users.py @@ -67,12 +67,12 @@ def social_account_to_dict(social_account: SocialAccount): def serialize_user(user: User): """Serialize a user that's been annotated with a `social_account_data` field.""" username = user.username - name = f'{user.first_name} {user.last_name}'.strip() + name = user.get_full_name() # Prefer social account info if present if user.social_account_data is not None: - username = user.social_account_data.get('login', username) - name = user.social_account_data.get('name', name) + username = user.social_account_data.get('login') or username + name = user.social_account_data.get('name') or name return { 'admin': user.is_superuser, diff --git a/dandiapi/settings/base.py b/dandiapi/settings/base.py index f257a8a78..8edcd33b1 100644 --- a/dandiapi/settings/base.py +++ b/dandiapi/settings/base.py @@ -10,6 +10,7 @@ from corsheaders.defaults import default_headers import django_stubs_ext from environ import Env + from resonant_settings.allauth import * from resonant_settings.celery import * from resonant_settings.django import * @@ -30,6 +31,8 @@ ROOT_URLCONF = 'dandiapi.urls' +WSGI_APPLICATION = 'dandiapi.wsgi.application' + INSTALLED_APPS = [ # Install local apps first, to ensure any overridden resources are found first 'dandiapi.api.apps.ApiConfig', @@ -79,6 +82,7 @@ # Add username middleware after authentication to capture username for gunicorn access logs 'dandiapi.api.middleware.GunicornUsernameMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', + 'django.contrib.sites.middleware.CurrentSiteMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'allauth.account.middleware.AccountMiddleware', ] @@ -90,7 +94,7 @@ 'default': { **env.db_url('DJANGO_DATABASE_URL', engine='django.db.backends.postgresql'), 'CONN_MAX_AGE': timedelta(minutes=10).total_seconds(), - } + }, } DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' @@ -102,7 +106,6 @@ 'BACKEND': 'whitenoise.storage.CompressedStaticFilesStorage', }, } -DANDI_DANDISETS_BUCKET_NAME: str STATIC_ROOT = BASE_DIR / 'staticfiles' # Django staticfiles auto-creates any intermediate directories, but do so here to prevent warnings. diff --git a/dandiapi/settings/development.py b/dandiapi/settings/development.py index 7fac23a24..9b5ecff75 100644 --- a/dandiapi/settings/development.py +++ b/dandiapi/settings/development.py @@ -5,14 +5,14 @@ from .base import * # Import these afterwards, to override -from resonant_settings.development.celery import * # isort: skip -from resonant_settings.development.debug_toolbar import * # isort: skip +from resonant_settings.development.celery import * +from resonant_settings.development.debug_toolbar import * INSTALLED_APPS += [ 'debug_toolbar', 'django_browser_reload', ] -# Force WhiteNoice to serve static files, even when using 'manage.py runserver_plus' +# Force WhiteNoise to serve static files, even when using "manage.py runserver_plus" staticfiles_index = INSTALLED_APPS.index('django.contrib.staticfiles') INSTALLED_APPS.insert(staticfiles_index, 'whitenoise.runserver_nostatic') @@ -33,7 +33,7 @@ # to add new settings as individual feature flags. DEBUG = True -SECRET_KEY = 'insecure-secret' # noqa: S105 +SECRET_KEY = 'insecure-secret' # This is typically only overridden when running from Docker. INTERNAL_IPS = InternalIPS(env.list('DJANGO_INTERNAL_IPS', cast=str, default=['127.0.0.1'])) @@ -56,7 +56,6 @@ 'media_url': _storage_media_url, }, } -DANDI_DANDISETS_BUCKET_NAME = STORAGES['default']['OPTIONS']['bucket_name'] # TODO: remove EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' diff --git a/dandiapi/settings/heroku_production.py b/dandiapi/settings/heroku_production.py index 06b55f86e..ed85a97e3 100644 --- a/dandiapi/settings/heroku_production.py +++ b/dandiapi/settings/heroku_production.py @@ -8,7 +8,7 @@ # Provided by https://github.com/ianpurvis/heroku-buildpack-version os.environ['DJANGO_SENTRY_RELEASE'] = os.environ['SOURCE_VERSION'] -from .production import * # isort: skip +from .production import * # This needs to be set by the HTTPS terminating reverse proxy. # Heroku and Render automatically set this. diff --git a/dandiapi/settings/production.py b/dandiapi/settings/production.py index a45e507a5..1be6bf7a0 100644 --- a/dandiapi/settings/production.py +++ b/dandiapi/settings/production.py @@ -12,10 +12,8 @@ from .base import * # Import these afterwards, to override -from resonant_settings.production.email import * # isort: skip -from resonant_settings.production.https import * # isort: skip - -WSGI_APPLICATION = 'dandiapi.wsgi.application' +from resonant_settings.production.email import * +from resonant_settings.production.https import * SECRET_KEY: str = env.str('DJANGO_SECRET_KEY') @@ -27,18 +25,17 @@ # Only allow GitHub auth on production, no username/password SOCIALACCOUNT_ONLY = True -# This only needs to be defined in production. Testing will add 'testserver'. In development -# (specifically when DEBUG is True), 'localhost' and '127.0.0.1' will be added. +# This only needs to be defined in production. Testing will add "testserver". In development +# (specifically when DEBUG is True), "localhost" and "127.0.0.1" will be added. ALLOWED_HOSTS: list[str] = env.list('DJANGO_ALLOWED_HOSTS', cast=str) -DANDI_DANDISETS_BUCKET_NAME: str = env.str('DJANGO_STORAGE_BUCKET_NAME') STORAGES['default'] = { 'BACKEND': 'dandiapi.storage.DandiS3Storage', 'OPTIONS': { 'region_name': env.str('AWS_DEFAULT_REGION'), 'access_key': env.str('AWS_ACCESS_KEY_ID'), 'secret_key': env.str('AWS_SECRET_ACCESS_KEY'), - 'bucket_name': DANDI_DANDISETS_BUCKET_NAME, + 'bucket_name': env.str('DJANGO_STORAGE_BUCKET_NAME'), 'querystring_expire': int(timedelta(hours=6).total_seconds()), 'max_memory_size': 5 * 1024 * 1024, }, @@ -53,8 +50,8 @@ DANDI_DEV_EMAIL: str = env.str('DJANGO_DANDI_DEV_EMAIL') DANDI_ADMIN_EMAIL: str = env.str('DJANGO_DANDI_ADMIN_EMAIL') -# sentry_sdk is able to directly use environment variables like 'SENTRY_DSN', but prefix them -# with 'DJANGO_' to avoid conflicts with other Sentry-using services. +# sentry_sdk is able to directly use environment variables like "SENTRY_DSN", but prefix them +# with "DJANGO_" to avoid conflicts with other Sentry-using services. sentry_sdk.init( dsn=env.str('DJANGO_SENTRY_DSN', default=None), environment=env.str('DJANGO_SENTRY_ENVIRONMENT', default=None), diff --git a/dandiapi/settings/testing.py b/dandiapi/settings/testing.py index f7e399a3f..bc5c91efe 100644 --- a/dandiapi/settings/testing.py +++ b/dandiapi/settings/testing.py @@ -1,8 +1,10 @@ from __future__ import annotations +from secrets import randbelow + from .base import * -SECRET_KEY = 'insecure-secret' # noqa: S105 +SECRET_KEY = 'insecure-secret' # Use a fast, insecure hasher to speed up tests PASSWORD_HASHERS = ['django.contrib.auth.hashers.MD5PasswordHasher'] @@ -14,11 +16,10 @@ 'endpoint_url': f'{_minio_url.scheme}://{_minio_url.hostname}:{_minio_url.port}', 'access_key': _minio_url.username, 'secret_key': _minio_url.password, - 'bucket_name': 'test-django-storage', + 'bucket_name': f'test-django-storage-{randbelow(1_000_000):06d}', 'querystring_expire': int(timedelta(hours=6).total_seconds()), }, } -DANDI_DANDISETS_BUCKET_NAME = 'test-django-storage' # Testing will set EMAIL_BACKEND to use the memory backend diff --git a/dandiapi/zarr/management/__init__.py b/dandiapi/zarr/management/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dandiapi/zarr/management/commands/__init__.py b/dandiapi/zarr/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/e2e/README.md b/e2e/README.md index 87588408c..169e0fb23 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -29,8 +29,8 @@ Prior to executing the tests, you will need to login at `localhost:8085` to prov ```bash cd e2e -# Install testdata -./manage.py loaddata playwright +# Install testdata (manage.py is in the repo root) +../manage.py loaddata playwright # Run all tests npx playwright test diff --git a/e2e/tests/dandisetLandingPage.spec.ts b/e2e/tests/dandisetLandingPage.spec.ts index 95314d263..fdbb25e8e 100644 --- a/e2e/tests/dandisetLandingPage.spec.ts +++ b/e2e/tests/dandisetLandingPage.spec.ts @@ -45,4 +45,76 @@ test.describe("dandiset landing page", async () => { await page.waitForLoadState("networkidle"); await expect(page.getByText("Error: Dandiset does not exist")).toHaveCount(1); }); + + test.describe("how to cite tab", () => { + let dandisetId: string | undefined; + + test.beforeEach(async ({ page }) => { + await registerNewUser(page); + const dandisetName = faker.lorem.words(); + const dandisetDescription = faker.lorem.sentences(); + dandisetId = await registerDandiset(page, dandisetName, dandisetDescription); + }); + + test("navigate to the How to Cite tab", async ({ page }) => { + // Click the "How to Cite" tab + await page.getByRole("tab", { name: "How to Cite" }).click(); + + // Verify the tab content is displayed + await expect(page.getByText("How to Cite this Dataset")).toBeVisible(); + }); + + test("displays draft warning for unpublished dandisets", async ({ page }) => { + await page.getByRole("tab", { name: "How to Cite" }).click(); + + await expect( + page.getByText("Citing draft dandisets is not recommended"), + ).toBeVisible(); + await expect( + page.getByText("Please contact the authors to request publication"), + ).toBeVisible(); + }); + + test("displays all expected sections", async ({ page }) => { + await page.getByRole("tab", { name: "How to Cite" }).click(); + + // Check all section headers are present + await expect(page.getByRole("heading", { name: "Full Citation" })).toBeVisible(); + await expect(page.getByRole("heading", { name: "Materials and Methods" })).toBeVisible(); + await expect(page.getByRole("heading", { name: "Data Availability Statement" })).toBeVisible(); + await expect(page.getByRole("heading", { name: "DANDI Identifier" })).toBeVisible(); + await expect(page.getByRole("heading", { name: "License" })).toBeVisible(); + }); + + test("displays DANDI identifier with correct format", async ({ page }) => { + await page.getByRole("tab", { name: "How to Cite" }).click(); + + // The identifier should follow the format DANDI:/draft + await expect(page.getByText(`DANDI:${dandisetId}/draft`)).toBeVisible(); + await expect(page.getByText("DANDI Archive RRID: SCR_017571")).toBeVisible(); + }); + + test("displays citation format selector", async ({ page }) => { + await page.getByRole("tab", { name: "How to Cite" }).click(); + + // The format dropdown should be visible with APA 7th as default + await expect(page.getByText("APA 7th")).toBeVisible(); + }); + + test("displays guide link", async ({ page }) => { + await page.getByRole("tab", { name: "How to Cite" }).click(); + + await expect( + page.getByRole("link", { name: "guide on citing dandisets" }), + ).toBeVisible(); + }); + + test("can switch back to Overview tab", async ({ page }) => { + await page.getByRole("tab", { name: "How to Cite" }).click(); + await expect(page.getByText("How to Cite this Dataset")).toBeVisible(); + + await page.getByRole("tab", { name: "Overview" }).click(); + await expect(page.getByText("How to Cite this Dataset")).not.toBeVisible(); + }); + }); }); diff --git a/gunicorn.conf.py b/gunicorn.conf.py index f74848b77..e8bd77b87 100644 --- a/gunicorn.conf.py +++ b/gunicorn.conf.py @@ -6,9 +6,11 @@ bind = f'0.0.0.0:{os.environ.get("PORT", "8000")}' -# Explicitly set the timeout to 5 seconds less than the Heroku request timeout so -# that gunicorn can gracefully shut down the worker if a request times out. -timeout = 25 +# Set `graceful_timeout` to shorter than the 30 second limit imposed by Heroku restarts +graceful_timeout = 25 + +# Set `timeout` to shorter than the 30 second limit imposed by the Heroku router +timeout = 15 # Add the username to the access log (set by Django middleware) access_log_format = AccessLogFormat.default + ' ' diff --git a/pyproject.toml b/pyproject.toml index bff794d16..040dffa34 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ "django-guardian", "django-oauth-toolkit", "django-resonant-settings[allauth,celery]", - "django-resonant-utils", + "django-resonant-utils[allauth,s3_storage]", "django-stubs-ext", # TODO: pin djangorestframework until we figure out what the cause of # https://github.com/dandi/dandi-archive/issues/1896 is. @@ -45,8 +45,8 @@ dependencies = [ # Production-only "django-s3-file-field[s3]", "django-storages[s3]", - "gunicorn", "sentry-sdk[celery,django,pure_eval]", + "gunicorn", # Development-only, but required "tqdm", ] @@ -75,15 +75,12 @@ dev = [ lint = [ "ruff", ] -format = [ - "ruff", -] type = [ "mypy", "boto3-stubs[s3]", "celery-types", "django-stubs[compatible-mypy]", - "djangorestframework-stubs", + "djangorestframework-stubs[compatible-mypy]", ] test = [ "djangorestframework-yaml", @@ -104,7 +101,7 @@ test = [ ] [tool.hatch.build] -packages = [ +only-include = [ "dandiapi", ] @@ -123,26 +120,8 @@ only-include = [ source = "vcs" raw-options = {version_scheme = "no-guess-dev"} -[tool.mypy] -ignore_missing_imports = true -show_error_codes = true -disable_error_code = ["attr-defined", "var-annotated"] -follow_imports = "skip" # Don't follow imports into other files. This should be removed once all type errors have been resolved. -exclude = [ - "^dandiapi/api/tests/", - "^dandiapi/api/views/", - "^dandiapi/zarr/tests/", -] - -# Re-enable these when https://github.com/typeddjango/django-stubs/issues/417 is fixed. -# plugins = ["mypy_django_plugin.main", "mypy_drf_plugin.main"] - -# [tool.django-stubs] -# django_settings_module = "dandiapi.settings" - [tool.ruff] line-length = 100 -target-version = "py313" [tool.ruff.lint] select = ["ALL"] @@ -151,7 +130,7 @@ ignore = [ # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules "COM812", # missing-trailing-comma "COM819", # prohibited-trailing-comma - "D206", # indent-with-spaces + "D206", # docstring-tab-indentation "D300", # triple-single-quotes "E111", # indentation-with-invalid-multiple "E114", # indentation-with-invalid-multiple-comment @@ -161,54 +140,60 @@ ignore = [ "Q", # flake8-quotes "W191", # tab-indentation - "A003", # Class attribute is shadowing a Python builtin + # Incompatible with Django + "A003", # builtin-attribute-shadowing + "ARG001", # unused-function-argument + "ARG002", # unused-method-argument + "RUF012", # mutable-class-default + + # Overly burdensome "ANN", # flake8-annotations - "ARG001", # Unused function argument - "ARG002", # Unused method argument - "D1", # Missing docstring - "EM101", # Exception must not use a string literal, assign to variable first - "EM102", # Exception must not use an f-string literal, assign to variable first - "ERA001", # Found commented-out code + "D1", # undocumented-* + "EM", # flake8-errmsg + "ERA001", # commented-out-code "FIX", # flake8-fixme - "TD002", # Missing author in TODO - "TD003", # Missing issue link on the line following this TODO - "TRY003", # Avoid specifying long messages outside the exception class + "PLR2004", # magic-value-comparison + "RET505", # superfluous-else-return + "RET506", # superfluous-else-raise + "RET507", # superfluous-else-continue + "RET508", # superfluous-else-break + "TD002", # missing-todo-author + "TD003", # missing-todo-link + "TRY003", # raise-vanilla-args - # Try to fix upstream - "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar` - - # Fix in DANDI codebase. PR that will fix this: - # https://github.com/dandi/dandi-archive/pull/1782 - "PTH119", # TODO: re-enable this when it's fixed - "PLC0415", # `import` should be at the top-level of a file + # Project-specific + "PLC0415", # import-outside-top-level ] [tool.ruff.lint.per-file-ignores] -"scripts/**" = [ - "INP001", # File is part of an implicit namespace package +"**/migrations/**" = [ + "N806", # non-lowercase-variable-in-function + "RUF012", # mutable-class-default ] "**/settings/**" = [ - "F403", # unable to detect undefined names - "F405", # may be undefined, or defined from star imports -] -"**/migrations/**" = [ - "N806", # Variable in function should be lowercase - "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar` + "F403", # undefined-local-with-import-star + "F405", # undefined-local-with-import-star-usage ] -"**/management/commands/**" = [ - "INP001", # File is part of an implicit namespace package +"**/settings/{development,testing}.py" = [ + "S105", # hardcoded-password-string ] "**/tests/**" = [ - "DJ007", # Do not use `__all__` - "DJ008", # Model does not define `__str__` method - "PLR0913", # Too many arguments to function call - "PLR2004", # Magic value used in comparison + "DJ007", # django-all-with-model-form + "DJ008", # django-model-without-dunder-str + "INP001", # implicit-namespace-package + "PLR0913", # too-many-arguments "S", # flake8-bandit - "SLF001", # Private member accessed + "SLF001", # private-member-access +] +"scripts/**" = [ + "INP001", # implicit-namespace-package ] -[tool.ruff.format] -quote-style = "single" +[tool.ruff.lint.flake8-boolean-trap] +extend-allowed-calls = ["pydantic.Field", "django.db.models.Value"] + +[tool.ruff.lint.flake8-bugbear] +extend-immutable-calls = ["ninja.Query"] [tool.ruff.lint.flake8-self] extend-ignore-names = ["_base_manager", "_default_manager", "_meta"] @@ -216,12 +201,35 @@ extend-ignore-names = ["_base_manager", "_default_manager", "_meta"] [tool.ruff.lint.isort] # Sort by name, don't cluster "from" vs "import" force-sort-within-sections = true -# Deferred annotations allows TCH rules to move more imports into TYPE_CHECKING blocks +# Deferred annotations allows TC* rules to move more imports into TYPE_CHECKING blocks required-imports = ["from __future__ import annotations"] +section-order = ["future", "standard-library", "third-party", "first-party", "local-folder", "resonant-settings"] +sections = {"resonant-settings" = ["resonant_settings"] } [tool.ruff.lint.pydocstyle] convention = "pep257" +[tool.ruff.format] +quote-style = "single" + +[tool.mypy] +files = [ + "dandiapi", +] +check_untyped_defs = true +ignore_missing_imports = true +show_error_codes = true +warn_redundant_casts = true +warn_unused_configs = true +warn_unused_ignores = true +plugins = [ + "mypy_django_plugin.main", + "mypy_drf_plugin.main", +] + +[tool.django-stubs] +django_settings_module = "dandiapi.settings.testing" + [tool.pytest.ini_options] addopts = [ # Test utilities should be imported absolutely from the pythonpath, @@ -237,7 +245,7 @@ addopts = [ filterwarnings = [ "error", # pytest often causes unclosed socket warnings - "ignore:unclosed as model type:UserWarning:pytest_factoryboy.fixture", ] diff --git a/tox.ini b/tox.ini index 7fda32016..b5d0aba57 100644 --- a/tox.ini +++ b/tox.ini @@ -26,7 +26,7 @@ commands = [testenv:format] package = skip dependency_groups = - format + lint commands = ruff check --fix-only ruff format diff --git a/uv.lock b/uv.lock index 773b6462d..f714f652d 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.13" resolution-markers = [ "python_full_version >= '3.14' and platform_machine != 'arm64' and sys_platform == 'darwin'", @@ -697,7 +697,7 @@ dependencies = [ { name = "django-guardian" }, { name = "django-oauth-toolkit" }, { name = "django-resonant-settings", extra = ["allauth", "celery"] }, - { name = "django-resonant-utils" }, + { name = "django-resonant-utils", extra = ["allauth", "s3-storage"] }, { name = "django-s3-file-field", extra = ["s3"] }, { name = "django-storages", extra = ["s3"] }, { name = "django-stubs-ext" }, @@ -734,9 +734,6 @@ dev = [ { name = "tox" }, { name = "tox-uv" }, ] -format = [ - { name = "ruff" }, -] lint = [ { name = "ruff" }, ] @@ -758,7 +755,7 @@ type = [ { name = "boto3-stubs", extra = ["s3"] }, { name = "celery-types" }, { name = "django-stubs", extra = ["compatible-mypy"] }, - { name = "djangorestframework-stubs" }, + { name = "djangorestframework-stubs", extra = ["compatible-mypy"] }, { name = "mypy" }, ] @@ -781,7 +778,7 @@ requires-dist = [ { name = "django-guardian" }, { name = "django-oauth-toolkit" }, { name = "django-resonant-settings", extras = ["allauth", "celery"] }, - { name = "django-resonant-utils" }, + { name = "django-resonant-utils", extras = ["allauth", "s3-storage"] }, { name = "django-s3-file-field", extras = ["s3"] }, { name = "django-storages", extras = ["s3"] }, { name = "django-stubs-ext" }, @@ -813,7 +810,6 @@ dev = [ { name = "tox" }, { name = "tox-uv" }, ] -format = [{ name = "ruff" }] lint = [{ name = "ruff" }] test = [ { name = "djangorestframework-yaml" }, @@ -833,7 +829,7 @@ type = [ { name = "boto3-stubs", extras = ["s3"] }, { name = "celery-types" }, { name = "django-stubs", extras = ["compatible-mypy"] }, - { name = "djangorestframework-stubs" }, + { name = "djangorestframework-stubs", extras = ["compatible-mypy"] }, { name = "mypy" }, ] @@ -1026,14 +1022,15 @@ wheels = [ [[package]] name = "django-resonant-settings" -version = "0.41.0" +version = "0.47.1" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "django" }, { name = "django-environ" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ee/19/4fa035cec41b5c7ceb1348f5aec7692ec6a71b0a1d79cec7746aa2e1926a/django_resonant_settings-0.41.0.tar.gz", hash = "sha256:1324c12d33329a36a198c500695434db57206ac75ee63fa8da35d390831bc8de", size = 17654, upload-time = "2025-10-16T20:11:42.172Z" } +sdist = { url = "https://files.pythonhosted.org/packages/07/e2/0961283ccaf797307f3671a7ff3965b5a54d816ff86c9091713cdcfde562/django_resonant_settings-0.47.1.tar.gz", hash = "sha256:3d58bed8d0595806a42007ef63f58348523d95699e5811068ca3337483ef5843", size = 19179, upload-time = "2026-02-26T15:46:42.839Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/38/d3/ac3e2ffd10139f3f209acafc1f8dd7b9d177c1b7b26b71cdd60e2f36020e/django_resonant_settings-0.41.0-py3-none-any.whl", hash = "sha256:10be63da455eaaa67a34b057dabe9f15b380917c945cb8c33f929ad1afde2b47", size = 24123, upload-time = "2025-10-16T20:11:41.112Z" }, + { url = "https://files.pythonhosted.org/packages/da/7d/a5f08da9e00e626b26beebcb94c5adfa26f78f54c1c9c3b61b8654ae5f02/django_resonant_settings-0.47.1-py3-none-any.whl", hash = "sha256:c96756aeac75248eca14b580228c6c41e2e6186030dbc47723cdb1b959f8a4ef", size = 26093, upload-time = "2026-02-26T15:46:41.69Z" }, ] [package.optional-dependencies] @@ -1056,6 +1053,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b9/49/394af41df5f9c73cd3703374364be8ce8581c0b0390761498958410dc648/django_resonant_utils-0.16.0-py3-none-any.whl", hash = "sha256:3f7b8b7e284269cac704122ef823be415eaf777d601d51a46369250294c276c4", size = 14201, upload-time = "2025-07-10T19:31:34.29Z" }, ] +[package.optional-dependencies] +allauth = [ + { name = "django-allauth" }, +] +s3-storage = [ + { name = "django-storages", extra = ["s3"] }, +] + [[package]] name = "django-s3-file-field" version = "1.1.0" @@ -1154,6 +1159,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ba/45/63fac5c34313986acc005529fdba638822bae973f2b61a0542a6c8494847/djangorestframework_stubs-3.16.4-py3-none-any.whl", hash = "sha256:3b27353fa797876f55da87eceafe4c2f265a93924fa7763d257e509d865df1b2", size = 56503, upload-time = "2025-09-29T20:11:19.317Z" }, ] +[package.optional-dependencies] +compatible-mypy = [ + { name = "django-stubs", extra = ["compatible-mypy"] }, + { name = "mypy" }, +] + [[package]] name = "djangorestframework-yaml" version = "2.0.0" diff --git a/web/src/components/AppBar/AppBar.vue b/web/src/components/AppBar/AppBar.vue index cb0d52a2b..068db7e1d 100644 --- a/web/src/components/AppBar/AppBar.vue +++ b/web/src/components/AppBar/AppBar.vue @@ -205,7 +205,7 @@ const navItems: NavigationItem[] = [ external: true, }, // { - // text: 'Help', + // text: 'Support', // to: dandiHelpUrl, // external: true, // }, diff --git a/web/src/components/DLP/HowToCiteTab.vue b/web/src/components/DLP/HowToCiteTab.vue new file mode 100644 index 000000000..0a480aaf8 --- /dev/null +++ b/web/src/components/DLP/HowToCiteTab.vue @@ -0,0 +1,472 @@ + + + + + diff --git a/web/src/components/DandiFooter.vue b/web/src/components/DandiFooter.vue index 57b407b15..32eed54c4 100644 --- a/web/src/components/DandiFooter.vue +++ b/web/src/components/DandiFooter.vue @@ -4,8 +4,8 @@ - © 2019 - 2025 The DANDI Team
- © 2024 - 2025 JHU/APL.
+ © 2019 - {{ currentYear }} The DANDI Team
+ © 2024 - {{ currentYear }} JHU/APL.