From 7fea5aca43fa4ba79352bbd16c3bce65f6c2f75a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Tue, 25 May 2021 21:41:48 +0200 Subject: [PATCH 01/29] Dataset versions stubs and first tests. --- .../versions/logic/dataset_version_action.py | 31 ++++ .../tests/test_dataset_version_action.py | 141 ++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 ckanext/versions/logic/dataset_version_action.py create mode 100644 ckanext/versions/tests/test_dataset_version_action.py diff --git a/ckanext/versions/logic/dataset_version_action.py b/ckanext/versions/logic/dataset_version_action.py new file mode 100644 index 0000000..06d7285 --- /dev/null +++ b/ckanext/versions/logic/dataset_version_action.py @@ -0,0 +1,31 @@ +from ckan.plugins import toolkit + + +def dataset_version_create(context, data_dict): + pass + + +@toolkit.side_effect_free +def dataset_version_list(context, data_dict): + pass + + +def dataset_version_current(context, data_dict): + pass + + +def dataset_history(context, data_dict): + # not sure if we need this + pass + + +def activity_dataset_show(context, data_dict): + pass + + +def get_activity_id_from_dataset_version_name(context, data_dict): + pass + + +def dataset_has_versions(context, data_dict): + pass diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py new file mode 100644 index 0000000..a526aef --- /dev/null +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -0,0 +1,141 @@ +import pytest + +from ckan.plugins import toolkit +from ckan.tests import factories +from ckanext.versions.logic.dataset_version_action import dataset_version_create, dataset_has_versions +from ckanext.versions.tests import get_context + + +@pytest.mark.usefixtures('clean_db', 'versions_setup') +class TestDatasetVersion(object): + + def test_data_version_create(self, org_admin, test_dataset): + version_name = "Test Version 1.0" + version_notes = "Some details about the version" + version = dataset_version_create( + get_context(org_admin), + { + "dataset_id": test_dataset['id'], + "name": version_name, + "notes": version_notes + } + ) + checks = {'package_id': test_dataset['id'], + 'resource_id': None, + 'notes': version_notes, + 'name': version_name, + 'creator_user_id': org_admin['id']} + + self._assert_version(version, checks) + + @pytest.mark.parametrize("user_role, can_create_version", [ + ('admin', True), + ('editor', True), + ('member', False), + ]) + def test_auth_version_create(self, test_organization, test_dataset, user_role, can_create_version): + for user in test_organization['users']: + if user['capacity'] == user_role: + if can_create_version: + self._create_version(test_dataset['id'], user) + else: + with pytest.raises(toolkit.NotAuthorized): + self._create_version(test_dataset['id'], user) + pytest.fail("Couldn't find user with required role %s", user_role) + + def test_should_not_create_data_version_with_same_name(self, test_dataset, org_editor): + version_name = "Not unique name" + self._create_version(test_dataset['id'], org_editor, version_name=version_name) + with pytest.raises(toolkit.ValidationError): + self._create_version(test_dataset['id'], org_editor, version_name=version_name) + + def test_should_fail_if_dataset_not_exists(self, org_editor): + with pytest.raises(toolkit.ObjectNotFound) as e: + self._create_version('fake_dataset_id', org_editor) + assert e.msg == "Dataset not found" + + def test_version_has_valid_activity_id(self, test_organization, org_editor): + old_name = "Initial name" + dataset = factories.Dataset(name=old_name, owner_org=test_organization['id']) + version = self._create_version(dataset['id'], org_editor) + context = get_context(org_editor) + toolkit.get_action('package_patch')( + context, + { + "id": dataset['id'], + "name": "Updated Name" + } + ) + + old_dataset = toolkit.get_action('activity_data_show')( + context, + {'id': version['activity_id']} + )['package'] + + assert old_dataset['name'] == old_name + assert old_dataset['id'] == dataset['id'] + + + def test_new_dataset_has_no_versions(self, test_dataset, org_editor): + context = get_context(org_editor) + assert False == dataset_has_versions( + context, + {'id': test_dataset['id']} + ) + + def test_dataset_has_versions_after_one_created(self, test_dataset, org_editor): + context = get_context(org_editor) + self._create_version(test_dataset['id'], org_editor) + assert True == dataset_has_versions( + context, + {'id': test_dataset['id']} + ) + + + def _create_version(self, dataset_id, user, version_name="Default Name"): + return dataset_version_create( + get_context(user), + { + "dataset_id": dataset_id, + "name": version_name + } + ) + + def _assert_version(self, version, checks): + assert version + for k, v in checks: + assert version[k] == v + + +@pytest.fixture() +def org_admin(): + return factories.User() + + +@pytest.fixture() +def org_editor(): + return factories.User() + + +@pytest.fixture() +def org_member(): + return factories.User() + + +@pytest.fixture() +def test_organization(org_admin, org_editor, org_member): + return factories.Organization(users=[ + {'name': org_admin['id'], 'capacity': 'admin'}, + {'name': org_editor['id'], 'capacity': 'editor'}, + {'name': org_member['id'], 'capacity': 'member'} + ]) + + +@pytest.fixture() +def test_dataset(test_organization): + return factories.Dataset(owner_org=test_organization['id']) + + +@pytest.fixture() +def test_resource(test_dataset): + return factories.Resource(package_id=test_dataset['id']) From 1abbcc81bc4e4eceae173d6f8d55fe963aeda9fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Wed, 26 May 2021 08:50:47 +0200 Subject: [PATCH 02/29] Added docstrings to method names. --- .../versions/logic/dataset_version_action.py | 61 +++++++++++++++++-- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/ckanext/versions/logic/dataset_version_action.py b/ckanext/versions/logic/dataset_version_action.py index 06d7285..1719cd0 100644 --- a/ckanext/versions/logic/dataset_version_action.py +++ b/ckanext/versions/logic/dataset_version_action.py @@ -2,30 +2,81 @@ def dataset_version_create(context, data_dict): + """Create a new version from the current dataset's activity_id + + Currently you must have editor level access on the dataset + to create a version. If creator_user_id is not present, it will be set as + the logged it user. + + :param dataset_id: the id of the dataset + :type dataset_id: string + :param name: A short name for the version, e.g. v1.0 + :type name: string + :param notes optional: Notes about the version + :type notes: string + :returns: the newly created version + :rtype: dictionary + """ pass @toolkit.side_effect_free def dataset_version_list(context, data_dict): - pass + """List versions of a given dataset - -def dataset_version_current(context, data_dict): + :param dataset_id: the id of the dataset + :type dataset_id: string + :returns: list of versions created for the dataset + :rtype: list + """ pass -def dataset_history(context, data_dict): - # not sure if we need this +@toolkit.side_effect_free +def dataset_version_latest(context, data_dict): + ''' Show the latest version for a dataset + + :param dataset_id: the if of the dataset + :type dataset_id: string + :returns the version dictionary + :rtype dict + ''' pass def activity_dataset_show(context, data_dict): + ''' Returns a dataset from the activity object. + + :param activity_id: the id of the activity + :type activity_id: string + :param dataset_id: the id of the resource + :type dataset_id: string + :returns: The dataset in the activity + :rtype: dict + ''' pass def get_activity_id_from_dataset_version_name(context, data_dict): + ''' Returns the activity_id for the dataset version + + :param dataset_id: the id of the resource + :type dataset_id: string + :param version: the name or id of the version + :type version: string + :returns: The activity_id of the version + :rtype: string + + ''' pass def dataset_has_versions(context, data_dict): + """Check if the dataset has versions. + + :param dataset_id: the id the dataset + :type dataset_id: string + :returns: True if the dataset has at least 1 version + :rtype: boolean + """ pass From 11b433a31531326d909099ccb855a53ece497e2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Wed, 26 May 2021 09:41:50 +0200 Subject: [PATCH 03/29] Test helper functions moved out from the class. --- .../tests/test_dataset_version_action.py | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py index a526aef..bdf9a85 100644 --- a/ckanext/versions/tests/test_dataset_version_action.py +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -26,7 +26,7 @@ def test_data_version_create(self, org_admin, test_dataset): 'name': version_name, 'creator_user_id': org_admin['id']} - self._assert_version(version, checks) + _assert_version(version, checks) @pytest.mark.parametrize("user_role, can_create_version", [ ('admin', True), @@ -37,27 +37,27 @@ def test_auth_version_create(self, test_organization, test_dataset, user_role, c for user in test_organization['users']: if user['capacity'] == user_role: if can_create_version: - self._create_version(test_dataset['id'], user) + _create_version(test_dataset['id'], user) else: with pytest.raises(toolkit.NotAuthorized): - self._create_version(test_dataset['id'], user) + _create_version(test_dataset['id'], user) pytest.fail("Couldn't find user with required role %s", user_role) def test_should_not_create_data_version_with_same_name(self, test_dataset, org_editor): version_name = "Not unique name" - self._create_version(test_dataset['id'], org_editor, version_name=version_name) + _create_version(test_dataset['id'], org_editor, version_name=version_name) with pytest.raises(toolkit.ValidationError): - self._create_version(test_dataset['id'], org_editor, version_name=version_name) + _create_version(test_dataset['id'], org_editor, version_name=version_name) def test_should_fail_if_dataset_not_exists(self, org_editor): with pytest.raises(toolkit.ObjectNotFound) as e: - self._create_version('fake_dataset_id', org_editor) + _create_version('fake_dataset_id', org_editor) assert e.msg == "Dataset not found" def test_version_has_valid_activity_id(self, test_organization, org_editor): old_name = "Initial name" dataset = factories.Dataset(name=old_name, owner_org=test_organization['id']) - version = self._create_version(dataset['id'], org_editor) + version = _create_version(dataset['id'], org_editor) context = get_context(org_editor) toolkit.get_action('package_patch')( context, @@ -85,26 +85,26 @@ def test_new_dataset_has_no_versions(self, test_dataset, org_editor): def test_dataset_has_versions_after_one_created(self, test_dataset, org_editor): context = get_context(org_editor) - self._create_version(test_dataset['id'], org_editor) + _create_version(test_dataset['id'], org_editor) assert True == dataset_has_versions( context, {'id': test_dataset['id']} ) - def _create_version(self, dataset_id, user, version_name="Default Name"): - return dataset_version_create( - get_context(user), - { - "dataset_id": dataset_id, - "name": version_name - } - ) +def _create_version(dataset_id, user, version_name="Default Name"): + return dataset_version_create( + get_context(user), + { + "dataset_id": dataset_id, + "name": version_name + } + ) - def _assert_version(self, version, checks): - assert version - for k, v in checks: - assert version[k] == v +def _assert_version(version, checks): + assert version + for k, v in checks: + assert version[k] == v @pytest.fixture() From fdda61b900247cdfa8a5161b3a06fd909d19eb1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Wed, 26 May 2021 10:17:49 +0200 Subject: [PATCH 04/29] Added missing test stubs. Renamed some test methods. --- .../tests/test_dataset_version_action.py | 74 +++++++++++++++++-- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py index bdf9a85..123d72c 100644 --- a/ckanext/versions/tests/test_dataset_version_action.py +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -2,14 +2,15 @@ from ckan.plugins import toolkit from ckan.tests import factories -from ckanext.versions.logic.dataset_version_action import dataset_version_create, dataset_has_versions +from ckanext.versions.logic.dataset_version_action import dataset_version_create, dataset_has_versions, \ + get_activity_id_from_dataset_version_name, activity_dataset_show from ckanext.versions.tests import get_context @pytest.mark.usefixtures('clean_db', 'versions_setup') class TestDatasetVersion(object): - def test_data_version_create(self, org_admin, test_dataset): + def test_dataset_version_create_should_create_version(self, org_admin, test_dataset): version_name = "Test Version 1.0" version_notes = "Some details about the version" version = dataset_version_create( @@ -33,7 +34,7 @@ def test_data_version_create(self, org_admin, test_dataset): ('editor', True), ('member', False), ]) - def test_auth_version_create(self, test_organization, test_dataset, user_role, can_create_version): + def test_dataset_version_create_auth(self, test_organization, test_dataset, user_role, can_create_version): for user in test_organization['users']: if user['capacity'] == user_role: if can_create_version: @@ -43,18 +44,18 @@ def test_auth_version_create(self, test_organization, test_dataset, user_role, c _create_version(test_dataset['id'], user) pytest.fail("Couldn't find user with required role %s", user_role) - def test_should_not_create_data_version_with_same_name(self, test_dataset, org_editor): + def test_dataset_version_create_should_not_create_version_with_same_name(self, test_dataset, org_editor): version_name = "Not unique name" _create_version(test_dataset['id'], org_editor, version_name=version_name) with pytest.raises(toolkit.ValidationError): _create_version(test_dataset['id'], org_editor, version_name=version_name) - def test_should_fail_if_dataset_not_exists(self, org_editor): + def test_dataset_version_create_should_fail_if_dataset_not_exists(self, org_editor): with pytest.raises(toolkit.ObjectNotFound) as e: _create_version('fake_dataset_id', org_editor) assert e.msg == "Dataset not found" - def test_version_has_valid_activity_id(self, test_organization, org_editor): + def test_dataset_version_create_returns_valid_activity_id(self, test_organization, org_editor): old_name = "Initial name" dataset = factories.Dataset(name=old_name, owner_org=test_organization['id']) version = _create_version(dataset['id'], org_editor) @@ -75,6 +76,66 @@ def test_version_has_valid_activity_id(self, test_organization, org_editor): assert old_dataset['name'] == old_name assert old_dataset['id'] == dataset['id'] + def test_dataset_version_latest_show_latest_version(self): + pass + + def test_dataset_version_latest_raises_when_dataset_not_found(self): + pass + + def test_dataset_version_latest_raises_when_dataset_has_no_versions(self): + pass + + def test_activity_dataset_show_returns_correct_dataset(self, test_dataset, org_editor): + version = _create_version(test_dataset['id'], org_editor) + context = get_context(org_editor) + updated_name = "Updated Name" + new_dataset = toolkit.get_action('package_patch')( + context, + { + "id": test_dataset['id'], + "name": updated_name + } + ) + + old_dataset = activity_dataset_show( + context, + { + 'dataset_id': new_dataset['id'], + 'activity_id': version['activity_id'] + } + ) + + assert test_dataset['name'] == old_dataset['name'] + assert new_dataset['id'] == old_dataset['id'] + + def test_get_activity_id_from_dataset_version_returns_correct(self, test_dataset, org_editor): + version = _create_version(test_dataset['id'], org_editor) + expected_activity_id = version['activity_id'] + + version = _create_version(test_dataset['id'], org_editor) + + context = get_context(org_editor) + actual_activity_id = get_activity_id_from_dataset_version_name( + context, + { + 'dataset_id': test_dataset['id'], + 'version': version['name'] + } + ) + + assert expected_activity_id == actual_activity_id + + def test_get_activity_id_from_dataset_version_raises_not_found(self, org_editor): + context = get_context(org_editor) + with pytest.raises(toolkit.ObjectNotFound) as e: + get_activity_id_from_dataset_version_name( + context, + { + 'dataset_id': test_dataset['id'], + 'version': 'Fake version name' + } + ) + assert "Version not found" == e.msg def test_new_dataset_has_no_versions(self, test_dataset, org_editor): context = get_context(org_editor) @@ -101,6 +162,7 @@ def _create_version(dataset_id, user, version_name="Default Name"): } ) + def _assert_version(version, checks): assert version for k, v in checks: From 8945197fc1728d51f5cdf13be1a20c76d426fdf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Sun, 30 May 2021 11:11:33 +0200 Subject: [PATCH 05/29] Added tests implementation for dataset version latest. --- .../tests/test_dataset_version_action.py | 56 ++++++++++++++++--- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py index 123d72c..d390b8f 100644 --- a/ckanext/versions/tests/test_dataset_version_action.py +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -2,8 +2,13 @@ from ckan.plugins import toolkit from ckan.tests import factories -from ckanext.versions.logic.dataset_version_action import dataset_version_create, dataset_has_versions, \ - get_activity_id_from_dataset_version_name, activity_dataset_show +from ckanext.versions.logic.dataset_version_action import ( + dataset_version_create, + dataset_has_versions, + get_activity_id_from_dataset_version_name, + activity_dataset_show, + dataset_version_latest +) from ckanext.versions.tests import get_context @@ -76,14 +81,49 @@ def test_dataset_version_create_returns_valid_activity_id(self, test_organizatio assert old_dataset['name'] == old_name assert old_dataset['id'] == dataset['id'] - def test_dataset_version_latest_show_latest_version(self): - pass + def test_dataset_version_latest_show_latest_version(self, test_dataset, org_editor): + context = get_context(org_editor) + version1 = _create_version(test_dataset['id'], org_editor, version_name="Version1") + toolkit.get_action('package_patch')( + context, + { + "id": test_dataset['id'], + "name": "Updated Name" + } + ) + version2 = _create_version(test_dataset['id'], org_editor, version_name="Version2") + + latest_version = dataset_version_latest( + context, + { + 'dataset_id': test_dataset['id'] + } + ) - def test_dataset_version_latest_raises_when_dataset_not_found(self): - pass + assert version1['id'] != latest_version['id'] + _assert_version(latest_version, {'id': version2['id'], 'name': version2['name']}) - def test_dataset_version_latest_raises_when_dataset_has_no_versions(self): - pass + def test_dataset_version_latest_raises_when_dataset_not_found(self, org_editor): + context = get_context(org_editor) + with pytest.raises(toolkit.ObjectNotFound) as e: + dataset_version_latest( + context, + { + 'dataset_id': 'fake-dataset-id' + } + ) + assert "Dataset not found" == e.msg + + def test_dataset_version_latest_raises_when_dataset_has_no_versions(self, test_dataset, org_editor): + context = get_context(org_editor) + with pytest.raises(toolkit.ObjectNotFound) as e: + dataset_version_latest( + context, + { + 'dataset_id': test_dataset['id'] + } + ) + assert "Versions not found in the dataset" == e.msg def test_activity_dataset_show_returns_correct_dataset(self, test_dataset, org_editor): version = _create_version(test_dataset['id'], org_editor) From 32d88058736f6ea3715e879cdeeb80ea4e074200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Sun, 30 May 2021 12:14:27 +0200 Subject: [PATCH 06/29] Fixed some minor bugs in the tests. --- .../versions/tests/test_dataset_version_action.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py index d390b8f..05b28ba 100644 --- a/ckanext/versions/tests/test_dataset_version_action.py +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -44,9 +44,11 @@ def test_dataset_version_create_auth(self, test_organization, test_dataset, user if user['capacity'] == user_role: if can_create_version: _create_version(test_dataset['id'], user) + return else: with pytest.raises(toolkit.NotAuthorized): _create_version(test_dataset['id'], user) + return pytest.fail("Couldn't find user with required role %s", user_role) def test_dataset_version_create_should_not_create_version_with_same_name(self, test_dataset, org_editor): @@ -61,7 +63,7 @@ def test_dataset_version_create_should_fail_if_dataset_not_exists(self, org_edit assert e.msg == "Dataset not found" def test_dataset_version_create_returns_valid_activity_id(self, test_organization, org_editor): - old_name = "Initial name" + old_name = "initial-name" dataset = factories.Dataset(name=old_name, owner_org=test_organization['id']) version = _create_version(dataset['id'], org_editor) context = get_context(org_editor) @@ -69,7 +71,7 @@ def test_dataset_version_create_returns_valid_activity_id(self, test_organizatio context, { "id": dataset['id'], - "name": "Updated Name" + "name": "updated-name" } ) @@ -205,23 +207,23 @@ def _create_version(dataset_id, user, version_name="Default Name"): def _assert_version(version, checks): assert version - for k, v in checks: + for k, v in checks.items(): assert version[k] == v @pytest.fixture() def org_admin(): - return factories.User() + return factories.User(name="admin") @pytest.fixture() def org_editor(): - return factories.User() + return factories.User(name="editor") @pytest.fixture() def org_member(): - return factories.User() + return factories.User(name="member") @pytest.fixture() From 42ccd3c55ffa0af5e78c2cf01a78862979012288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Sun, 30 May 2021 12:15:17 +0200 Subject: [PATCH 07/29] WIP: Removed ForeignKey constraint for resource_id to allow nullable values. --- ckanext/versions/model.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ckanext/versions/model.py b/ckanext/versions/model.py index 89dae94..1357c55 100644 --- a/ckanext/versions/model.py +++ b/ckanext/versions/model.py @@ -23,7 +23,10 @@ class Version(Base): id = Column(UuidType, primary_key=True, default=UuidType.default) package_id = Column(UuidType, ForeignKey('package.id'), nullable=False) - resource_id = Column(UuidType, ForeignKey('resource.id'), nullable=True) + # TODO: How to make resource_id foreign key and also nullable? + # Currenlty throws IntegrityError Key(resource_id)=(None) is not present + # resource_id = Column(UuidType, ForeignKey('resource.id'), nullable=True) + resource_id = Column(UuidType, nullable=True) activity_id = Column(UuidType, ForeignKey('activity.id'), nullable=False) name = Column(Unicode, nullable=False) notes = Column(Unicode, nullable=True) From e68de9508b010af0761e2d71e08bb72348dff3e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Sun, 30 May 2021 12:15:35 +0200 Subject: [PATCH 08/29] dataset_version_create implementation --- .../versions/logic/dataset_version_action.py | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/ckanext/versions/logic/dataset_version_action.py b/ckanext/versions/logic/dataset_version_action.py index 1719cd0..f454a68 100644 --- a/ckanext/versions/logic/dataset_version_action.py +++ b/ckanext/versions/logic/dataset_version_action.py @@ -1,5 +1,13 @@ +import logging + +from datetime import datetime +from sqlalchemy.exc import IntegrityError + +from ckan import model as core_model from ckan.plugins import toolkit +from ckanext.versions.model import Version +log = logging.getLogger(__name__) def dataset_version_create(context, data_dict): """Create a new version from the current dataset's activity_id @@ -17,7 +25,55 @@ def dataset_version_create(context, data_dict): :returns: the newly created version :rtype: dictionary """ - pass + model = context.get('model', core_model) + dataset_id, name = toolkit.get_or_bust( + data_dict, ['dataset_id', 'name']) + + dataset = model.Package.get(dataset_id) + if not dataset: + raise toolkit.ObjectNotFound("Dataset not found") + + toolkit.check_access('version_create', + context, + {"package_id": dataset_id} ) + creator_user_id = context['auth_user_obj'].id + + # TODO: add support for specific activity_id in data_dict + activity = model.Session.query(model.Activity). \ + filter_by(object_id=dataset_id). \ + order_by(model.Activity.timestamp.desc()). \ + first() + + if not activity: + # TODO: Add test for this exception + raise toolkit.ObjectNotFound('Activity not found') + + version = Version( + package_id=dataset_id, + activity_id=activity.id, + name=name, + notes=data_dict.get('notes', None), + created=datetime.utcnow(), + creator_user_id=creator_user_id) + + model.Session.add(version) + try: + model.Session.commit() + except IntegrityError as e: + # Name not unique, or foreign key constraint violated + model.Session.rollback() + log.debug("DB integrity error (version name not unique?): %s", e) + raise toolkit.ValidationError( + 'Version names must be unique per dataset' + ) + + log.info( + 'Version "%s" created for dataset %s', + name, + dataset_id + ) + + return version.as_dict() @toolkit.side_effect_free From d14e2fb7ea4714372a62bd3a0860a5f8e838af93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Sun, 30 May 2021 12:41:13 +0200 Subject: [PATCH 09/29] dataset_version_list implementation --- .../versions/logic/dataset_version_action.py | 15 +++++- .../tests/test_dataset_version_action.py | 47 ++++++++++++++++++- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/ckanext/versions/logic/dataset_version_action.py b/ckanext/versions/logic/dataset_version_action.py index f454a68..225eb5d 100644 --- a/ckanext/versions/logic/dataset_version_action.py +++ b/ckanext/versions/logic/dataset_version_action.py @@ -85,7 +85,20 @@ def dataset_version_list(context, data_dict): :returns: list of versions created for the dataset :rtype: list """ - pass + model = context.get('model', core_model) + dataset_id = toolkit.get_or_bust(data_dict, ['dataset_id']) + dataset = model.Package.get(dataset_id) + if not dataset: + raise toolkit.ObjectNotFound('Dataset not found') + + toolkit.check_access('version_list', context, + {"package_id": dataset_id}) + + versions = model.Session.query(Version). \ + filter(Version.package_id == dataset.id). \ + order_by(Version.created.desc()) + + return [v.as_dict() for v in versions] @toolkit.side_effect_free diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py index 05b28ba..e235982 100644 --- a/ckanext/versions/tests/test_dataset_version_action.py +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -7,7 +7,7 @@ dataset_has_versions, get_activity_id_from_dataset_version_name, activity_dataset_show, - dataset_version_latest + dataset_version_latest, dataset_version_list ) from ckanext.versions.tests import get_context @@ -83,6 +83,49 @@ def test_dataset_version_create_returns_valid_activity_id(self, test_organizatio assert old_dataset['name'] == old_name assert old_dataset['id'] == dataset['id'] + def test_dataset_version_list_return_all_version(self, test_dataset, org_editor): + context = get_context(org_editor) + version1 = _create_version(test_dataset['id'], org_editor, version_name="Version1") + toolkit.get_action('package_patch')( + context, + { + "id": test_dataset['id'], + "name": "updated-name" + } + ) + version2 = _create_version(test_dataset['id'], org_editor, version_name="Version2") + + version_list = dataset_version_list( + context, + { + 'dataset_id': test_dataset['id'] + } + ) + version_ids = [v['id'] for v in version_list] + assert version1['id'] in version_ids + assert version2['id'] in version_ids + + def test_dataset_version_should_fail_if_dataset_not_exists(self, org_editor): + context = get_context(org_editor) + with pytest.raises(toolkit.ObjectNotFound) as e: + dataset_version_list( + context, + { + 'dataset_id': 'fake-dataset-id' + } + ) + assert "Dataset not found" == e.msg + + def test_dataset_version_should_return_empty_list_if_dataset_no_versions(self, test_dataset, org_editor): + context = get_context(org_editor) + versions_list = dataset_version_list( + context, + { + 'dataset_id': test_dataset['id'] + } + ) + assert [] == versions_list + def test_dataset_version_latest_show_latest_version(self, test_dataset, org_editor): context = get_context(org_editor) version1 = _create_version(test_dataset['id'], org_editor, version_name="Version1") @@ -90,7 +133,7 @@ def test_dataset_version_latest_show_latest_version(self, test_dataset, org_edit context, { "id": test_dataset['id'], - "name": "Updated Name" + "name": "updated-name" } ) version2 = _create_version(test_dataset['id'], org_editor, version_name="Version2") From 1cd8ade5ee1bebd23e9f8716b1f42964e7fdaf75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Sun, 30 May 2021 12:43:43 +0200 Subject: [PATCH 10/29] dataset_version_latest implementation --- ckanext/versions/logic/dataset_version_action.py | 5 ++++- ckanext/versions/tests/test_dataset_version_action.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ckanext/versions/logic/dataset_version_action.py b/ckanext/versions/logic/dataset_version_action.py index 225eb5d..20e3a55 100644 --- a/ckanext/versions/logic/dataset_version_action.py +++ b/ckanext/versions/logic/dataset_version_action.py @@ -110,7 +110,10 @@ def dataset_version_latest(context, data_dict): :returns the version dictionary :rtype dict ''' - pass + version_list = dataset_version_list(context, data_dict) + if len(version_list) < 1: + raise toolkit.ObjectNotFound("Versions not found for this dataset") + return version_list[0] def activity_dataset_show(context, data_dict): diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py index e235982..68cf62b 100644 --- a/ckanext/versions/tests/test_dataset_version_action.py +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -168,7 +168,7 @@ def test_dataset_version_latest_raises_when_dataset_has_no_versions(self, test_d 'dataset_id': test_dataset['id'] } ) - assert "Versions not found in the dataset" == e.msg + assert "Versions not found for this dataset" == e.msg def test_activity_dataset_show_returns_correct_dataset(self, test_dataset, org_editor): version = _create_version(test_dataset['id'], org_editor) From 30b53c6a8cb84e3b2b6940a5e1e752b807ce89ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Sun, 30 May 2021 12:55:05 +0200 Subject: [PATCH 11/29] activity_dataset_show implementation --- .../versions/logic/dataset_version_action.py | 13 ++++++++++- .../tests/test_dataset_version_action.py | 23 +++++++++++++++---- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/ckanext/versions/logic/dataset_version_action.py b/ckanext/versions/logic/dataset_version_action.py index 20e3a55..06d58bf 100644 --- a/ckanext/versions/logic/dataset_version_action.py +++ b/ckanext/versions/logic/dataset_version_action.py @@ -126,7 +126,18 @@ def activity_dataset_show(context, data_dict): :returns: The dataset in the activity :rtype: dict ''' - pass + activity_id, dataset_id = toolkit.get_or_bust( + data_dict, + ['activity_id', 'dataset_id'] + ) + dataset = toolkit.get_action('activity_data_show')( + context, + {'id': activity_id, 'object_type': 'package'} + ) + if not dataset or dataset['id'] != dataset_id: + raise toolkit.ObjectNotFound('Dataset not found in the activity object.') + + return dataset def get_activity_id_from_dataset_version_name(context, data_dict): diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py index 68cf62b..d0414a8 100644 --- a/ckanext/versions/tests/test_dataset_version_action.py +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -173,7 +173,7 @@ def test_dataset_version_latest_raises_when_dataset_has_no_versions(self, test_d def test_activity_dataset_show_returns_correct_dataset(self, test_dataset, org_editor): version = _create_version(test_dataset['id'], org_editor) context = get_context(org_editor) - updated_name = "Updated Name" + updated_name = "updated-name" new_dataset = toolkit.get_action('package_patch')( context, { @@ -193,18 +193,31 @@ def test_activity_dataset_show_returns_correct_dataset(self, test_dataset, org_e assert test_dataset['name'] == old_dataset['name'] assert new_dataset['id'] == old_dataset['id'] - def test_get_activity_id_from_dataset_version_returns_correct(self, test_dataset, org_editor): + def test_activity_dataset_show_fails_if_no_dataset_in_activity(self, test_dataset, org_editor): version = _create_version(test_dataset['id'], org_editor) - expected_activity_id = version['activity_id'] + context = get_context(org_editor) + with pytest.raises(toolkit.ObjectNotFound) as e: + activity_dataset_show( + context, + { + 'dataset_id': 'fake-dataset-id', + 'activity_id': version['activity_id'] + } + ) + assert 'Dataset not found in the activity object.' == e.msg - version = _create_version(test_dataset['id'], org_editor) + def test_get_activity_id_from_dataset_version_returns_correct(self, test_dataset, org_editor): + version1 = _create_version(test_dataset['id'], org_editor, version_name="Version1") + expected_activity_id = version1['activity_id'] + + _create_version(test_dataset['id'], org_editor, version_name="Version2") context = get_context(org_editor) actual_activity_id = get_activity_id_from_dataset_version_name( context, { 'dataset_id': test_dataset['id'], - 'version': version['name'] + 'version': version1['name'] } ) From 54961ec4c82f8704a680487fa621216b24d6aca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Sun, 30 May 2021 13:02:07 +0200 Subject: [PATCH 12/29] get_activity_id_from_dataset_version_name implementation --- ckanext/versions/logic/dataset_version_action.py | 14 +++++++++++--- .../versions/tests/test_dataset_version_action.py | 8 ++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/ckanext/versions/logic/dataset_version_action.py b/ckanext/versions/logic/dataset_version_action.py index 06d58bf..a450f6c 100644 --- a/ckanext/versions/logic/dataset_version_action.py +++ b/ckanext/versions/logic/dataset_version_action.py @@ -143,15 +143,23 @@ def activity_dataset_show(context, data_dict): def get_activity_id_from_dataset_version_name(context, data_dict): ''' Returns the activity_id for the dataset version - :param dataset_id: the id of the resource + :param dataset_id: the id of the dataset :type dataset_id: string :param version: the name or id of the version :type version: string :returns: The activity_id of the version :rtype: string - ''' - pass + version_name, dataset_id = toolkit.get_or_bust( + data_dict, + ['version', 'dataset_id'] + ) + version_list = dataset_version_list(context, data_dict) + for version in version_list: + if version_name in {version['name'], version['id']}: + return version['activity_id'] + + raise toolkit.ObjectNotFound('Version not found in the dataset.') def dataset_has_versions(context, data_dict): diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py index d0414a8..1a9d9e7 100644 --- a/ckanext/versions/tests/test_dataset_version_action.py +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -223,7 +223,7 @@ def test_get_activity_id_from_dataset_version_returns_correct(self, test_dataset assert expected_activity_id == actual_activity_id - def test_get_activity_id_from_dataset_version_raises_not_found(self, org_editor): + def test_get_activity_id_from_dataset_version_raises_not_found(self, test_dataset, org_editor): context = get_context(org_editor) with pytest.raises(toolkit.ObjectNotFound) as e: get_activity_id_from_dataset_version_name( @@ -233,11 +233,11 @@ def test_get_activity_id_from_dataset_version_raises_not_found(self, org_editor) 'version': 'Fake version name' } ) - assert "Version not found" == e.msg + assert "Version not found in the dataset" == e.msg def test_new_dataset_has_no_versions(self, test_dataset, org_editor): context = get_context(org_editor) - assert False == dataset_has_versions( + assert False is dataset_has_versions( context, {'id': test_dataset['id']} ) @@ -245,7 +245,7 @@ def test_new_dataset_has_no_versions(self, test_dataset, org_editor): def test_dataset_has_versions_after_one_created(self, test_dataset, org_editor): context = get_context(org_editor) _create_version(test_dataset['id'], org_editor) - assert True == dataset_has_versions( + assert True is dataset_has_versions( context, {'id': test_dataset['id']} ) From 304205b525c38962f9730f5a5e09e57ecd4c173c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Sun, 30 May 2021 13:04:30 +0200 Subject: [PATCH 13/29] dataset_has_versions implementation --- ckanext/versions/logic/dataset_version_action.py | 5 ++++- ckanext/versions/tests/test_dataset_version_action.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ckanext/versions/logic/dataset_version_action.py b/ckanext/versions/logic/dataset_version_action.py index a450f6c..b9e23ce 100644 --- a/ckanext/versions/logic/dataset_version_action.py +++ b/ckanext/versions/logic/dataset_version_action.py @@ -170,4 +170,7 @@ def dataset_has_versions(context, data_dict): :returns: True if the dataset has at least 1 version :rtype: boolean """ - pass + version_list = dataset_version_list(context, data_dict) + if not version_list: + return False + return True diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py index 1a9d9e7..ca92bc7 100644 --- a/ckanext/versions/tests/test_dataset_version_action.py +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -239,7 +239,7 @@ def test_new_dataset_has_no_versions(self, test_dataset, org_editor): context = get_context(org_editor) assert False is dataset_has_versions( context, - {'id': test_dataset['id']} + {'dataset_id': test_dataset['id']} ) def test_dataset_has_versions_after_one_created(self, test_dataset, org_editor): @@ -247,7 +247,7 @@ def test_dataset_has_versions_after_one_created(self, test_dataset, org_editor): _create_version(test_dataset['id'], org_editor) assert True is dataset_has_versions( context, - {'id': test_dataset['id']} + {'dataset_id': test_dataset['id']} ) From baf5a4326d96d7a5790d8a4424eabaa3742872b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Sun, 30 May 2021 13:23:38 +0200 Subject: [PATCH 14/29] Defined dataset version actions in plugin def. --- ckanext/versions/plugin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ckanext/versions/plugin.py b/ckanext/versions/plugin.py index 2a89ed0..41ff3e2 100644 --- a/ckanext/versions/plugin.py +++ b/ckanext/versions/plugin.py @@ -6,7 +6,7 @@ from ckanext.versions import cli, helpers from ckanext.versions.blueprints import blueprint -from ckanext.versions.logic import action, auth +from ckanext.versions.logic import action, auth, dataset_version_action from ckanext.versions.model import tables_exist log = logging.getLogger(__name__) @@ -52,6 +52,9 @@ def get_actions(self): 'version_show': action.version_show, 'version_delete': action.version_delete, 'resource_view_list': action.resource_view_list, + 'dataset_version_create': dataset_version_action.dataset_version_create, + 'dataset_version_list': dataset_version_action.dataset_version_list, + 'dataset_version_latest': dataset_version_action.dataset_version_latest } # IAuthFunctions From 6339ef16a3298d5a0e61c0f77eba30fae09d3a16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Mon, 31 May 2021 09:01:35 +0200 Subject: [PATCH 15/29] Support for custom activity_id in dataset_version_create --- .../versions/logic/dataset_version_action.py | 14 ++--- .../tests/test_dataset_version_action.py | 52 ++++++++++++++++++- 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/ckanext/versions/logic/dataset_version_action.py b/ckanext/versions/logic/dataset_version_action.py index b9e23ce..2a4ec6c 100644 --- a/ckanext/versions/logic/dataset_version_action.py +++ b/ckanext/versions/logic/dataset_version_action.py @@ -28,6 +28,7 @@ def dataset_version_create(context, data_dict): model = context.get('model', core_model) dataset_id, name = toolkit.get_or_bust( data_dict, ['dataset_id', 'name']) + activity_id = data_dict.get('activity_id') dataset = model.Package.get(dataset_id) if not dataset: @@ -38,14 +39,15 @@ def dataset_version_create(context, data_dict): {"package_id": dataset_id} ) creator_user_id = context['auth_user_obj'].id - # TODO: add support for specific activity_id in data_dict - activity = model.Session.query(model.Activity). \ - filter_by(object_id=dataset_id). \ - order_by(model.Activity.timestamp.desc()). \ - first() + if activity_id: + activity = model.Activity.get(activity_id) + else: + activity = model.Session.query(model.Activity). \ + filter_by(object_id=dataset_id). \ + order_by(model.Activity.timestamp.desc()). \ + first() if not activity: - # TODO: Add test for this exception raise toolkit.ObjectNotFound('Activity not found') version = Version( diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py index ca92bc7..c0bdc08 100644 --- a/ckanext/versions/tests/test_dataset_version_action.py +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -34,6 +34,56 @@ def test_dataset_version_create_should_create_version(self, org_admin, test_data _assert_version(version, checks) + def test_dataset_create_should_create_version_for_given_activity(self, test_dataset, org_editor): + version_name = "V1.0" + context = get_context(org_editor) + for i in range(2): + toolkit.get_action('package_patch')( + context, + { + "id": test_dataset['id'], + "name": "updated-name%s" % i + } + ) + activities = toolkit.get_action('package_activity_list')( + context, + { + 'id': test_dataset['id'] + } + ) + past_activity_id = activities[-1]['id'] + + version = dataset_version_create( + context, + { + "dataset_id": test_dataset['id'], + "name": version_name, + "activity_id": past_activity_id + } + ) + + checks = {'package_id': test_dataset['id'], + 'resource_id': None, + 'notes': None, + 'name': version_name, + 'activity_id': past_activity_id, + 'creator_user_id': org_editor['id']} + + _assert_version(version, checks) + + def test_dataset_create_should_fail_when_incorrect_activity_id(self, test_dataset, org_editor): + context = get_context(org_editor) + with pytest.raises(toolkit.ObjectNotFound) as e: + dataset_version_create( + context, + { + "dataset_id": test_dataset['id'], + "name": "V1.0", + "activity_id": "fake-activity-id" + } + ) + assert "Activity not found" == e.msg + @pytest.mark.parametrize("user_role, can_create_version", [ ('admin', True), ('editor', True), @@ -264,7 +314,7 @@ def _create_version(dataset_id, user, version_name="Default Name"): def _assert_version(version, checks): assert version for k, v in checks.items(): - assert version[k] == v + assert version[k] == v, "found incorrect %s of version" % k @pytest.fixture() From 82ab9c4cce61ae77c2f04470a5a67762362d0b1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Mon, 31 May 2021 13:34:36 +0200 Subject: [PATCH 16/29] Extracted fixtures to fixtures.py and common test helpers to __init__.py --- ckanext/versions/tests/__init__.py | 17 ++++ ckanext/versions/tests/fixtures.py | 34 +++++++ .../tests/test_dataset_version_action.py | 88 ++++--------------- 3 files changed, 70 insertions(+), 69 deletions(-) diff --git a/ckanext/versions/tests/__init__.py b/ckanext/versions/tests/__init__.py index 11e7b86..0b3867a 100644 --- a/ckanext/versions/tests/__init__.py +++ b/ckanext/versions/tests/__init__.py @@ -1,4 +1,5 @@ from ckan import model +from ckanext.versions.logic.dataset_version_action import dataset_version_create def get_context(user): @@ -6,3 +7,19 @@ def get_context(user): 'model': model, 'user': user if isinstance(user, str) else user['name'] } + + +def assert_version(version, checks): + assert version + for k, v in checks.items(): + assert version[k] == v, "found incorrect %s of version" % k + + +def create_version(dataset_id, user, version_name="Default Name"): + return dataset_version_create( + get_context(user), + { + "dataset_id": dataset_id, + "name": version_name + } + ) \ No newline at end of file diff --git a/ckanext/versions/tests/fixtures.py b/ckanext/versions/tests/fixtures.py index 460aa19..53b7e44 100644 --- a/ckanext/versions/tests/fixtures.py +++ b/ckanext/versions/tests/fixtures.py @@ -1,5 +1,6 @@ import pytest +from ckan.tests import factories from ckanext.versions.model import create_tables, tables_exist @@ -7,3 +8,36 @@ def versions_setup(): if not tables_exist(): create_tables() + +@pytest.fixture() +def org_admin(): + return factories.User(name="admin") + + +@pytest.fixture() +def org_editor(): + return factories.User(name="editor") + + +@pytest.fixture() +def org_member(): + return factories.User(name="member") + + +@pytest.fixture() +def test_organization(org_admin, org_editor, org_member): + return factories.Organization(users=[ + {'name': org_admin['id'], 'capacity': 'admin'}, + {'name': org_editor['id'], 'capacity': 'editor'}, + {'name': org_member['id'], 'capacity': 'member'} + ]) + + +@pytest.fixture() +def test_dataset(test_organization): + return factories.Dataset(owner_org=test_organization['id']) + + +@pytest.fixture() +def test_resource(test_dataset): + return factories.Resource(package_id=test_dataset['id']) diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py index c0bdc08..a2f0f19 100644 --- a/ckanext/versions/tests/test_dataset_version_action.py +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -9,7 +9,7 @@ activity_dataset_show, dataset_version_latest, dataset_version_list ) -from ckanext.versions.tests import get_context +from ckanext.versions.tests import get_context, assert_version, create_version @pytest.mark.usefixtures('clean_db', 'versions_setup') @@ -32,7 +32,7 @@ def test_dataset_version_create_should_create_version(self, org_admin, test_data 'name': version_name, 'creator_user_id': org_admin['id']} - _assert_version(version, checks) + assert_version(version, checks) def test_dataset_create_should_create_version_for_given_activity(self, test_dataset, org_editor): version_name = "V1.0" @@ -69,7 +69,7 @@ def test_dataset_create_should_create_version_for_given_activity(self, test_data 'activity_id': past_activity_id, 'creator_user_id': org_editor['id']} - _assert_version(version, checks) + assert_version(version, checks) def test_dataset_create_should_fail_when_incorrect_activity_id(self, test_dataset, org_editor): context = get_context(org_editor) @@ -93,29 +93,29 @@ def test_dataset_version_create_auth(self, test_organization, test_dataset, user for user in test_organization['users']: if user['capacity'] == user_role: if can_create_version: - _create_version(test_dataset['id'], user) + create_version(test_dataset['id'], user) return else: with pytest.raises(toolkit.NotAuthorized): - _create_version(test_dataset['id'], user) + create_version(test_dataset['id'], user) return pytest.fail("Couldn't find user with required role %s", user_role) def test_dataset_version_create_should_not_create_version_with_same_name(self, test_dataset, org_editor): version_name = "Not unique name" - _create_version(test_dataset['id'], org_editor, version_name=version_name) + create_version(test_dataset['id'], org_editor, version_name=version_name) with pytest.raises(toolkit.ValidationError): - _create_version(test_dataset['id'], org_editor, version_name=version_name) + create_version(test_dataset['id'], org_editor, version_name=version_name) def test_dataset_version_create_should_fail_if_dataset_not_exists(self, org_editor): with pytest.raises(toolkit.ObjectNotFound) as e: - _create_version('fake_dataset_id', org_editor) + create_version('fake_dataset_id', org_editor) assert e.msg == "Dataset not found" def test_dataset_version_create_returns_valid_activity_id(self, test_organization, org_editor): old_name = "initial-name" dataset = factories.Dataset(name=old_name, owner_org=test_organization['id']) - version = _create_version(dataset['id'], org_editor) + version = create_version(dataset['id'], org_editor) context = get_context(org_editor) toolkit.get_action('package_patch')( context, @@ -135,7 +135,7 @@ def test_dataset_version_create_returns_valid_activity_id(self, test_organizatio def test_dataset_version_list_return_all_version(self, test_dataset, org_editor): context = get_context(org_editor) - version1 = _create_version(test_dataset['id'], org_editor, version_name="Version1") + version1 = create_version(test_dataset['id'], org_editor, version_name="Version1") toolkit.get_action('package_patch')( context, { @@ -143,7 +143,7 @@ def test_dataset_version_list_return_all_version(self, test_dataset, org_editor) "name": "updated-name" } ) - version2 = _create_version(test_dataset['id'], org_editor, version_name="Version2") + version2 = create_version(test_dataset['id'], org_editor, version_name="Version2") version_list = dataset_version_list( context, @@ -178,7 +178,7 @@ def test_dataset_version_should_return_empty_list_if_dataset_no_versions(self, t def test_dataset_version_latest_show_latest_version(self, test_dataset, org_editor): context = get_context(org_editor) - version1 = _create_version(test_dataset['id'], org_editor, version_name="Version1") + version1 = create_version(test_dataset['id'], org_editor, version_name="Version1") toolkit.get_action('package_patch')( context, { @@ -186,7 +186,7 @@ def test_dataset_version_latest_show_latest_version(self, test_dataset, org_edit "name": "updated-name" } ) - version2 = _create_version(test_dataset['id'], org_editor, version_name="Version2") + version2 = create_version(test_dataset['id'], org_editor, version_name="Version2") latest_version = dataset_version_latest( context, @@ -196,7 +196,7 @@ def test_dataset_version_latest_show_latest_version(self, test_dataset, org_edit ) assert version1['id'] != latest_version['id'] - _assert_version(latest_version, {'id': version2['id'], 'name': version2['name']}) + assert_version(latest_version, {'id': version2['id'], 'name': version2['name']}) def test_dataset_version_latest_raises_when_dataset_not_found(self, org_editor): context = get_context(org_editor) @@ -221,7 +221,7 @@ def test_dataset_version_latest_raises_when_dataset_has_no_versions(self, test_d assert "Versions not found for this dataset" == e.msg def test_activity_dataset_show_returns_correct_dataset(self, test_dataset, org_editor): - version = _create_version(test_dataset['id'], org_editor) + version = create_version(test_dataset['id'], org_editor) context = get_context(org_editor) updated_name = "updated-name" new_dataset = toolkit.get_action('package_patch')( @@ -244,7 +244,7 @@ def test_activity_dataset_show_returns_correct_dataset(self, test_dataset, org_e assert new_dataset['id'] == old_dataset['id'] def test_activity_dataset_show_fails_if_no_dataset_in_activity(self, test_dataset, org_editor): - version = _create_version(test_dataset['id'], org_editor) + version = create_version(test_dataset['id'], org_editor) context = get_context(org_editor) with pytest.raises(toolkit.ObjectNotFound) as e: activity_dataset_show( @@ -257,10 +257,10 @@ def test_activity_dataset_show_fails_if_no_dataset_in_activity(self, test_datase assert 'Dataset not found in the activity object.' == e.msg def test_get_activity_id_from_dataset_version_returns_correct(self, test_dataset, org_editor): - version1 = _create_version(test_dataset['id'], org_editor, version_name="Version1") + version1 = create_version(test_dataset['id'], org_editor, version_name="Version1") expected_activity_id = version1['activity_id'] - _create_version(test_dataset['id'], org_editor, version_name="Version2") + create_version(test_dataset['id'], org_editor, version_name="Version2") context = get_context(org_editor) actual_activity_id = get_activity_id_from_dataset_version_name( @@ -294,58 +294,8 @@ def test_new_dataset_has_no_versions(self, test_dataset, org_editor): def test_dataset_has_versions_after_one_created(self, test_dataset, org_editor): context = get_context(org_editor) - _create_version(test_dataset['id'], org_editor) + create_version(test_dataset['id'], org_editor) assert True is dataset_has_versions( context, {'dataset_id': test_dataset['id']} ) - - -def _create_version(dataset_id, user, version_name="Default Name"): - return dataset_version_create( - get_context(user), - { - "dataset_id": dataset_id, - "name": version_name - } - ) - - -def _assert_version(version, checks): - assert version - for k, v in checks.items(): - assert version[k] == v, "found incorrect %s of version" % k - - -@pytest.fixture() -def org_admin(): - return factories.User(name="admin") - - -@pytest.fixture() -def org_editor(): - return factories.User(name="editor") - - -@pytest.fixture() -def org_member(): - return factories.User(name="member") - - -@pytest.fixture() -def test_organization(org_admin, org_editor, org_member): - return factories.Organization(users=[ - {'name': org_admin['id'], 'capacity': 'admin'}, - {'name': org_editor['id'], 'capacity': 'editor'}, - {'name': org_member['id'], 'capacity': 'member'} - ]) - - -@pytest.fixture() -def test_dataset(test_organization): - return factories.Dataset(owner_org=test_organization['id']) - - -@pytest.fixture() -def test_resource(test_dataset): - return factories.Resource(package_id=test_dataset['id']) From f825de13083d5e09a3ee0769c59e43f1478b9c6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Mon, 31 May 2021 14:21:03 +0200 Subject: [PATCH 17/29] Added version_update to IActions. --- ckanext/versions/plugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckanext/versions/plugin.py b/ckanext/versions/plugin.py index 41ff3e2..ddc90d1 100644 --- a/ckanext/versions/plugin.py +++ b/ckanext/versions/plugin.py @@ -50,6 +50,7 @@ def get_actions(self): 'resource_version_list': action.resource_version_list, 'resource_version_current': action.resource_version_current, 'version_show': action.version_show, + 'version_update': action.version_update, 'version_delete': action.version_delete, 'resource_view_list': action.resource_view_list, 'dataset_version_create': dataset_version_action.dataset_version_create, From e547ec4455ed3520c484bce5331c20d192f0fc7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Mon, 31 May 2021 14:46:18 +0200 Subject: [PATCH 18/29] Fix typos in version_update --- ckanext/versions/logic/action.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/versions/logic/action.py b/ckanext/versions/logic/action.py index c7bfd60..a625513 100644 --- a/ckanext/versions/logic/action.py +++ b/ckanext/versions/logic/action.py @@ -30,7 +30,7 @@ def version_update(context, data_dict): :rtype: dictionary """ model = context.get('model', core_model) - version_id, name = toolkit.get_or_bust(data_dict, ['version', 'name']) + version_id, name = toolkit.get_or_bust(data_dict, ['version_id', 'name']) # I'll create my own session! With Blackjack! And H**kers! session = model.meta.create_local_session() @@ -42,7 +42,7 @@ def version_update(context, data_dict): if not version: raise toolkit.ObjectNotFound('Version not found') - toolkit.check_access('dataset_version_create', context, data_dict) + toolkit.check_access('version_create', context, data_dict) assert context.get('auth_user_obj') # Should be here after `check_access` version.name = name From 72226e638e6f93fed4f200dcea51e0ce65aa63f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Mon, 31 May 2021 15:35:02 +0200 Subject: [PATCH 19/29] Added helper to get dataset version for activity_id. --- ckanext/versions/helpers.py | 23 ++++++++++++++++ ckanext/versions/plugin.py | 1 + ckanext/versions/tests/fixtures.py | 5 ++++ ckanext/versions/tests/test_helpers.py | 36 ++++++++++++++++++++++++++ 4 files changed, 65 insertions(+) diff --git a/ckanext/versions/helpers.py b/ckanext/versions/helpers.py index 3604e8a..51539d2 100644 --- a/ckanext/versions/helpers.py +++ b/ckanext/versions/helpers.py @@ -1,4 +1,5 @@ from ckan.plugins import toolkit +import json def resources_list_with_current_version(resources): @@ -78,3 +79,25 @@ def download_url(resource_url, version_id): ) return url + + +def dataset_version_for_activity_id(dataset_id, activity_id): + """Return dataset version created for given activity. + + :param dataset_id: the id or name of the dataset + :type dataset_id: string + :param activity_id: the id of the activity + :type activity_id: string + :returns: version, None if no version created for the given activity. + :rtype: dictionary + """ + # TODO: Think if instead `IPackageController.after_show` could be + # used to include version dict in package_dict + context = {'user': toolkit.g.user} + versions = toolkit.get_action('dataset_version_list')( + context, {'dataset_id': dataset_id} + ) + for version in versions: + if version['activity_id'] == activity_id: + return version + return None diff --git a/ckanext/versions/plugin.py b/ckanext/versions/plugin.py index ddc90d1..71a79ad 100644 --- a/ckanext/versions/plugin.py +++ b/ckanext/versions/plugin.py @@ -77,6 +77,7 @@ def get_helpers(self): 'versions_resource_version_from_activity_id': helpers.resource_version_from_activity_id, 'versions_resource_current_version': helpers.resource_current_version, 'versions_download_url': helpers.download_url, + 'dataset_version_for_activity_id': helpers.dataset_version_for_activity_id, } return helper_functions diff --git a/ckanext/versions/tests/fixtures.py b/ckanext/versions/tests/fixtures.py index 53b7e44..abe95dd 100644 --- a/ckanext/versions/tests/fixtures.py +++ b/ckanext/versions/tests/fixtures.py @@ -2,6 +2,7 @@ from ckan.tests import factories from ckanext.versions.model import create_tables, tables_exist +from ckanext.versions.tests import create_version @pytest.fixture @@ -41,3 +42,7 @@ def test_dataset(test_organization): @pytest.fixture() def test_resource(test_dataset): return factories.Resource(package_id=test_dataset['id']) + +@pytest.fixture() +def test_version(test_dataset, org_editor): + return create_version(test_dataset['id'], org_editor, version_name="Version1") diff --git a/ckanext/versions/tests/test_helpers.py b/ckanext/versions/tests/test_helpers.py index e820f86..1426c5b 100644 --- a/ckanext/versions/tests/test_helpers.py +++ b/ckanext/versions/tests/test_helpers.py @@ -1,6 +1,11 @@ +import mock import pytest +from ckan.plugins import toolkit from ckanext.versions import helpers +from ckanext.versions.helpers import dataset_version_for_activity_id +from ckanext.versions.tests import get_context + @pytest.mark.ckan_config('ckan.site_url', 'http://ckan:5000') def test_version_download_url_without_filename(): @@ -24,3 +29,34 @@ def test_version_download_url_with_external_url(): download_url = helpers.download_url(url, '') assert download_url == url + + +@pytest.mark.usefixtures('clean_db', 'versions_setup', 'with_request_context') +def test_dataset_version_for_activity_id_return_version(org_editor, test_dataset, test_version): + activity_id = test_version['activity_id'] + + with mock.patch('ckanext.versions.helpers.toolkit.g', user=org_editor['name']): + actual_version = dataset_version_for_activity_id(test_dataset['id'], activity_id) + assert test_version['id'] == actual_version['id'] + + +@pytest.mark.usefixtures('clean_db', 'versions_setup', 'with_request_context') +def test_dataset_version_for_activity_returns_none_if_no_version(app, test_dataset, org_editor): + context = get_context(org_editor) + toolkit.get_action('package_patch')( + context, + { + "id": test_dataset['id'], + "name": "updated-name" + } + ) + activities = toolkit.get_action('package_activity_list')( + context, + { + 'id': test_dataset['id'] + } + ) + activity_id = activities[0]['id'] + with mock.patch('ckanext.versions.helpers.toolkit.g', user=org_editor['name']): + version = dataset_version_for_activity_id(test_dataset['id'], activity_id) + assert None is version From 8e211d6361f5d17a871e8d8ba54b0de78c135ada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Tue, 1 Jun 2021 13:54:25 +0200 Subject: [PATCH 20/29] Dataset version create accepts id or name. --- ckanext/versions/logic/dataset_version_action.py | 7 +++---- ckanext/versions/tests/test_dataset_version_action.py | 4 ++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ckanext/versions/logic/dataset_version_action.py b/ckanext/versions/logic/dataset_version_action.py index 2a4ec6c..cf9a3e9 100644 --- a/ckanext/versions/logic/dataset_version_action.py +++ b/ckanext/versions/logic/dataset_version_action.py @@ -26,14 +26,14 @@ def dataset_version_create(context, data_dict): :rtype: dictionary """ model = context.get('model', core_model) - dataset_id, name = toolkit.get_or_bust( + dataset_name_or_id, name = toolkit.get_or_bust( data_dict, ['dataset_id', 'name']) activity_id = data_dict.get('activity_id') - dataset = model.Package.get(dataset_id) + dataset = model.Package.get(dataset_name_or_id) if not dataset: raise toolkit.ObjectNotFound("Dataset not found") - + dataset_id = dataset.id toolkit.check_access('version_create', context, {"package_id": dataset_id} ) @@ -46,7 +46,6 @@ def dataset_version_create(context, data_dict): filter_by(object_id=dataset_id). \ order_by(model.Activity.timestamp.desc()). \ first() - if not activity: raise toolkit.ObjectNotFound('Activity not found') diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py index a2f0f19..2ef691a 100644 --- a/ckanext/versions/tests/test_dataset_version_action.py +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -34,6 +34,10 @@ def test_dataset_version_create_should_create_version(self, org_admin, test_data assert_version(version, checks) + def test_dataset_create_version_accepts_dataset_name(self, test_dataset, org_editor): + version = create_version(test_dataset['name'], org_editor) + assert test_dataset['id'] == version['package_id'] + def test_dataset_create_should_create_version_for_given_activity(self, test_dataset, org_editor): version_name = "V1.0" context = get_context(org_editor) From 04e5efd6b925b1b2c8aba0bd7bacdb786080efa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Wed, 2 Jun 2021 09:53:20 +0200 Subject: [PATCH 21/29] Dataset version restore tests. --- .../versions/logic/dataset_version_action.py | 14 +++++ ckanext/versions/tests/__init__.py | 12 +++- .../tests/test_dataset_version_action.py | 61 ++++++++++++++++++- 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/ckanext/versions/logic/dataset_version_action.py b/ckanext/versions/logic/dataset_version_action.py index cf9a3e9..92ee845 100644 --- a/ckanext/versions/logic/dataset_version_action.py +++ b/ckanext/versions/logic/dataset_version_action.py @@ -77,6 +77,20 @@ def dataset_version_create(context, data_dict): return version.as_dict() +def dataset_version_restore(context, data_dict): + """Restores dataset version by restoring dataset + metadata and creating a new version with it + + :param dataset_id: the id or name of the dataset + :type dataset_id: string + :param version_id: the id or name of the dataset + :type version_id: string + :returns: restored dataset + :rtype: dict + """ + pass + + @toolkit.side_effect_free def dataset_version_list(context, data_dict): """List versions of a given dataset diff --git a/ckanext/versions/tests/__init__.py b/ckanext/versions/tests/__init__.py index 0b3867a..4c2f703 100644 --- a/ckanext/versions/tests/__init__.py +++ b/ckanext/versions/tests/__init__.py @@ -1,5 +1,5 @@ from ckan import model -from ckanext.versions.logic.dataset_version_action import dataset_version_create +from ckanext.versions.logic.dataset_version_action import dataset_version_create, dataset_version_restore def get_context(user): @@ -22,4 +22,12 @@ def create_version(dataset_id, user, version_name="Default Name"): "dataset_id": dataset_id, "name": version_name } - ) \ No newline at end of file + ) + + +def restore_version(dataset_id, version_id, user): + context = get_context(user) + dataset_version_restore(context, { + 'dataset_id': dataset_id, + 'version_id': version_id + }) \ No newline at end of file diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py index 2ef691a..968ed46 100644 --- a/ckanext/versions/tests/test_dataset_version_action.py +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -1,5 +1,6 @@ import pytest +from ckan import model from ckan.plugins import toolkit from ckan.tests import factories from ckanext.versions.logic.dataset_version_action import ( @@ -9,7 +10,8 @@ activity_dataset_show, dataset_version_latest, dataset_version_list ) -from ckanext.versions.tests import get_context, assert_version, create_version +from ckanext.versions.model import Version +from ckanext.versions.tests import get_context, assert_version, create_version, restore_version @pytest.mark.usefixtures('clean_db', 'versions_setup') @@ -137,6 +139,63 @@ def test_dataset_version_create_returns_valid_activity_id(self, test_organizatio assert old_dataset['name'] == old_name assert old_dataset['id'] == dataset['id'] + @pytest.mark.parametrize("object_ref", [ + "id", + "name" + ]) + def test_dataset_version_restore_retrives_dataset(self, test_version, test_dataset, org_editor, object_ref): + context = get_context(org_editor) + dataset_title = test_dataset['title'] + dataset_id = test_dataset['id'] + dataset_ref = test_dataset[object_ref] + version_ref = test_version[object_ref] + toolkit.get_action('package_patch')( + context, + { + "id": dataset_id, + "title": "updated-title" + } + ) + restored_dataset = restore_version(dataset_ref, version_ref, org_editor) + + assert dataset_title == restored_dataset['title'], "restored dataset title does not match" + assert dataset_id == restored_dataset['id'], "restored dataset has different id" + + def test_dataset_version_restore_creates_new_version_after_restore(self, test_version, test_dataset, org_editor): + restore_version(test_dataset['id'], test_version['id'], org_editor) + latest_version = model.Session.query(Version). \ + filter(Version.package_id == test_dataset['id']). \ + order_by(Version.created.desc()).first() + + assert latest_version.id != test_version['id'], "restore action should create a new version" + + def test_dataset_version_restore_fails_if_dataset_not_found(self, test_version, test_dataset, org_editor): + with pytest.raises(toolkit.ObjectNotFound) as e: + restore_version('fake-dataset-id', test_version['id'], org_editor) + assert "Dataset not found" == e.msg + + def test_dataset_version_restore_fails_if_dataset_not_have_version(self, test_version, test_dataset, org_editor): + with pytest.raises(toolkit.ObjectNotFound) as e: + restore_version(test_dataset['id'], 'fake-version-id', org_editor) + assert "Version not found" == e.msg + + @pytest.mark.parametrize("user_role, can_create_version", [ + ('admin', True), + ('editor', True), + ('member', False), + ]) + def test_dataset_version_restore_auth(self, test_version, test_organization, test_dataset, user_role, can_create_version): + for user in test_organization['users']: + if user['capacity'] == user_role: + if can_create_version: + restore_version(test_dataset['id'], test_version['id'], user) + return + else: + with pytest.raises(toolkit.NotAuthorized): + restore_version(test_dataset['id'], test_version['id'], user) + return + pytest.fail("Couldn't find user with required role %s", user_role) + def test_dataset_version_list_return_all_version(self, test_dataset, org_editor): context = get_context(org_editor) version1 = create_version(test_dataset['id'], org_editor, version_name="Version1") From 3d03661311fc303134edb2f8a5498bdf72d21f55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Wed, 2 Jun 2021 10:04:37 +0200 Subject: [PATCH 22/29] Fixup, codestyle. --- ckanext/versions/logic/dataset_version_action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/versions/logic/dataset_version_action.py b/ckanext/versions/logic/dataset_version_action.py index 92ee845..5fc734f 100644 --- a/ckanext/versions/logic/dataset_version_action.py +++ b/ckanext/versions/logic/dataset_version_action.py @@ -36,7 +36,7 @@ def dataset_version_create(context, data_dict): dataset_id = dataset.id toolkit.check_access('version_create', context, - {"package_id": dataset_id} ) + {"package_id": dataset_id}) creator_user_id = context['auth_user_obj'].id if activity_id: From 94876290420329a5e26fb0fa9426a2c196a1a472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Wed, 2 Jun 2021 11:53:05 +0200 Subject: [PATCH 23/29] version_show supports version_id or version_name & dataset_id --- ckanext/versions/logic/action.py | 18 ++++++++++++++--- ckanext/versions/tests/test_actions.py | 28 ++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/ckanext/versions/logic/action.py b/ckanext/versions/logic/action.py index a625513..bc4389d 100644 --- a/ckanext/versions/logic/action.py +++ b/ckanext/versions/logic/action.py @@ -199,14 +199,26 @@ def version_delete(context, data_dict): def version_show(context, data_dict): """Show a specific version object - :param version_id: the id of the version + :param version_id: the id or name of the version :type version_id: string + :param dataset_id: [Optional] the id or name of a dataset. Mandatory + if version name provided as version_id + :type dataset_id: string :returns: the version dictionary :rtype: dict """ model = context.get('model', core_model) - version_id = toolkit.get_or_bust(data_dict, ['version_id']) - version = model.Session.query(Version).get(version_id) + version_name_or_id = toolkit.get_or_bust(data_dict, ['version_id']) + version = model.Session.query(Version).get(version_name_or_id) + if not version: + version_name = version_name_or_id + dataset_name_or_id = data_dict.get('dataset_id') + dataset = model.Package.get(dataset_name_or_id) + if dataset: + dataset_id = dataset.id + version = model.Session.query(Version). \ + filter(Version.package_id == dataset_id). \ + filter(Version.name == version_name).one_or_none() if not version: raise toolkit.ObjectNotFound('Version not found') diff --git a/ckanext/versions/tests/test_actions.py b/ckanext/versions/tests/test_actions.py index 79f4385..f50e92a 100644 --- a/ckanext/versions/tests/test_actions.py +++ b/ckanext/versions/tests/test_actions.py @@ -321,6 +321,34 @@ def test_version_show(self): assert result['notes'] == 'Version notes' assert result['creator_user_id'] == user['id'] + def test_version_show_for_version_name(self): + dataset = factories.Dataset() + resource = factories.Resource( + package_id=dataset['id'], + name='First name' + ) + user = factories.Sysadmin() + context = get_context(user) + + version = resource_version_create( + context, { + 'resource_id': resource['id'], + 'name': '1', + 'notes': 'Version notes' + } + ) + + result = version_show( + context, + {'version_id': version['name'], + 'dataset_id': dataset['id']} + ) + + assert result['id'] == version['id'] + assert result['name'] == '1' + assert result['notes'] == 'Version notes' + assert result['creator_user_id'] == user['id'] + @pytest.mark.usefixtures('clean_db', 'versions_setup') class TestVersionDelete(object): From ce79afb24845280d89ecf15df9a1a7cfc8bffa20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Wed, 2 Jun 2021 13:13:50 +0200 Subject: [PATCH 24/29] dataset_version_restore implementation --- .../versions/logic/dataset_version_action.py | 30 ++++++++++++++++++- ckanext/versions/plugin.py | 1 + ckanext/versions/tests/__init__.py | 2 +- .../tests/test_dataset_version_action.py | 6 ++-- 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/ckanext/versions/logic/dataset_version_action.py b/ckanext/versions/logic/dataset_version_action.py index 5fc734f..4322f39 100644 --- a/ckanext/versions/logic/dataset_version_action.py +++ b/ckanext/versions/logic/dataset_version_action.py @@ -5,6 +5,7 @@ from ckan import model as core_model from ckan.plugins import toolkit +from ckanext.versions.logic.action import version_show from ckanext.versions.model import Version log = logging.getLogger(__name__) @@ -88,7 +89,34 @@ def dataset_version_restore(context, data_dict): :returns: restored dataset :rtype: dict """ - pass + model = context.get('model', core_model) + dataset_name_or_id, version_name_or_id = toolkit.get_or_bust( + data_dict, ['dataset_id', 'version_id']) + + dataset = model.Package.get(dataset_name_or_id) + if not dataset: + raise toolkit.ObjectNotFound("Dataset not found") + dataset_id = dataset.id + toolkit.check_access('version_create', + context, + {"package_id": dataset_id}) + + version = version_show(context, data_dict) + if not version: + raise toolkit.ObjectNotFound("Version not found") + v_name = "restored_{}".format(version['name']) + v_notes = "Restored from version: {}".format(version['name']) + old_dataset = activity_dataset_show( + context, + { + 'activity_id': version['activity_id'], + 'dataset_id': dataset_id + } + ) + restored_dataset = toolkit.get_action('package_update')(context, old_dataset) + dataset_version_create(context, {'dataset_id': dataset.id, 'name': v_name, 'notes': v_notes}) + + return restored_dataset @toolkit.side_effect_free diff --git a/ckanext/versions/plugin.py b/ckanext/versions/plugin.py index 71a79ad..8436d9d 100644 --- a/ckanext/versions/plugin.py +++ b/ckanext/versions/plugin.py @@ -54,6 +54,7 @@ def get_actions(self): 'version_delete': action.version_delete, 'resource_view_list': action.resource_view_list, 'dataset_version_create': dataset_version_action.dataset_version_create, + 'dataset_version_restore': dataset_version_action.dataset_version_restore, 'dataset_version_list': dataset_version_action.dataset_version_list, 'dataset_version_latest': dataset_version_action.dataset_version_latest } diff --git a/ckanext/versions/tests/__init__.py b/ckanext/versions/tests/__init__.py index 4c2f703..b4a6bee 100644 --- a/ckanext/versions/tests/__init__.py +++ b/ckanext/versions/tests/__init__.py @@ -27,7 +27,7 @@ def create_version(dataset_id, user, version_name="Default Name"): def restore_version(dataset_id, version_id, user): context = get_context(user) - dataset_version_restore(context, { + return dataset_version_restore(context, { 'dataset_id': dataset_id, 'version_id': version_id }) \ No newline at end of file diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py index 968ed46..13e38a3 100644 --- a/ckanext/versions/tests/test_dataset_version_action.py +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -170,14 +170,12 @@ def test_dataset_version_restore_creates_new_version_after_restore(self, test_ve assert latest_version.id != test_version['id'], "restore action should create a new version" def test_dataset_version_restore_fails_if_dataset_not_found(self, test_version, test_dataset, org_editor): - with pytest.raises(toolkit.ObjectNotFound) as e: + with pytest.raises(toolkit.ObjectNotFound, match="Dataset not found"): restore_version('fake-dataset-id', test_version['id'], org_editor) - assert "Dataset not found" == e.msg def test_dataset_version_restore_fails_if_dataset_not_have_version(self, test_version, test_dataset, org_editor): - with pytest.raises(toolkit.ObjectNotFound) as e: + with pytest.raises(toolkit.ObjectNotFound, match="Version not found"): restore_version(test_dataset['id'], 'fake-version-id', org_editor) - assert "Version not found" == e.msg @pytest.mark.parametrize("user_role, can_create_version", [ ('admin', True), From 694ddb2068726903eceee81972e900e0183ce6d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Wed, 2 Jun 2021 13:17:57 +0200 Subject: [PATCH 25/29] Refactored checks for error messages to use `match` kwarg. --- .../tests/test_dataset_version_action.py | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py index 13e38a3..38d8427 100644 --- a/ckanext/versions/tests/test_dataset_version_action.py +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -79,7 +79,7 @@ def test_dataset_create_should_create_version_for_given_activity(self, test_data def test_dataset_create_should_fail_when_incorrect_activity_id(self, test_dataset, org_editor): context = get_context(org_editor) - with pytest.raises(toolkit.ObjectNotFound) as e: + with pytest.raises(toolkit.ObjectNotFound, match="Activity not found"): dataset_version_create( context, { @@ -88,7 +88,6 @@ def test_dataset_create_should_fail_when_incorrect_activity_id(self, test_datase "activity_id": "fake-activity-id" } ) - assert "Activity not found" == e.msg @pytest.mark.parametrize("user_role, can_create_version", [ ('admin', True), @@ -114,9 +113,8 @@ def test_dataset_version_create_should_not_create_version_with_same_name(self, t create_version(test_dataset['id'], org_editor, version_name=version_name) def test_dataset_version_create_should_fail_if_dataset_not_exists(self, org_editor): - with pytest.raises(toolkit.ObjectNotFound) as e: + with pytest.raises(toolkit.ObjectNotFound, match="Dataset not found"): create_version('fake_dataset_id', org_editor) - assert e.msg == "Dataset not found" def test_dataset_version_create_returns_valid_activity_id(self, test_organization, org_editor): old_name = "initial-name" @@ -218,14 +216,13 @@ def test_dataset_version_list_return_all_version(self, test_dataset, org_editor) def test_dataset_version_should_fail_if_dataset_not_exists(self, org_editor): context = get_context(org_editor) - with pytest.raises(toolkit.ObjectNotFound) as e: + with pytest.raises(toolkit.ObjectNotFound, match="Dataset not found"): dataset_version_list( context, { 'dataset_id': 'fake-dataset-id' } ) - assert "Dataset not found" == e.msg def test_dataset_version_should_return_empty_list_if_dataset_no_versions(self, test_dataset, org_editor): context = get_context(org_editor) @@ -261,25 +258,23 @@ def test_dataset_version_latest_show_latest_version(self, test_dataset, org_edit def test_dataset_version_latest_raises_when_dataset_not_found(self, org_editor): context = get_context(org_editor) - with pytest.raises(toolkit.ObjectNotFound) as e: + with pytest.raises(toolkit.ObjectNotFound, match="Dataset not found"): dataset_version_latest( context, { 'dataset_id': 'fake-dataset-id' } ) - assert "Dataset not found" == e.msg def test_dataset_version_latest_raises_when_dataset_has_no_versions(self, test_dataset, org_editor): context = get_context(org_editor) - with pytest.raises(toolkit.ObjectNotFound) as e: + with pytest.raises(toolkit.ObjectNotFound, match="Versions not found for this dataset"): dataset_version_latest( context, { 'dataset_id': test_dataset['id'] } ) - assert "Versions not found for this dataset" == e.msg def test_activity_dataset_show_returns_correct_dataset(self, test_dataset, org_editor): version = create_version(test_dataset['id'], org_editor) @@ -307,7 +302,7 @@ def test_activity_dataset_show_returns_correct_dataset(self, test_dataset, org_e def test_activity_dataset_show_fails_if_no_dataset_in_activity(self, test_dataset, org_editor): version = create_version(test_dataset['id'], org_editor) context = get_context(org_editor) - with pytest.raises(toolkit.ObjectNotFound) as e: + with pytest.raises(toolkit.ObjectNotFound, match='Dataset not found in the activity object.'): activity_dataset_show( context, { @@ -315,7 +310,6 @@ def test_activity_dataset_show_fails_if_no_dataset_in_activity(self, test_datase 'activity_id': version['activity_id'] } ) - assert 'Dataset not found in the activity object.' == e.msg def test_get_activity_id_from_dataset_version_returns_correct(self, test_dataset, org_editor): version1 = create_version(test_dataset['id'], org_editor, version_name="Version1") @@ -336,7 +330,7 @@ def test_get_activity_id_from_dataset_version_returns_correct(self, test_dataset def test_get_activity_id_from_dataset_version_raises_not_found(self, test_dataset, org_editor): context = get_context(org_editor) - with pytest.raises(toolkit.ObjectNotFound) as e: + with pytest.raises(toolkit.ObjectNotFound, match="Version not found in the dataset"): get_activity_id_from_dataset_version_name( context, { @@ -344,7 +338,6 @@ def test_get_activity_id_from_dataset_version_raises_not_found(self, test_datase 'version': 'Fake version name' } ) - assert "Version not found in the dataset" == e.msg def test_new_dataset_has_no_versions(self, test_dataset, org_editor): context = get_context(org_editor) From 1ea6a63e9f52a950c3575f697e3fde861f0818ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Mon, 7 Jun 2021 09:57:08 +0200 Subject: [PATCH 26/29] ADX-680 Can't create multiple versions for single activity_id --- ckanext/versions/logic/dataset_version_action.py | 6 ++++++ ckanext/versions/tests/test_dataset_version_action.py | 8 ++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ckanext/versions/logic/dataset_version_action.py b/ckanext/versions/logic/dataset_version_action.py index 4322f39..faa64d2 100644 --- a/ckanext/versions/logic/dataset_version_action.py +++ b/ckanext/versions/logic/dataset_version_action.py @@ -50,6 +50,12 @@ def dataset_version_create(context, data_dict): if not activity: raise toolkit.ObjectNotFound('Activity not found') + version_for_activity = model.Session.query(Version). \ + filter_by(activity_id=activity.id). \ + first() + if version_for_activity: + raise toolkit.ValidationError("Version already exists for this activity") + version = Version( package_id=dataset_id, activity_id=activity.id, diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py index 38d8427..4253e74 100644 --- a/ckanext/versions/tests/test_dataset_version_action.py +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -89,6 +89,12 @@ def test_dataset_create_should_fail_when_incorrect_activity_id(self, test_datase } ) + def test_dataset_version_create_fails_if_version_for_activity_exists(self, test_dataset, org_editor): + create_version(test_dataset['id'], org_editor, version_name="Version1") + with pytest.raises(toolkit.ValidationError, match="Version already exists for this activity"): + create_version(test_dataset['id'], org_editor, version_name="Version2") + + @pytest.mark.parametrize("user_role, can_create_version", [ ('admin', True), ('editor', True), @@ -315,8 +321,6 @@ def test_get_activity_id_from_dataset_version_returns_correct(self, test_dataset version1 = create_version(test_dataset['id'], org_editor, version_name="Version1") expected_activity_id = version1['activity_id'] - create_version(test_dataset['id'], org_editor, version_name="Version2") - context = get_context(org_editor) actual_activity_id = get_activity_id_from_dataset_version_name( context, From 36d9f37e63bf94bf28057bd480a200e7eb69e5bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Mon, 7 Jun 2021 12:37:41 +0200 Subject: [PATCH 27/29] ADX-689 Added a test to verify create by time order of version list. --- .../tests/test_dataset_version_action.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py index 4253e74..fa7e794 100644 --- a/ckanext/versions/tests/test_dataset_version_action.py +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -217,9 +217,31 @@ def test_dataset_version_list_return_all_version(self, test_dataset, org_editor) } ) version_ids = [v['id'] for v in version_list] + assert 2 == len(version_ids), "only 2 versions created for dataset" assert version1['id'] in version_ids assert version2['id'] in version_ids + def test_dataset_version_list_return_version_in_create_time_desc_order(self, test_dataset, org_editor): + context = get_context(org_editor) + version1 = create_version(test_dataset['id'], org_editor, version_name="Version1") + toolkit.get_action('package_patch')( + context, + { + "id": test_dataset['id'], + "title": "New Title" + } + ) + version2 = create_version(test_dataset['id'], org_editor, version_name="Version2") + + version_list = dataset_version_list( + context, + { + 'dataset_id': test_dataset['id'] + } + ) + assert version2['id'] == version_list[0]['id'], "version2 should be first as newest" + assert version1['id'] == version_list[-1]['id'], "version1 should be last as oldest" + def test_dataset_version_should_fail_if_dataset_not_exists(self, org_editor): context = get_context(org_editor) with pytest.raises(toolkit.ObjectNotFound, match="Dataset not found"): From 73961a860207e196c9b693cc2c497ea667e8b6f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Mon, 7 Jun 2021 12:42:56 +0200 Subject: [PATCH 28/29] Codestyle. --- ckanext/versions/helpers.py | 1 - ckanext/versions/logic/dataset_version_action.py | 1 + ckanext/versions/tests/__init__.py | 2 +- ckanext/versions/tests/fixtures.py | 2 ++ ckanext/versions/tests/test_dataset_version_action.py | 1 - 5 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ckanext/versions/helpers.py b/ckanext/versions/helpers.py index 51539d2..2478953 100644 --- a/ckanext/versions/helpers.py +++ b/ckanext/versions/helpers.py @@ -1,5 +1,4 @@ from ckan.plugins import toolkit -import json def resources_list_with_current_version(resources): diff --git a/ckanext/versions/logic/dataset_version_action.py b/ckanext/versions/logic/dataset_version_action.py index faa64d2..44f9dc7 100644 --- a/ckanext/versions/logic/dataset_version_action.py +++ b/ckanext/versions/logic/dataset_version_action.py @@ -10,6 +10,7 @@ log = logging.getLogger(__name__) + def dataset_version_create(context, data_dict): """Create a new version from the current dataset's activity_id diff --git a/ckanext/versions/tests/__init__.py b/ckanext/versions/tests/__init__.py index b4a6bee..b9f1cb5 100644 --- a/ckanext/versions/tests/__init__.py +++ b/ckanext/versions/tests/__init__.py @@ -30,4 +30,4 @@ def restore_version(dataset_id, version_id, user): return dataset_version_restore(context, { 'dataset_id': dataset_id, 'version_id': version_id - }) \ No newline at end of file + }) diff --git a/ckanext/versions/tests/fixtures.py b/ckanext/versions/tests/fixtures.py index abe95dd..d8019e4 100644 --- a/ckanext/versions/tests/fixtures.py +++ b/ckanext/versions/tests/fixtures.py @@ -10,6 +10,7 @@ def versions_setup(): if not tables_exist(): create_tables() + @pytest.fixture() def org_admin(): return factories.User(name="admin") @@ -43,6 +44,7 @@ def test_dataset(test_organization): def test_resource(test_dataset): return factories.Resource(package_id=test_dataset['id']) + @pytest.fixture() def test_version(test_dataset, org_editor): return create_version(test_dataset['id'], org_editor, version_name="Version1") diff --git a/ckanext/versions/tests/test_dataset_version_action.py b/ckanext/versions/tests/test_dataset_version_action.py index fa7e794..b72666a 100644 --- a/ckanext/versions/tests/test_dataset_version_action.py +++ b/ckanext/versions/tests/test_dataset_version_action.py @@ -94,7 +94,6 @@ def test_dataset_version_create_fails_if_version_for_activity_exists(self, test_ with pytest.raises(toolkit.ValidationError, match="Version already exists for this activity"): create_version(test_dataset['id'], org_editor, version_name="Version2") - @pytest.mark.parametrize("user_role, can_create_version", [ ('admin', True), ('editor', True), From ab248d764a792b91f8332badde2e5deaf2730ab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Saba=C5=82a?= Date: Tue, 8 Jun 2021 18:44:38 +0200 Subject: [PATCH 29/29] Fix for version_update using legacy notes field name --- ckanext/versions/logic/action.py | 4 +--- ckanext/versions/tests/test_actions.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ckanext/versions/logic/action.py b/ckanext/versions/logic/action.py index 21c4eea..1559543 100644 --- a/ckanext/versions/logic/action.py +++ b/ckanext/versions/logic/action.py @@ -18,8 +18,6 @@ def version_update(context, data_dict): """Update a version from the current dataset. - :param package_id: the id the dataset - :type package_id: string :param version_id: the id of the version :type version_id: string :param name: A short name for the version @@ -46,7 +44,7 @@ def version_update(context, data_dict): assert context.get('auth_user_obj') # Should be here after `check_access` version.name = name - version.description = data_dict.get('description', None) + version.notes = data_dict.get('notes', None) session.add(version) diff --git a/ckanext/versions/tests/test_actions.py b/ckanext/versions/tests/test_actions.py index fb32730..36905f2 100644 --- a/ckanext/versions/tests/test_actions.py +++ b/ckanext/versions/tests/test_actions.py @@ -302,6 +302,26 @@ def test_resource_version_current_no_versions(self): assert helpers.call_action('resource_version_current', {}, resource_id=resource['id']) is None +@pytest.mark.usefixtures('clean_db', 'versions_setup') +class TestVersionUpdate(object): + + def test_version_update(self, test_version, org_editor): + context = get_context(org_editor) + updated_version = toolkit.get_action('version_update')( + context, + { + 'version_id': test_version['id'], + 'package_id': test_version['package_id'], + 'name': "updated-name", + 'notes': "updated-notes" + } + ) + + assert test_version['id'] == updated_version['id'] + assert "updated-name" == updated_version['name'] + assert "updated-notes" == updated_version['notes'] + + @pytest.mark.usefixtures('clean_db', 'versions_setup') class TestVersionShow(object):