From 1042236010b8ee43ea5c5f8a9fc5d9abc4bec581 Mon Sep 17 00:00:00 2001 From: Kostis-S-Z Date: Wed, 3 Dec 2025 14:59:07 +0200 Subject: [PATCH 01/10] Add unit tests --- .../dataset_loading_scripts/test_registry.py | 33 +++++++++++++++++++ tests/test_datasets.py | 27 +++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 tests/dataset_loading_scripts/test_registry.py create mode 100644 tests/test_datasets.py diff --git a/tests/dataset_loading_scripts/test_registry.py b/tests/dataset_loading_scripts/test_registry.py new file mode 100644 index 0000000..e2623f5 --- /dev/null +++ b/tests/dataset_loading_scripts/test_registry.py @@ -0,0 +1,33 @@ +import pytest + +from datacollective.dataset_loading_scripts import registry + + +def test_load_dataset_routes_to_scripted(monkeypatch, tmp_path): + sentinel = object() + + def fake_loader(extract_dir): + assert extract_dir == tmp_path + return sentinel + + monkeypatch.setattr(registry, "_load_scripted", fake_loader) + result = registry.load_dataset_from_name_as_dataframe("common voice scripted", tmp_path) + assert result is sentinel + + +def test_load_dataset_routes_to_spontaneous(monkeypatch, tmp_path): + sentinel = object() + + def fake_loader(extract_dir): + assert extract_dir == tmp_path + return sentinel + + monkeypatch.setattr(registry, "_load_spontaneous", fake_loader) + result = registry.load_dataset_from_name_as_dataframe("common voice spontaneous", tmp_path) + assert result is sentinel + + +def test_load_dataset_invalid_name(tmp_path): + with pytest.raises(ValueError): + registry.load_dataset_from_name_as_dataframe("unknown dataset", tmp_path) + diff --git a/tests/test_datasets.py b/tests/test_datasets.py new file mode 100644 index 0000000..3c72f49 --- /dev/null +++ b/tests/test_datasets.py @@ -0,0 +1,27 @@ +from datacollective.datasets import _resolve_download_dir, _strip_archive_suffix + + +def test_strip_archive_suffix_removes_known_extensions(tmp_path): + tar_path = tmp_path / "sample.tar.gz" + zip_path = tmp_path / "sample.zip" + + assert _strip_archive_suffix(tar_path).name == "sample" + assert _strip_archive_suffix(zip_path).name == "sample" + + +def test_resolve_download_dir_prefers_argument(tmp_path, monkeypatch): + monkeypatch.delenv("MDC_DOWNLOAD_PATH", raising=False) + custom_dir = tmp_path / "custom" + resolved = _resolve_download_dir(str(custom_dir)) + + assert resolved == custom_dir + assert custom_dir.exists() + + +def test_resolve_download_dir_uses_env_default(tmp_path, monkeypatch): + env_dir = tmp_path / "env" + monkeypatch.setenv("MDC_DOWNLOAD_PATH", str(env_dir)) + resolved = _resolve_download_dir(None) + + assert resolved == env_dir + assert env_dir.exists() From a36c6e816efdf61665f80ebc9601e2756ce3ad91 Mon Sep 17 00:00:00 2001 From: Kostis-S-Z Date: Wed, 3 Dec 2025 15:02:06 +0200 Subject: [PATCH 02/10] Run lint --- src/datacollective/datasets.py | 2 +- .../dataset_loading_scripts/test_registry.py | 26 +++++++++++++------ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/datacollective/datasets.py b/src/datacollective/datasets.py index b0b95ca..0422bed 100644 --- a/src/datacollective/datasets.py +++ b/src/datacollective/datasets.py @@ -7,6 +7,7 @@ from typing import Any import pandas as pd +from fox_progress_bar import ProgressBar from datacollective.api_utils import ( ENV_DOWNLOAD_PATH, @@ -17,7 +18,6 @@ from datacollective.dataset_loading_scripts.registry import ( load_dataset_from_name_as_dataframe, ) -from fox_progress_bar import ProgressBar def get_dataset_details(dataset_id: str) -> dict[str, Any]: diff --git a/tests/dataset_loading_scripts/test_registry.py b/tests/dataset_loading_scripts/test_registry.py index e2623f5..db73dcb 100644 --- a/tests/dataset_loading_scripts/test_registry.py +++ b/tests/dataset_loading_scripts/test_registry.py @@ -1,33 +1,43 @@ +from pathlib import Path + import pytest +from _pytest.monkeypatch import MonkeyPatch from datacollective.dataset_loading_scripts import registry -def test_load_dataset_routes_to_scripted(monkeypatch, tmp_path): +def test_load_dataset_routes_to_scripted( + monkeypatch: MonkeyPatch, tmp_path: Path +) -> None: sentinel = object() - def fake_loader(extract_dir): + def fake_loader(extract_dir: Path) -> object: assert extract_dir == tmp_path return sentinel monkeypatch.setattr(registry, "_load_scripted", fake_loader) - result = registry.load_dataset_from_name_as_dataframe("common voice scripted", tmp_path) + result = registry.load_dataset_from_name_as_dataframe( + "common voice scripted", tmp_path + ) assert result is sentinel -def test_load_dataset_routes_to_spontaneous(monkeypatch, tmp_path): +def test_load_dataset_routes_to_spontaneous( + monkeypatch: MonkeyPatch, tmp_path: Path +) -> None: sentinel = object() - def fake_loader(extract_dir): + def fake_loader(extract_dir: Path) -> object: assert extract_dir == tmp_path return sentinel monkeypatch.setattr(registry, "_load_spontaneous", fake_loader) - result = registry.load_dataset_from_name_as_dataframe("common voice spontaneous", tmp_path) + result = registry.load_dataset_from_name_as_dataframe( + "common voice spontaneous", tmp_path + ) assert result is sentinel -def test_load_dataset_invalid_name(tmp_path): +def test_load_dataset_invalid_name(tmp_path: Path) -> None: with pytest.raises(ValueError): registry.load_dataset_from_name_as_dataframe("unknown dataset", tmp_path) - From b100fbeb531c620fd33614d77917dc36894eaafe Mon Sep 17 00:00:00 2001 From: Kostis-S-Z Date: Wed, 3 Dec 2025 15:03:09 +0200 Subject: [PATCH 03/10] Add e2e tests --- tests/dataset_loading_scripts/__init__.py | 0 tests/test_e2e.py | 47 +++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 tests/dataset_loading_scripts/__init__.py create mode 100644 tests/test_e2e.py diff --git a/tests/dataset_loading_scripts/__init__.py b/tests/dataset_loading_scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_e2e.py b/tests/test_e2e.py new file mode 100644 index 0000000..d68e081 --- /dev/null +++ b/tests/test_e2e.py @@ -0,0 +1,47 @@ +import os +from pathlib import Path + +import pandas as pd +import pytest +from _pytest.monkeypatch import MonkeyPatch + +from datacollective import get_dataset_details, load_dataset + +MDC_API_KEY = os.getenv("MDC_API_KEY") + +pytestmark = pytest.mark.skipif( + not MDC_API_KEY, + reason="Set MDC_API_KEY to run live API tests.", +) + + +def test_get_dataset_details_live_api( + dataset_id: str = "cmhvzlidq0326mn07hk4do3pj", +) -> None: + """NOTE: This test calls a live MDC API endpoint.""" + details = get_dataset_details(dataset_id) + + assert isinstance(details, dict) + assert details.get("id") == dataset_id + assert isinstance(details.get("name"), str) and details["name"].strip() + + +def test_load_dataset_live_api( + tmp_path: Path, + monkeypatch: MonkeyPatch, + dataset_id: str = "cmhvzlidq0326mn07hk4do3pj", +) -> None: + """NOTE: This test calls a live MDC API endpoint.""" + + monkeypatch.setenv("MDC_DOWNLOAD_PATH", str(tmp_path)) + + df = load_dataset( + dataset_id, + download_directory=str(tmp_path), + show_progress=False, + overwrite_existing=True, + ) + + assert isinstance(df, pd.DataFrame) + assert not df.empty + assert len(df.columns) > 0 From 59496008a96aee4f344f0591a6772976f8b5ffd4 Mon Sep 17 00:00:00 2001 From: Kostis-S-Z Date: Wed, 3 Dec 2025 15:27:24 +0200 Subject: [PATCH 04/10] Move e2e tests --- tests/e2e/__init__.py | 0 tests/{ => e2e}/test_e2e.py | 13 +++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/__init__.py rename tests/{ => e2e}/test_e2e.py (69%) diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_e2e.py b/tests/e2e/test_e2e.py similarity index 69% rename from tests/test_e2e.py rename to tests/e2e/test_e2e.py index d68e081..a0966bc 100644 --- a/tests/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -8,17 +8,21 @@ from datacollective import get_dataset_details, load_dataset MDC_API_KEY = os.getenv("MDC_API_KEY") +MDC_TEST_API_URL = os.getenv("MDC_TEST_API_URL") pytestmark = pytest.mark.skipif( - not MDC_API_KEY, - reason="Set MDC_API_KEY to run live API tests.", + not (MDC_API_KEY and MDC_TEST_API_URL), + reason="Set MDC_API_KEY and MDC_TEST_API_URL to run live API tests.", ) def test_get_dataset_details_live_api( + monkeypatch: MonkeyPatch, dataset_id: str = "cmhvzlidq0326mn07hk4do3pj", ) -> None: - """NOTE: This test calls a live MDC API endpoint.""" + """NOTE: This test calls a live MDC API endpoint (dev).""" + monkeypatch.setenv("MDC_API_URL", MDC_TEST_API_URL) + details = get_dataset_details(dataset_id) assert isinstance(details, dict) @@ -31,9 +35,10 @@ def test_load_dataset_live_api( monkeypatch: MonkeyPatch, dataset_id: str = "cmhvzlidq0326mn07hk4do3pj", ) -> None: - """NOTE: This test calls a live MDC API endpoint.""" + """NOTE: This test calls a live MDC API endpoint (dev).""" monkeypatch.setenv("MDC_DOWNLOAD_PATH", str(tmp_path)) + monkeypatch.setenv("MDC_API_URL", MDC_TEST_API_URL) df = load_dataset( dataset_id, From b98590417b39c06f096d04d1dc21102b537da0df Mon Sep 17 00:00:00 2001 From: Kostis-S-Z Date: Wed, 3 Dec 2025 15:28:30 +0200 Subject: [PATCH 05/10] Type hints --- tests/test_datasets.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test_datasets.py b/tests/test_datasets.py index 3c72f49..0dff864 100644 --- a/tests/test_datasets.py +++ b/tests/test_datasets.py @@ -1,7 +1,11 @@ +from pathlib import Path + +from _pytest.monkeypatch import MonkeyPatch + from datacollective.datasets import _resolve_download_dir, _strip_archive_suffix -def test_strip_archive_suffix_removes_known_extensions(tmp_path): +def test_strip_archive_suffix_removes_known_extensions(tmp_path: Path) -> None: tar_path = tmp_path / "sample.tar.gz" zip_path = tmp_path / "sample.zip" @@ -9,7 +13,9 @@ def test_strip_archive_suffix_removes_known_extensions(tmp_path): assert _strip_archive_suffix(zip_path).name == "sample" -def test_resolve_download_dir_prefers_argument(tmp_path, monkeypatch): +def test_resolve_download_dir_prefers_argument( + tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: monkeypatch.delenv("MDC_DOWNLOAD_PATH", raising=False) custom_dir = tmp_path / "custom" resolved = _resolve_download_dir(str(custom_dir)) @@ -18,7 +24,9 @@ def test_resolve_download_dir_prefers_argument(tmp_path, monkeypatch): assert custom_dir.exists() -def test_resolve_download_dir_uses_env_default(tmp_path, monkeypatch): +def test_resolve_download_dir_uses_env_default( + tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: env_dir = tmp_path / "env" monkeypatch.setenv("MDC_DOWNLOAD_PATH", str(env_dir)) resolved = _resolve_download_dir(None) From 0a936e08350dd5ecf9bdf6d316b576d8dc853a5d Mon Sep 17 00:00:00 2001 From: Kostis-S-Z Date: Wed, 3 Dec 2025 15:28:52 +0200 Subject: [PATCH 06/10] Add note for tests --- docs/index.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index 6f5c6f1..ca3028b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -123,10 +123,20 @@ print(info) For a detailed API reference, see the [API Reference](api.md) section of the documentation. -## Release Workflow - > [!NOTE] > This section is intended for maintainers of the `datacollective` library. +## Tests + +Run the full test suite: +```bash +pytest -v +``` + +Note that the e2e tests require a valid `MDC_API_KEY` and a `MDC_TEST_API_URL` key set in your environment. Pytest will skip the live e2e tests automatically if either is missing. + + +## Release Workflow + Check out the [Release Workflow](release.md) document for details on how to publish new versions of the library to PyPI using GitHub Actions. \ No newline at end of file From 5024a250fbafd90b413cdd8e4d4e7ef24a100985 Mon Sep 17 00:00:00 2001 From: Kostis-S-Z Date: Wed, 3 Dec 2025 15:29:50 +0200 Subject: [PATCH 07/10] Only run e2e tests for PR in main & releases --- .github/workflows/tests.yml | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index be2ca27..8c3f04a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,6 +10,8 @@ on: paths: - 'src/**' - 'tests/**' + release: + types: [published] workflow_dispatch: jobs: @@ -30,5 +32,30 @@ jobs: - name: Install test dependencies run: pip install -e '.[dev]' - - name: Run Tests - run: pytest -v tests + - name: Run unit & integration tests (exclude e2e) + run: pytest -v --ignore=tests/e2e tests + + run-e2e: + needs: run-tests + timeout-minutes: 30 + runs-on: ubuntu-latest + if: > + github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main') || + (github.event_name == 'release' && github.event.action == 'published') + + steps: + - name: Check out the repository + uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.9' + cache: "pip" + + - name: Install test dependencies + run: pip install -e '.[dev]' + + - name: Run e2e tests + run: pytest -q tests/e2e/test_e2e.py From 8062d0a0d58ffd491a30c645fbc49dfaaed16b2f Mon Sep 17 00:00:00 2001 From: Kostis-S-Z Date: Wed, 3 Dec 2025 16:14:47 +0200 Subject: [PATCH 08/10] Update e2e test for TEST_API_KEY & test dataset ID --- tests/e2e/test_e2e.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py index a0966bc..4470ae4 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -7,20 +7,21 @@ from datacollective import get_dataset_details, load_dataset -MDC_API_KEY = os.getenv("MDC_API_KEY") +MDC_TEST_API_KEY = os.getenv("MDC_TEST_API_KEY") MDC_TEST_API_URL = os.getenv("MDC_TEST_API_URL") pytestmark = pytest.mark.skipif( - not (MDC_API_KEY and MDC_TEST_API_URL), + not (MDC_TEST_API_KEY and MDC_TEST_API_URL), reason="Set MDC_API_KEY and MDC_TEST_API_URL to run live API tests.", ) def test_get_dataset_details_live_api( monkeypatch: MonkeyPatch, - dataset_id: str = "cmhvzlidq0326mn07hk4do3pj", + dataset_id: str = "cmiq2s3q5000fo207k9g6g7ou", ) -> None: """NOTE: This test calls a live MDC API endpoint (dev).""" + monkeypatch.setenv("MDC_API_KEY", MDC_TEST_API_KEY) monkeypatch.setenv("MDC_API_URL", MDC_TEST_API_URL) details = get_dataset_details(dataset_id) @@ -33,10 +34,11 @@ def test_get_dataset_details_live_api( def test_load_dataset_live_api( tmp_path: Path, monkeypatch: MonkeyPatch, - dataset_id: str = "cmhvzlidq0326mn07hk4do3pj", + dataset_id: str = "cmiq2s3q5000fo207k9g6g7ou", ) -> None: """NOTE: This test calls a live MDC API endpoint (dev).""" + monkeypatch.setenv("MDC_API_KEY", MDC_TEST_API_KEY) monkeypatch.setenv("MDC_DOWNLOAD_PATH", str(tmp_path)) monkeypatch.setenv("MDC_API_URL", MDC_TEST_API_URL) From 3e013c5262767ed5c56130f410baa10b0ff46984 Mon Sep 17 00:00:00 2001 From: Kostis-S-Z Date: Wed, 3 Dec 2025 16:20:22 +0200 Subject: [PATCH 09/10] Skip tests if 429 Rate Limit Reached --- tests/e2e/test_e2e.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py index 4470ae4..4aa441a 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -4,6 +4,7 @@ import pandas as pd import pytest from _pytest.monkeypatch import MonkeyPatch +from requests import HTTPError from datacollective import get_dataset_details, load_dataset @@ -16,6 +17,17 @@ ) +def _skip_if_rate_limited(exc: Exception) -> None: + """ + Since our backend implements strict rate limiting + there is a chance that our e2e tests might hit it, + so we skip the tests when backend returns HTTP 429 (rate limit).""" + if isinstance(exc, HTTPError) and getattr(exc, "response", None): + if getattr(exc.response, "status_code", None) == 429: + pytest.skip("Skipped due to API rate limiting (HTTP 429)") + raise exc + + def test_get_dataset_details_live_api( monkeypatch: MonkeyPatch, dataset_id: str = "cmiq2s3q5000fo207k9g6g7ou", @@ -24,7 +36,10 @@ def test_get_dataset_details_live_api( monkeypatch.setenv("MDC_API_KEY", MDC_TEST_API_KEY) monkeypatch.setenv("MDC_API_URL", MDC_TEST_API_URL) - details = get_dataset_details(dataset_id) + try: + details = get_dataset_details(dataset_id) + except Exception as exc: + _skip_if_rate_limited(exc) assert isinstance(details, dict) assert details.get("id") == dataset_id @@ -42,12 +57,15 @@ def test_load_dataset_live_api( monkeypatch.setenv("MDC_DOWNLOAD_PATH", str(tmp_path)) monkeypatch.setenv("MDC_API_URL", MDC_TEST_API_URL) - df = load_dataset( - dataset_id, - download_directory=str(tmp_path), - show_progress=False, - overwrite_existing=True, - ) + try: + df = load_dataset( + dataset_id, + download_directory=str(tmp_path), + show_progress=False, + overwrite_existing=True, + ) + except Exception as exc: # noqa: BLE001 + _skip_if_rate_limited(exc) assert isinstance(df, pd.DataFrame) assert not df.empty From 9c0877ee938bc65aaf3b272fbc941d0018438e78 Mon Sep 17 00:00:00 2001 From: Kostis-S-Z Date: Wed, 3 Dec 2025 16:23:17 +0200 Subject: [PATCH 10/10] Update mention of MDC_TEST_API_KEY --- docs/index.md | 2 +- tests/e2e/test_e2e.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index ca3028b..cab67fd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -133,7 +133,7 @@ Run the full test suite: pytest -v ``` -Note that the e2e tests require a valid `MDC_API_KEY` and a `MDC_TEST_API_URL` key set in your environment. Pytest will skip the live e2e tests automatically if either is missing. +Note that the e2e tests require a valid `MDC_TEST_API_KEY` and a `MDC_TEST_API_URL` key set in your environment. Pytest will skip the live e2e tests automatically if either is missing. ## Release Workflow diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py index 4aa441a..9d1d4d7 100644 --- a/tests/e2e/test_e2e.py +++ b/tests/e2e/test_e2e.py @@ -13,7 +13,7 @@ pytestmark = pytest.mark.skipif( not (MDC_TEST_API_KEY and MDC_TEST_API_URL), - reason="Set MDC_API_KEY and MDC_TEST_API_URL to run live API tests.", + reason="Set MDC_TEST_API_KEY and MDC_TEST_API_URL to run live API tests.", )