diff --git a/.env.example b/.env.example index dd80dc5..d50ecf2 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,3 @@ -MDC_API_KEY= # change to your MDC API Key -MDC_API_URL=https://datacollective.mozillafoundation.org/api # change to MDC API URL endpoint -MDC_DOWNLOAD_PATH=~/.mozdata/datasets # change to where you want to download datasets \ No newline at end of file +MDC_API_KEY= +MDC_API_URL=https://datacollective.mozillafoundation.org/api +MDC_DOWNLOAD_PATH=~/.mozdata/datasets \ No newline at end of file diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..99c7642 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,47 @@ +name: Documentation + +on: + push: + branches: [main] + paths: + - mkdocs.yml + - 'docs/**' + - 'src/**' + pull_request: + paths: + - mkdocs.yml + - 'docs/**' + - 'src/**' + workflow_dispatch: + +jobs: + docs: + permissions: + contents: write + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.9' + cache: "pip" + - name: Configure git + run: | + git config user.name 'github-actions[bot]' + git config user.email 'github-actions[bot]@users.noreply.github.com' + + - name: Install requirements + run: pip install -e '.[docs]' + + - name: Build docs + if: github.event_name == 'pull_request' + run: mkdocs build -s + + - name: Publish docs + if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + run: mkdocs gh-deploy \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2723c92..6936dac 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,4 +1,4 @@ -name: Publish Packages +name: Publish on: pull_request: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..be2ca27 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,34 @@ +name: Tests + +on: + push: + branches: [main] + paths: + - 'src/**' + - 'tests/**' + pull_request: + paths: + - 'src/**' + - 'tests/**' + workflow_dispatch: + +jobs: + run-tests: + timeout-minutes: 30 + runs-on: ubuntu-latest + + steps: + - name: Check out the repository + uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.9' + cache: "pip" + + - name: Install test dependencies + run: pip install -e '.[dev]' + + - name: Run Tests + run: pytest -v tests diff --git a/.gitignore b/.gitignore index 23d12dc..68b9b51 100644 --- a/.gitignore +++ b/.gitignore @@ -175,7 +175,7 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ # Abstra # Abstra is an AI-powered process automation framework. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index baf9892..bf9074c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,19 +1,19 @@ repos: - repo: https://github.com/psf/black - rev: 23.12.1 + rev: 25.11.0 hooks: - id: black language_version: python3.9 - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.1.8 + rev: v0.14.6 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.8.0 + rev: v1.18.2 hooks: - id: mypy additional_dependencies: [types-requests] diff --git a/README.md b/README.md index e22afa6..17805d2 100644 --- a/README.md +++ b/README.md @@ -1,140 +1,70 @@ +

+ + + + + + + Project logo + +

+ +
+ +[![Published](https://github.com/Mozilla-Data-Collective/datacollective-python/actions/workflows/publish.yml/badge.svg)](https://github.com/Mozilla-Data-Collective/datacollective-python/actions/workflows/publish.yml/) +[![Docs](https://github.com/Mozilla-Data-Collective/datacollective-python/actions/workflows/docs.yml/badge.svg)](https://github.com/Mozilla-Data-Collective/datacollective-python/actions/workflows/docs.yml/) +[![Tests](https://github.com/Mozilla-Data-Collective/datacollective-python/actions/workflows/tests.yml/badge.svg)](https://github.com/Mozilla-Data-Collective/datacollective-python/actions/workflows/tests.yml/) + +
+ # Mozilla Data Collective Python API Library Python library for interfacing with the [Mozilla Data Collective](https://datacollective.mozillafoundation.org/) REST API. ## Installation -Install the package using pip: - ```bash pip install datacollective ``` ## Quick Start -1. **Get your API key** from the Mozilla Data Collective dashboard - -2. **Set up your environment**: - -If you have cloned the repository, you can run the following command: - - ```bash - # Copy the example environment file - cp .env.example .env - ``` +1. **Get your API key** from the Mozilla Data Collective [dashboard](https://datacollective.mozillafoundation.org/api-reference) -Otherwise, copy and paste the following into a file called `.env` in your present working directory. +2. **Set the API key in your environment variable (or create `.env` file add it there)**: -```bash -MDC_API_KEY= # change to your MDC API Key -MDC_API_URL=https://datacollective.mozillafoundation.org/api # change to MDC API URL endpoint -MDC_DOWNLOAD_PATH=~/.mozdata/datasets # change to where you want to download datasets ``` - -3. **Configure your API key** by editing `.env`: - ```bash - # Required: Your MDC API key - MDC_API_KEY=your-api-key-here - - # Optional: Download path for datasets (defaults to ~/.mozdata/datasets) - MDC_DOWNLOAD_PATH=~/.mozdata/datasets - ``` - -4. **Start using the library**: - ```python - from datacollective import DataCollective - - # Initialize the client - client = DataCollective() - - # Download a dataset - client.get_dataset('mdc-dataset-id') - ``` - -## Configuration - -The client loads configuration from environment variables or `.env` files: - -- `MDC_API_KEY` - Your Mozilla Data Collective API key (required) -- `MDC_API_URL` - API endpoint (defaults to production) -- `MDC_DOWNLOAD_PATH` - Where to download datasets (defaults to `~/.mozdata/datasets`) - -### Environment Files - -Create a `.env` file in your project root: - -```bash -# MDC API Configuration -MDC_API_KEY=your-api-key-here -MDC_API_URL=https://datacollective.mozillafoundation.org/api -MDC_DOWNLOAD_PATH=~/.mozdata/datasets +export MDC_API_KEY=your-api-key-here ``` -**Note:** Never commit `.env` files to version control as they contain sensitive information. - -## Basic Usage - -```python -from datacollective import DataCollective +3. **Get your dataset ID from the last section of the dataset URL at the MDC website**. -# Initialize client (loads from .env automatically) -client = DataCollective() +For example, in the URL `https://datacollective.mozillafoundation.org/datasets/cmflnuzw43exbql8uukllvnqg`, the dataset ID is `cmflnuzw43exbql8uukllvnqg`. -# Verify your configuration -print(f"API URL: {client.api_url}") -print(f"Download path: {client.download_path}") - -# Download a dataset -dataset = client.get_dataset('your-dataset-id') +4. **Save a dataset locally**: ``` +from datacollective import save_dataset_to_disk -## Load and query datasets - -**note:** today, this feature only works with Mozilla Common Voice datasets +dataset = save_dataset_to_disk("your-dataset-id") ``` -from datacollective import DataCollective -client = DataCollective() +5. **Get information & metadata about a dataset**: -dataset = client.load_dataset("") # Load dasaset into memory -df = dataset.to_pandas() # Convert to pandas for queryable form -dataset.splits # A list of all splits available in the dataset ``` +from datacollective import get_dataset_details - -## Multiple Environments - -You can use different environment configurations: - -```python -# Production environment (default, uses .env) -client = DataCollective() - -# Development environment (uses .env.development) -client = DataCollective(environment='development') - -# Staging environment (uses .env.staging) -client = DataCollective(environment='staging') +details = get_dataset_details("your-dataset-id") ``` -## Release Workflow - -The repository uses branch-specific GitHub Actions for releases: +6. **Load the dataset into a pandas DataFrame _(Only Common Voice datasets are supported right now)_**: -- When a pull request is merged into `main`, the workflow runs the full check suite, bumps the version, and opens a `release/vX.Y.Z` pull request back onto `main`. Auto-merge is enabled on that PR, so once required checks pass the version commit lands on `main` automatically. -- Merge the updated `main` into `test-pypi` to deploy that version to TestPyPI (`uv run python scripts/dev.py publish-test` runs automatically). -- After validating on TestPyPI, merge `main` into `pypi` to deploy to the production PyPI index (`uv run python scripts/dev.py publish` runs automatically). - -Recommended local prep before opening release pull requests: - -1. Run `uv run python scripts/dev.py all` to make sure checks pass without modifying files. -2. Optionally run `uv run python scripts/dev.py prepare-release` locally if you want to rehearse the bump; the workflow performs the same steps when `main` changes. -3. Follow the branch merge order (`main` โžœ `test-pypi`, `main` โžœ `pypi`) so TestPyPI always receives the version before production. +``` +from datacollective import load_dataset -Required GitHub Actions secrets: +dataset = load_dataset("your-dataset-id") +``` -- `TEST_PYPI_API_TOKEN` โ€“ token for publishing to TestPyPI (username `__token__`). -- `PYPI_API_TOKEN` โ€“ token for publishing to PyPI (username `__token__`). +## For more details, visit [our docs](https://Mozilla-Data-Collective.github.io/datacollective-python/) ## License diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..e0a0c4d --- /dev/null +++ b/docs/api.md @@ -0,0 +1,9 @@ +# API Reference + +::: datacollective.datasets + +::: datacollective.api_utils + +::: datacollective.dataset_loading_scripts.registry + +::: datacollective.dataset_loading_scripts.common_voice diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..6f5c6f1 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,132 @@ +# Mozilla Data Collective Python SDK Library + +Welcome to the documentation for the `datacollective` Python client for the +[Mozilla Data Collective](https://datacollective.mozillafoundation.org/) REST API. + +This library helps you: + +- Authenticate with the Mozilla Data Collective. +- Download datasets to local storage. +- Load supported datasets into AI-friendly formats, such as pandas DataFrames. + +## Installation + +Install from PyPI: + +```bash +pip install datacollective +``` + +You can also use uv or other Python tooling as desired, as long as the package datacollective is installed in your environment. + + +## Getting an API Key + +To use the Mozilla Data Collective API, you need an API key: + +1. Sign in to the Mozilla Data Collective dashboard. +2. Create or retrieve an API key from your account/settings page. +3. Keep your key secret and do not commit it to version control. + +## Configuration + +The client reads configuration from environment variables and `.env` files. + +### Environment variables + +Required: + +- `MDC_API_KEY` - Your Mozilla Data Collective API key. + +Optional: + +- `MDC_API_URL` - API endpoint (defaults to the production URL). +- `MDC_DOWNLOAD_PATH` - Local directory where datasets will be downloaded + (defaults to `~/.mozdata/datasets`). + +Example using environment variables directly: + +```bash +export MDC_API_KEY=your-api-key-here +export MDC_API_URL=https://datacollective.mozillafoundation.org/api +export MDC_DOWNLOAD_PATH=~/.mozdata/datasets +``` + +### `.env` file + +The client will automatically load configuration from a `.env` file in your +project root or present working directory. + +Create a file named `.env`: + +```bash +# MDC API Configuration +MDC_API_KEY=your-api-key-here +MDC_API_URL=https://datacollective.mozillafoundation.org/api +MDC_DOWNLOAD_PATH=~/.mozdata/datasets +``` + +> **Security note:** do not commit `.env` files to version control, as they +> contain secrets. + +## Basic Usage + +### Download a dataset + +Use `save_dataset_to_disk` to download a dataset to the configured download path: + +```python +from datacollective import save_dataset_to_disk + +dataset = save_dataset_to_disk("your-dataset-id") + +# Depending on the implementation, `dataset` may contain metadata +# about the downloaded files or a higher-level dataset object. +``` + +The files will be stored under `MDC_DOWNLOAD_PATH` (default `~/.mozdata/datasets`). + +## Loading and Querying Datasets + +> **Note:** in-memory dataset loading is currently supported only for certain datasets. + +You can load supported datasets into memory and convert them to a `pandas` +`DataFrame` for analysis: + +```python +from datacollective import load_dataset + +dataset = load_dataset("your-dataset-id") + +# Convert to pandas +df = dataset.to_pandas() + +# Inspect available splits (e.g., train, dev, test) +print(dataset.splits) +``` + +Once loaded into a `DataFrame`, you can use standard `pandas` operations +to filter, aggregate, and analyze the data. + +## Get dataset details + +You can retrieve info from the datasheet of a dataset without downloading it: + +```python +from datacollective import get_dataset_info + +info = get_dataset_info("your-dataset-id") +print(info) +``` + +## API Reference + +For a detailed API reference, see the [API Reference](api.md) section of the documentation. + +## Release Workflow + +> [!NOTE] +> This section is intended for maintainers of the `datacollective` library. + +Check out the [Release Workflow](release.md) document for details on how to +publish new versions of the library to PyPI using GitHub Actions. \ No newline at end of file diff --git a/docs/mdc_ccc.png b/docs/mdc_ccc.png new file mode 100644 index 0000000..839e4a8 Binary files /dev/null and b/docs/mdc_ccc.png differ diff --git a/docs/mdc_logo.png b/docs/mdc_logo.png new file mode 100644 index 0000000..3fd90d8 Binary files /dev/null and b/docs/mdc_logo.png differ diff --git a/docs/mdc_logo_white.png b/docs/mdc_logo_white.png new file mode 100644 index 0000000..1c5b0f4 Binary files /dev/null and b/docs/mdc_logo_white.png differ diff --git a/docs/release.md b/docs/release.md new file mode 100644 index 0000000..88db84b --- /dev/null +++ b/docs/release.md @@ -0,0 +1,63 @@ +## Release Workflow + +This repository uses GitHub Actions and branch-specific workflows for +publishing releases. + +### Branches + +- `main` - primary development branch. When a pull request is merged into `main` the repository workflow: + - Runs the full check suite. + - Bumps the version. + - Opens a `release/vX.Y.Z` pull request back onto `main`. Auto-merge is enabled on that PR, so once required checks pass the version commit lands on `main` automatically. +- `test-pypi` - receives releases from `main` to deploy to TestPyPI. +- `pypi` - receives releases from `main` to deploy to the production PyPI index. + +### Automated steps + +1. **Prepare release on `main`** + + When a pull request is merged into `main`, the release workflow runs the full checks, performs the version bump, and opens the `release/vX.Y.Z` pull request onto `main`. That PR is configured to auto-merge once required checks complete, so the version commit is applied to `main` without manual intervention. + +2. **Deploy to TestPyPI** + + Merge the updated `main` into `test-pypi` to deploy that version to TestPyPI. The following command runs automatically in the workflow: + + ```bash + uv run python scripts/dev.py publish-test + ``` + +3. **Deploy to PyPI** + + After validating the package on TestPyPI, merge `main` into `pypi` to deploy to production. The following command runs automatically in the workflow: + + ```bash + uv run python scripts/dev.py publish + ``` + +### Recommended local workflow + +Before opening release-related pull requests: + +1. Run the full checks without modifying files: + + ```bash + uv run python scripts/dev.py all + ``` + +2. Optionally rehearse the version bump locally: + + ```bash + uv run python scripts/dev.py prepare-release + ``` + + The repository workflow performs the same steps when `main` changes. + +3. Follow the branch merge order so TestPyPI receives the version before production: + + - `main` -> `test-pypi` + - `main` -> `pypi` + +### Required GitHub Actions secrets + +- `TEST_PYPI_API_TOKEN` - token for publishing to TestPyPI (username `__token__`). +- `PYPI_API_TOKEN` - token for publishing to PyPI (username `__token__`). diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..084af95 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,49 @@ +site_name: Mozilla Data Collective Python SDK +repo_url: https://github.com/Mozilla-Data-Collective/datacollective-python +repo_name: datacollective-python + +nav: + - Home: index.md + - API Reference: api.md + - Release Workflow: release.md + +theme: + icon: + repo: fontawesome/brands/github + name: material + palette: + - scheme: default + primary: black + toggle: + icon: material/lightbulb + name: Switch to dark mode + - scheme: slate + primary: grey + toggle: + icon: material/lightbulb-outline + name: Switch to light mode + logo: docs/mdc_logo.png + favicon: docs/mdc_ccc.png + extra_css: + - assets/custom.css + features: + - content.code.copy + - content.tabs.link + +markdown_extensions: + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true +plugins: +- search +- mkdocstrings: + handlers: + python: + options: + show_root_heading: true \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 4c3f4fb..3fd4808 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,8 +34,15 @@ dev = [ "bump2version>=1.0.0", ] +docs = [ + "mkdocs", + "mkdocs-material", + "mkdocstrings-python", +] + [project.urls] Homepage = "https://github.com/Mozilla-Data-Collective/datacollective-python" +Documentation = "https://Mozilla-Data-Collective.github.io/datacollective-python/" Issues = "https://github.com/Mozilla-Data-Collective/datacollective-python/issues" [project.scripts] diff --git a/scripts/dev.py b/scripts/dev.py deleted file mode 100755 index fdccc5d..0000000 --- a/scripts/dev.py +++ /dev/null @@ -1,301 +0,0 @@ -#!/usr/bin/env python3 -""" -Development scripts for the datacollective package. -Run with: python scripts/dev.py -""" - -import subprocess -import sys -from pathlib import Path - - -def run_command(cmd: list[str]) -> int: - """Run a command and return its exit code.""" - print(f"Running: {' '.join(cmd)}") - result = subprocess.run(cmd) - return result.returncode - - -def format_code() -> int: - """Format code with Black.""" - print("๐ŸŽจ Formatting code with Black...") - return run_command(["uv", "run", "black", "src/", "tests/"]) - - -def format_check() -> int: - """Verify code formatting with Black without modifying files.""" - print("๐ŸŽจ Checking formatting with Black...") - return run_command(["uv", "run", "black", "--check", "src/", "tests/"]) - - -def lint_code() -> int: - """Lint code with Ruff.""" - print("๐Ÿ” Linting code with Ruff...") - return run_command(["uv", "run", "ruff", "check", "src/", "tests/"]) - - -def type_check() -> int: - """Type check with MyPy.""" - print("๐Ÿ”ฌ Type checking with MyPy...") - return run_command(["uv", "run", "mypy", "src/"]) - - -def fix_lint() -> int: - """Fix linting issues automatically.""" - print("๐Ÿ”ง Fixing linting issues...") - return run_command(["uv", "run", "ruff", "check", "--fix", "src/", "tests/"]) - - -def run_tests() -> int: - """Run tests with pytest.""" - print("๐Ÿงช Running tests...") - return run_command(["uv", "run", "pytest", "tests/"]) - - -def bump_version(part: str) -> int: - """Bump version using bump2version.""" - print(f"๐Ÿ“ฆ Bumping {part} version...") - return run_command(["uv", "run", "bump2version", part]) - - -def show_version() -> int: - """Show current version.""" - print("๐Ÿ“‹ Current version information:") - - # Read version from pyproject.toml - pyproject_path = Path("pyproject.toml") - if pyproject_path.exists(): - content = pyproject_path.read_text() - for line in content.split("\n"): - if line.strip().startswith("version = "): - version = line.split('"')[1] - print(f" pyproject.toml: {version}") - break - - # Read version from __init__.py - init_path = Path("src/datacollective/__init__.py") - if init_path.exists(): - content = init_path.read_text() - for line in content.split("\n"): - if "__version__" in line: - version = line.split('"')[1] - print(f" __init__.py: {version}") - break - - return 0 - - -def clean_build() -> int: - """Clean build artifacts.""" - print("๐Ÿงน Cleaning build artifacts...") - import shutil - import os - - # Remove dist directory - if os.path.exists("dist"): - shutil.rmtree("dist") - print(" Removed dist/ directory") - - # Remove build directory - if os.path.exists("build"): - shutil.rmtree("build") - print(" Removed build/ directory") - - # Remove __pycache__ directories - for root, dirs, files in os.walk("."): - for dir_name in dirs[:]: # Use slice to avoid modifying list while iterating - if dir_name == "__pycache__": - shutil.rmtree(os.path.join(root, dir_name)) - print(f" Removed {os.path.join(root, dir_name)}") - dirs.remove(dir_name) - - print("โœ… Cleanup complete!") - return 0 - - -def build_package() -> int: - """Build the package.""" - print("๐Ÿ“ฆ Building package...") - return run_command(["uv", "build"]) - - -def publish_package(index: str = "pypi") -> int: - """Publish package to PyPI or TestPyPI.""" - print(f"๐Ÿš€ Publishing to {index}...") - - # Clean first - if clean_build() != 0: - print("โŒ Clean failed") - return 1 - - # Build package - if build_package() != 0: - print("โŒ Build failed") - return 1 - - # Publish - publish_cmd = ["uv", "publish"] - if index == "testpypi": - publish_cmd.extend(["--publish-url", "https://test.pypi.org/legacy/"]) - return run_command(publish_cmd) - - -def publish_with_bump(index: str = "pypi", part: str = "patch") -> int: - """Bump version and publish package to PyPI or TestPyPI.""" - print(f"๐Ÿš€ Bumping {part} version and publishing to {index}...") - - # Show current version - print("๐Ÿ“‹ Current version:") - if show_version() != 0: - print("โŒ Failed to get current version") - return 1 - - # Run all checks before bumping so we don't consume versions on failure - print("๐Ÿ” Running pre-publish checks...") - if all_checks() != 0: - print("โŒ Pre-publish checks failed") - return 1 - - # Bump version - if bump_version(part) != 0: - print("โŒ Version bump failed") - return 1 - - # Show new version - print("๐Ÿ“‹ New version:") - if show_version() != 0: - print("โŒ Failed to get new version") - return 1 - - # Publish - return publish_package(index) - - -def prepare_release(part: str = "patch") -> int: - """Run checks and bump version without publishing.""" - print(f"๐Ÿš€ Preparing release by bumping {part} version...") - - print("๐Ÿ“‹ Current version:") - if show_version() != 0: - print("โŒ Failed to get current version") - return 1 - - print("๐Ÿ” Running pre-release checks...") - if all_checks() != 0: - print("โŒ Pre-release checks failed") - return 1 - - # uv run may touch uv.lock when syncing environments; discard those changes - lock_path = Path("uv.lock") - if lock_path.exists(): - run_command(["git", "checkout", "--", "uv.lock"]) - - print("๐Ÿ“ฆ Bumping version...") - if bump_version(part) != 0: - print("โŒ Version bump failed") - return 1 - - print("๐Ÿ“‹ New version:") - if show_version() != 0: - print("โŒ Failed to get new version") - return 1 - - return 0 - - -def all_checks() -> int: - """Run all checks: format, lint, type check, and tests.""" - print("๐Ÿš€ Running all checks...") - - # Check formatting - if format_check() != 0: - print("โŒ Formatting check failed") - return 1 - - # Run linting - if lint_code() != 0: - print("โŒ Linting failed") - return 1 - - # Type check - if type_check() != 0: - print("โŒ Type checking failed") - return 1 - - # Run tests - if run_tests() != 0: - print("โŒ Tests failed") - return 1 - - print("โœ… All checks passed!") - return 0 - - -def main(): - """Main entry point.""" - if len(sys.argv) < 2: - print("Usage: python scripts/dev.py ") - print("Commands:") - print(" format - Format code with Black") - print(" lint - Lint code with Ruff") - print(" typecheck - Type check with MyPy") - print(" fix - Fix linting issues automatically") - print(" test - Run tests") - print(" all - Run all checks") - print(" clean - Clean build artifacts") - print(" build - Build package") - print(" publish - Clean, build, and publish to PyPI") - print(" publish-test - Clean, build, and publish to TestPyPI") - print(" publish-bump - Bump patch version and publish to PyPI") - print(" publish-bump-test - Bump patch version and publish to TestPyPI") - print(" prepare-release - Run checks and bump patch version without publishing") - print(" version - Show current version") - print(" bump-patch - Bump patch version (0.0.1 -> 0.0.2)") - print(" bump-minor - Bump minor version (0.0.1 -> 0.1.0)") - print(" bump-major - Bump major version (0.0.1 -> 1.0.0)") - return 1 - - command = sys.argv[1].lower() - - # Handle version bumping commands - if command.startswith("bump-"): - part = command.split("-")[1] - if part in ["patch", "minor", "major"]: - return bump_version(part) - else: - print(f"Unknown version part: {part}") - return 1 - - # Handle publish commands - if command == "publish": - return publish_package("pypi") - elif command == "publish-test": - return publish_package("testpypi") - elif command == "publish-bump": - return publish_with_bump("pypi", "patch") - elif command == "publish-bump-test": - return publish_with_bump("testpypi", "patch") - elif command == "prepare-release": - return prepare_release("patch") - - commands = { - "format": format_code, - "lint": lint_code, - "typecheck": type_check, - "fix": fix_lint, - "test": run_tests, - "all": all_checks, - "clean": clean_build, - "build": build_package, - "version": show_version, - } - - if command not in commands: - print(f"Unknown command: {command}") - return 1 - - return commands[command]() - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/datacollective/__init__.py b/src/datacollective/__init__.py index 125c910..29f731e 100644 --- a/src/datacollective/__init__.py +++ b/src/datacollective/__init__.py @@ -1,13 +1,9 @@ """ Mozilla Data Collective Python Client Library - -Usage: - from datacollective import DataCollective - client = DataCollective() """ -from .client import DataCollective +from .datasets import get_dataset_details, load_dataset, save_dataset_to_disk -__all__ = ["DataCollective"] +__all__ = ["save_dataset_to_disk", "load_dataset", "get_dataset_details"] __version__ = "0.0.34" diff --git a/src/datacollective/api_utils.py b/src/datacollective/api_utils.py new file mode 100644 index 0000000..47a91a7 --- /dev/null +++ b/src/datacollective/api_utils.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import os +from typing import Any + +import requests + +DEFAULT_API_URL = "https://datacollective.mozillafoundation.org/api" +ENV_API_KEY = "MDC_API_KEY" +ENV_API_URL = "MDC_API_URL" +ENV_DOWNLOAD_PATH = "MDC_DOWNLOAD_PATH" +HTTP_TIMEOUT = (10, 60) # (connect, read) + +RATE_LIMIT_ERROR = "Rate limit exceeded. Please try again later." + + +def api_request( + method: str, + url: str, + *, + headers: dict[str, str] | None = None, + timeout: tuple[int, int] | None = None, + raise_known_errors: bool = True, + **kwargs: Any, +) -> requests.Response: + """ + Send an HTTP request with default MDC auth headers and timeout, and + normalize common error codes (403/404/429) to exceptions. + """ + merged_headers = {**_auth_headers(), **(headers or {})} + resp = requests.request( + method=method.upper(), + url=url, + headers=merged_headers, + timeout=HTTP_TIMEOUT if timeout is None else timeout, + **kwargs, + ) + + if raise_known_errors: + if resp.status_code == 404: + raise FileNotFoundError("Dataset not found") + if resp.status_code == 403: + raise PermissionError( + "Access denied. Private dataset requires organization membership" + ) + if resp.status_code == 429: + raise RuntimeError(RATE_LIMIT_ERROR) + resp.raise_for_status() + + return resp + + +def _get_api_url() -> str: + return os.getenv(ENV_API_URL, DEFAULT_API_URL).rstrip("/") + + +def _get_api_key() -> str: + key = os.getenv(ENV_API_KEY) + if not key: + raise ValueError( + f"Missing API key. Set env {ENV_API_KEY} to your MDC API token." + ) + return key + + +def _auth_headers() -> dict[str, str]: + return {"Authorization": f"Bearer {_get_api_key()}"} diff --git a/src/datacollective/client.py b/src/datacollective/client.py deleted file mode 100644 index 9cbbf76..0000000 --- a/src/datacollective/client.py +++ /dev/null @@ -1,369 +0,0 @@ -import os -import shutil -import sys -import tarfile -import time -from pathlib import Path -from typing import Any, Optional, cast - -import requests -from dotenv import load_dotenv - -from .dataset import Dataset - - -class ProgressBar: - """A custom progress bar with a fox emoji that moves across the bar""" - - def __init__(self, total_size: int, bar_length: int = 50): - self.total_size = total_size - self.downloaded = 0 - self.bar_length = bar_length - self.start_time = time.time() - self.last_update_time = 0.0 - self.update_interval = 0.1 # Update every 100ms max - - def update(self, chunk_size: int) -> None: - """Update the progress bar with new downloaded data""" - self.downloaded += chunk_size - - # Only update display if enough time has passed - current_time = time.time() - if current_time - self.last_update_time >= self.update_interval: - self._display() - self.last_update_time = current_time - - def _display(self) -> None: - """Display the current progress bar""" - if self.total_size <= 0: - # If we don't know the total size, show a spinning fox - spinner = ["๐ŸฆŠ", "๐ŸฆŠ", "๐ŸฆŠ", "๐ŸฆŠ"] - spin_char = spinner[int(time.time() * 2) % len(spinner)] - sys.stdout.write( - f"\r{spin_char} Downloading... {self._format_bytes(self.downloaded)}" - ) - sys.stdout.flush() - return - - # Calculate percentage and bar position - percentage = min(100.0, (self.downloaded / self.total_size) * 100) - filled_length = int(self.bar_length * self.downloaded // self.total_size) - - # Create the progress bar - always show fox, even at 0% - bar = "โ–ˆ" * filled_length + "โ–‘" * (self.bar_length - filled_length) - - # Position the fox emoji - always visible at position 0 or current progress - if filled_length == 0: - # Fox at the start when no progress yet - bar = "๐ŸฆŠ" + bar[1:] - else: - # Fox at the leading edge of progress - fox_position = min(filled_length, self.bar_length - 1) - bar = bar[:fox_position] + "๐ŸฆŠ" + bar[fox_position + 1 :] - - # Calculate speed and ETA - elapsed_time = time.time() - self.start_time - if elapsed_time > 0 and self.downloaded > 0: - speed = self.downloaded / elapsed_time - eta = (self.total_size - self.downloaded) / speed if speed > 0 else 0 - speed_str = f"{self._format_bytes(speed)}/s" - eta_str = f"ETA: {self._format_time(eta)}" - else: - speed_str = "0 B/s" - eta_str = "ETA: --:--" - - # Display the progress bar - sys.stdout.write( - f"\r{bar} {percentage:.1f}% " - f"({self._format_bytes(self.downloaded)}/{self._format_bytes(self.total_size)}) " - f"{speed_str} {eta_str}" - ) - sys.stdout.flush() - - def finish(self) -> None: - """Complete the progress bar and move to next line""" - if self.total_size > 0: - # Show completed bar with fox at the end - bar = "โ–ˆ" * (self.bar_length - 1) + "๐ŸฆŠ" - elapsed_time = time.time() - self.start_time - avg_speed = self.downloaded / elapsed_time if elapsed_time > 0 else 0 - sys.stdout.write( - f"\r{bar} 100.0% " - f"({self._format_bytes(self.downloaded)}/{self._format_bytes(self.total_size)}) " - f"Average: {self._format_bytes(avg_speed)}/s " - f"Total time: {self._format_time(elapsed_time)}\n" - ) - else: - elapsed_time = time.time() - self.start_time - avg_speed = self.downloaded / elapsed_time if elapsed_time > 0 else 0 - sys.stdout.write( - f"\n๐ŸฆŠ Download complete! {self._format_bytes(self.downloaded)} " - f"in {self._format_time(elapsed_time)} " - f"(avg: {self._format_bytes(avg_speed)}/s)\n" - ) - sys.stdout.flush() - - @staticmethod - def _format_bytes(bytes_val: float) -> str: - """Format bytes into human readable format""" - for unit in ["B", "KB", "MB", "GB", "TB"]: - if bytes_val < 1024.0: - return f"{bytes_val:.1f} {unit}" - bytes_val /= 1024.0 - return f"{bytes_val:.1f} PB" - - @staticmethod - def _format_time(seconds: float) -> str: - """Format seconds into MM:SS format""" - if seconds < 0: - return "--:--" - mins, secs = divmod(int(seconds), 60) - return f"{mins:02d}:{secs:02d}" - - -class DataCollective: - - def __init__( - self, - api_key: Optional[str] = None, - environment: str = "production", - download_path: Optional[str] = None, - **kwargs: Any, - ) -> None: - """ - Initialize the DataCollective client object - """ - - env = environment or os.getenv("ENVIRONMENT", "development") - env_file = f".env.{env}" if env != "production" else ".env" - - if os.path.exists(env_file): - load_dotenv( - dotenv_path=env_file - ) # load in environmental specific .env file - else: - load_dotenv() # load in default .env file - - # set up API URL - self.api_url = ( - os.getenv("MDC_API_URL") - or "https://datacollective.mozillafoundation.org/api" - ) - if not self.api_url.endswith("/"): - self.api_url += "/" # add trailing slash if it isn't already included - - # set up API Key - self.api_key = api_key or os.getenv("MDC_API_KEY") - - if not self.api_key: - raise ValueError( - "API key missing. Please provide one when creating this object with the api_key parameter or provide it in your .env file as MDC_API_KEY" - ) - - # set up download path - download_path_env = download_path or os.getenv( - "MDC_DOWNLOAD_PATH", "~/.mozdata/datasets" - ) - # Expand user path (handle ~) - self.download_path = os.path.expanduser(download_path_env) # type: ignore - - def _ensure_download_directory(self, download_path: str) -> None: - """ - Ensure the download directory exists and is writable. - Raises an error if the directory cannot be created or is not writable. - """ - try: - # Create the directory if it doesn't exist - Path(download_path).mkdir(parents=True, exist_ok=True) - - # Check if the directory is writable - if not os.access(download_path, os.W_OK): - raise PermissionError(f"Directory {download_path} is not writable") - - except PermissionError as e: - raise PermissionError( - f"Cannot create or write to directory {download_path}: {e}" - ) from e - except Exception as e: - raise OSError(f"Failed to create directory {download_path}: {e}") from e - - def get_dataset( - self, - dataset: str, - download_path: Optional[str] = None, - show_progress: bool = True, - overwrite: bool = False, - ) -> Optional[str]: - """ - Download a dataset from the DataCollective API. - - Args: - dataset (str): The name/ID of the dataset to download - download_path (str, optional): Override the default download path for this download - show_progress (bool): Whether to show the progress bar (default: True) - overwrite (bool): Whether to download a dataset that is already on disk (default: False) - - Returns: - str: The full path to the downloaded file, or None if download failed - """ - - # Determine the download path for this download - if download_path is not None: - # Expand user path (handle ~) - final_download_path = os.path.expanduser(download_path) - else: - final_download_path = self.download_path # type: ignore - - # Ensure the download directory exists and is writable - self._ensure_download_directory(final_download_path) - - # create a download session - download_session_url = self.api_url + "datasets/" + dataset + "/download" - headers = {"Authorization": "Bearer " + self.api_key} # type: ignore - - print(f"Requesting dataset: {dataset}") - try: - r = requests.post(download_session_url, headers=headers) - r.raise_for_status() - # parse response once - response_data = r.json() - except requests.exceptions.HTTPError as e: - if e.response.status_code == 429: # rate limit exceeded - print("Rate limit exceeded") - return None - print(f"HTTP Error: {e}") - return None - except requests.exceptions.RequestException as e: - print(f"Request Error: {e}") - return None - - if "error" in response_data: - response_error = response_data["error"] - if response_error == "Rate limit exceeded": - print("Rate limit exceeded") - return None - else: - print(f"API Error: {response_error}") - return None - - if "downloadUrl" not in response_data or "filename" not in response_data: - print(f"Unexpected response format: {response_data}") - - dataset_file_url = response_data["downloadUrl"] - dataset_filename = response_data["filename"] - - # Create the full file path - full_file_path = os.path.join(final_download_path, dataset_filename) - - # Don't try to re-download a dataset that already exists, unless - # `overwrite` is set to True. - if os.path.exists(full_file_path): - if overwrite: - print( - f"Dataset at {full_file_path} already exists. " - "However, `overwrite` is set to True, so downloading again." - ) - else: - print( - f"Dataset at {full_file_path} already exists. " - "Loading from disk instead of downloading again." - ) - return full_file_path - - # download dataset file - try: - headers = {"Authorization": "Bearer " + self.api_key} # type: ignore - r = requests.get(dataset_file_url, stream=True, headers=headers) - r.raise_for_status() - except requests.exceptions.HTTPError as e: - print(f"HTTP Error Downloading File: {e}") - return None - except requests.exceptions.RequestException as e: - print(f"Request Error Downloading File: {e}") - return None - - # Get the total file size for the progress bar - total_size = int(r.headers.get("content-length", 0)) - - if show_progress: - print(f"Downloading dataset: {dataset_filename}") - progress_bar = ProgressBar(total_size) - # Show initial progress bar with fox at the start - progress_bar._display() - else: - print(f"Downloading dataset: {dataset_filename}") - - # Download with progress tracking - with open(full_file_path, "wb") as f: - for chunk in r.iter_content(chunk_size=65536): # Increased chunk size - if chunk: - f.write(chunk) - if show_progress: - progress_bar.update(len(chunk)) - - if show_progress: - progress_bar.finish() - - print(f"Dataset downloaded to: {full_file_path}") - return full_file_path - - def load_dataset(self, dataset: str, overwrite: bool = False) -> Dataset: - - filepath = self.get_dataset(dataset, overwrite=overwrite) - if not filepath: - raise Exception("Downloading dataset failed") - - extract_path = self._extract_dataset(filepath) - return Dataset(extract_path) - - def _extract_dataset(self, filepath: str) -> str: - - archive_suffix = ".tar.gz" - if filepath.endswith(archive_suffix): - extract_path = filepath[: -len(archive_suffix)] - else: - raise Exception( - f"Downloaded archive {filepath} does not end with {archive_suffix}" - ) - - if os.path.exists(extract_path): - print(f"Deleting old extract {extract_path}") - shutil.rmtree(extract_path) - - print(f"Extracting {filepath} to {extract_path}") - with tarfile.open(filepath, "r:gz") as tar: - tar.extractall(path=extract_path) - print(f"Extracted {filepath} to {extract_path}") - return extract_path - - def get_dataset_details(self, dataset_id: str) -> dict[str, Any]: - """ - Retrieve details of a specific dataset. - - Args: - dataset_id: The dataset ID (as shown in MDC platform). - - Returns: - A dict with dataset details as returned by the API. - - Raises: - ValueError: If dataset_id is empty. - FileNotFoundError: If the dataset does not exist (404). - PermissionError: If access is denied (403). - requests.HTTPError: For other non-2xx responses. - """ - if not dataset_id or not dataset_id.strip(): - raise ValueError("dataset_id is required") - - dataset_details_url = self.api_url + "datasets/" + dataset_id - headers = {"Authorization": "Bearer " + self.api_key} # type: ignore - - resp = requests.get(dataset_details_url, headers=headers) - if resp.status_code == 404: - raise FileNotFoundError("Dataset not found") - if resp.status_code == 403: - raise PermissionError( - "Access denied. Private dataset requires organization membership" - ) - resp.raise_for_status() - return cast(dict[str, Any], resp.json()) diff --git a/src/datacollective/dataset.py b/src/datacollective/dataset.py deleted file mode 100644 index 10a4b98..0000000 --- a/src/datacollective/dataset.py +++ /dev/null @@ -1,105 +0,0 @@ -import os - -import pandas as pd - -SCRIPTED_SPEECH_SPLITS = [ - "dev", - "train", - "test", - "validated", - "invalidated", - "reported", - "other", -] - - -class Dataset: - """ - Represents a dataset. Should be the jumping off point to access its data, metadata, anything that comes from it. - A dataset is backed by a directory, that contains all of its data. - """ - - def __init__(self, directory: str): - self.directory = directory - self.corpus_filepath = None - - @property - def splits(self) -> list[str]: - """ - A list of splits available for the dataset - """ - return [str(x) for x in self._data["split"].dropna().unique().tolist()] - - @property - def _data(self) -> pd.DataFrame: - """ - A single opinion of how a dataset's data should be presented - A table of all splits in a dataset, can be differentiated via the split column - """ - - if "/mcv-scripted-" in self.directory: - return self._get_scripted_speech_data() - elif "/mcv-spontaneous-" in self.directory: - return self._get_spontaneous_speech_data() - else: - raise Exception( - f"Dataset directory {self.directory} cannot be identified as MCV scripted or spontaneous" - ) - - def _get_scripted_speech_data(self) -> pd.DataFrame: - """ - A crude method of getting all of the data for a scripted speech dataset - Transforms it into the canonical representation of several splits of data - In the future, we will aim for a more robust solution - """ - split_files: dict[str, str] = {} - for root, _, files in os.walk(self.directory): - for file in files: - if not file.endswith(".tsv"): - continue - - # Store the corpus directory for reference - self.corpus_filepath = root # type: ignore - full_path = os.path.join(root, file) - data_file_name = file[:-4] - if data_file_name not in SCRIPTED_SPEECH_SPLITS: - continue - - split_files[data_file_name] = full_path - - dfs = [] - for split, file in split_files.items(): - df = pd.read_csv(file, sep="\t", header="infer") - df["split"] = split - dfs.append(df) - return pd.concat(dfs, ignore_index=True) - - def _get_spontaneous_speech_data(self) -> pd.DataFrame: - """ - A crude method of getting all of the data for a spontaneous speech dataset - Transforms it into the canonical representation of several splits of data - In the future, we will aim for a more robust solution - """ - - for root, _, files in os.walk(self.directory): - for file in files: - if not file.startswith("ss-corpus-"): - continue - - if not file.endswith(".tsv"): - continue - - # Store the corpus directory for reference - self.corpus_filepath = root # type: ignore - full_path = os.path.join(root, file) - return pd.read_csv(full_path, sep="\t", header="infer") - - raise Exception("Could nof find dataset file in directory") - - # This may look redundant today, but this is intentionally designed to present an API which is agnostic to its own insides. - # The inside might be anything, you call this to know you've got pandas - def to_pandas(self) -> pd.DataFrame: - """ - Provides the dataset in a pandas format. - """ - return self._data diff --git a/src/datacollective/dataset_loading_scripts/README.md b/src/datacollective/dataset_loading_scripts/README.md new file mode 100644 index 0000000..a8dacad --- /dev/null +++ b/src/datacollective/dataset_loading_scripts/README.md @@ -0,0 +1,3 @@ +`load_dataset()` requires a certain dataset-specific logic in order to parse the data correctly from the downloaded files into a Pandas DataFrame. + +This directory contains dataset loading scripts for different datasets hosted on Mozilla Data Collective to enable the `load_dataset()` functionality. \ No newline at end of file diff --git a/src/datacollective/dataset_loading_scripts/__init__.py b/src/datacollective/dataset_loading_scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/datacollective/dataset_loading_scripts/common_voice.py b/src/datacollective/dataset_loading_scripts/common_voice.py new file mode 100644 index 0000000..f82b74a --- /dev/null +++ b/src/datacollective/dataset_loading_scripts/common_voice.py @@ -0,0 +1,50 @@ +from pathlib import Path + +import pandas as pd + +SCRIPTED_SPEECH_SPLITS = [ + "dev", + "train", + "test", + "validated", + "invalidated", + "reported", + "other", +] + + +def _load_scripted(root_dir: Path) -> pd.DataFrame: + """ + Load Common Voice spontaneous speech datasets from the given root directory. + The function searches for TSV files corresponding to predefined scripted speech splits, + reads them into DataFrames, adds a 'split' column, and concatenates them into a single DataFrame. + """ + split_files: dict[str, Path] = {} + for path in root_dir.rglob("*.tsv"): + split_name = path.stem + if split_name in SCRIPTED_SPEECH_SPLITS: + split_files[split_name] = path + + if not split_files: + raise RuntimeError(f"No scripted split files found under `{str(root_dir)}`") + + frames = [] + for split, file_path in sorted(split_files.items()): + df = pd.read_csv(file_path, sep="\t", header="infer") + df["split"] = split + frames.append(df) + return pd.concat(frames, ignore_index=True) + + +def _load_spontaneous(root_dir: Path) -> pd.DataFrame: + """ + Load Common Voice spontaneous speech datasets from the given root directory. + The function searches for a TSV file with a name starting with 'ss-corpus-', + reads it into a DataFrame, and returns it. + """ + for path in root_dir.rglob("*.tsv"): + if path.name.startswith("ss-corpus-"): + return pd.read_csv(path, sep="\t", header="infer") + raise RuntimeError( + f"No spontaneous corpus file (`ss-corpus-*.tsv`) found under `{str(root_dir)}`" + ) diff --git a/src/datacollective/dataset_loading_scripts/registry.py b/src/datacollective/dataset_loading_scripts/registry.py new file mode 100644 index 0000000..8637c7d --- /dev/null +++ b/src/datacollective/dataset_loading_scripts/registry.py @@ -0,0 +1,35 @@ +from pathlib import Path + +import pandas as pd + +from datacollective.dataset_loading_scripts.common_voice import ( + _load_scripted, + _load_spontaneous, +) + + +def load_dataset_from_name_as_dataframe( + dataset_name: str, extract_dir: Path +) -> pd.DataFrame: + """ + In order to enable loading MDC datasets as Pandas DataFrames, this function + routes the loading process to the appropriate dataset-specific loader based on + the dataset name. Each dataset loader is implemented in its own module under + `datacollective.dataset_loading_scripts`. + + Args: + dataset_name (str): The name of the dataset (lowercased). + extract_dir (Path): The directory where the dataset has been extracted. + Returns: + A pandas DataFrame containing the loaded dataset. + Raises: + ValueError: If the dataset name is not supported for loading. + """ + if "scripted" in dataset_name: + return _load_scripted(extract_dir) + if "spontaneous" in dataset_name: + return _load_spontaneous(extract_dir) + + raise ValueError( + f"Dataset name `{dataset_name}` currently not supported for loading as DataFrame." + ) diff --git a/src/datacollective/datasets.py b/src/datacollective/datasets.py new file mode 100644 index 0000000..ea08155 --- /dev/null +++ b/src/datacollective/datasets.py @@ -0,0 +1,235 @@ +from __future__ import annotations + +import os +import tarfile +import zipfile +from pathlib import Path +from typing import Any + +import pandas as pd + +from datacollective.api_utils import ( + ENV_DOWNLOAD_PATH, + HTTP_TIMEOUT, + _get_api_url, + api_request, +) +from datacollective.dataset_loading_scripts.registry import ( + load_dataset_from_name_as_dataframe, +) +from datacollective.progress_bar import ProgressBar + + +def get_dataset_details(dataset_id: str) -> dict[str, Any]: + """ + Return dataset details from the MDC API as a dictionary. + Args: + dataset_id: The dataset ID (as shown in MDC platform). + Returns: + A dict with dataset details as returned by the API. + Raises: + ValueError: If dataset_id is empty. + FileNotFoundError: If the dataset does not exist (404). + PermissionError: If access is denied (403). + RuntimeError: If rate limit is exceeded (429). + requests.HTTPError: For other non-2xx responses. + """ + if not dataset_id or not dataset_id.strip(): + raise ValueError("`dataset_id` must be a non-empty string") + + url = f"{_get_api_url()}/datasets/{dataset_id}" + resp = api_request("GET", url) + return dict(resp.json()) + + +def save_dataset_to_disk( + dataset_id: str, + download_directory: str | None = None, + show_progress: bool = True, + overwrite_existing: bool = False, +) -> Path: + """ + Download the dataset archive to a local directory and return the archive path. + Skips download if the target file already exists (unless `overwrite_existing=True`). + Args: + dataset_id: The dataset ID (as shown in MDC platform). + download_directory: Directory where to save the downloaded dataset. + If None or empty, falls back to env MDC_DOWNLOAD_PATH or default. + show_progress: Whether to show a progress bar during download. + overwrite_existing: Whether to overwrite existing files. + Returns: + Path to the downloaded dataset archive. + Raises: + ValueError: If dataset_id is empty. + FileNotFoundError: If the dataset does not exist (404). + PermissionError: If access is denied (403) or download directory is not writable. + RuntimeError: If rate limit is exceeded (429) or unexpected response format. + requests.HTTPError: For other non-2xx responses. + """ + if not dataset_id or not dataset_id.strip(): + raise ValueError("`dataset_id` must be a non-empty string") + + base_dir = _resolve_download_dir(download_directory) + + # Create a download session to get `downloadUrl` and `filename` + session_url = f"{_get_api_url()}/datasets/{dataset_id}/download" + resp = api_request("POST", session_url) + payload: dict[str, Any] = dict(resp.json()) + + download_url = payload.get("downloadUrl") + filename = payload.get("filename") + if not download_url or not filename: + raise RuntimeError(f"Unexpected response format: {payload}") + + target_path = base_dir / filename + if target_path.exists() and not overwrite_existing: + print(f"File already exists. Skipping download: `{str(target_path)}`") + return Path(target_path) + + # Stream download to a temporary file for atomicity + tmp_path = target_path.with_suffix(target_path.suffix + ".part") + + with api_request( + "GET", + download_url, + stream=True, + timeout=HTTP_TIMEOUT, + ) as r: + total = int(r.headers.get("content-length", "0")) + + if show_progress: + print(f"Downloading dataset: {filename}") + progress_bar = ProgressBar(total) + # Show initial progress bar with fox at the start + progress_bar._display() + else: + print(f"Downloading dataset: {filename}") + + with open(tmp_path, "wb") as f: + for chunk in r.iter_content(chunk_size=1 << 16): + if not chunk: + continue + f.write(chunk) + if show_progress: + progress_bar.update(len(chunk)) + + if show_progress: + progress_bar.finish() + + tmp_path.replace(target_path) + print(f"Saved dataset to `{str(target_path)}`") + return Path(target_path) + + +def load_dataset( + dataset_id: str, + download_directory: str | None = None, + show_progress: bool = True, + overwrite_existing: bool = False, +) -> pd.DataFrame: + """ + Download (if needed), extract, and load the dataset into a pandas DataFrame. + Uses dataset `details['name']` to check in registry.py for dataset-specific loading logic. + Args: + dataset_id: The dataset ID (as shown in MDC platform). + download_directory: Directory where to save the downloaded dataset. + If None or empty, falls back to env MDC_DOWNLOAD_PATH or default. + show_progress: Whether to show a progress bar during download. + overwrite_existing: Whether to overwrite existing files. + Returns: + A pandas DataFrame with the loaded dataset. + Raises: + ValueError: If dataset_id is empty. + FileNotFoundError: If the dataset does not exist (404). + PermissionError: If access is denied (403) or download directory is not writable. + RuntimeError: If rate limit is exceeded (429) or unexpected response format. + requests.HTTPError: For other non-2xx responses. + """ + archive_path = save_dataset_to_disk( + dataset_id=dataset_id, + download_directory=download_directory, + show_progress=show_progress, + overwrite_existing=overwrite_existing, + ) + base_dir = _resolve_download_dir(download_directory) + extract_dir = _extract_archive(archive_path, base_dir) + + details = get_dataset_details(dataset_id) + dataset_name = str(details.get("name", "")).lower() + + return load_dataset_from_name_as_dataframe(dataset_name, extract_dir) + + +def _resolve_download_dir(download_directory: str | None) -> Path: + """ + Resolve and ensure the download directory exists and is writable. + + Args: + download_directory (str | None): User-specified download directory. + If None or empty, falls back to env MDC_DOWNLOAD_PATH or default. + + Returns: + The resolved Path object for the download directory. + """ + if download_directory and download_directory.strip(): + base = download_directory + else: + base = os.getenv(ENV_DOWNLOAD_PATH, "~/.mozdata/datasets") + p = Path(os.path.expanduser(base)) + p.mkdir(parents=True, exist_ok=True) + if not os.access(p, os.W_OK): + raise PermissionError(f"Directory `{str(p)}` is not writable") + return p + + +def _strip_archive_suffix(path: Path) -> Path: + """ + Strip known archive suffixes from the filename. + Args: + path: Path to the archive file. + Returns: + Path with the archive suffix removed. + """ + name = path.name + if name.endswith(".tar.gz"): + return path.with_name(name[: -len(".tar.gz")]) + if name.endswith(".tgz"): + return path.with_name(name[: -len(".tgz")]) + if name.endswith(".zip"): + return path.with_name(name[: -len(".zip")]) + # Unknown; drop one suffix if present + return path.with_suffix("") + + +def _extract_archive(archive_path: Path, dest_dir: Path) -> Path: + """ + Extract the given archive (.tar.gz, .tgz, .zip) into `dest_dir`. + Args: + archive_path: Path to the archive file. + dest_dir: Directory where to extract the contents. + Returns: + Path to the extracted root directory. + Raises: + ValueError: If the archive type is unsupported. + """ + extract_root = _strip_archive_suffix(archive_path) + # Extract into a dedicated directory under `dest_dir` using stripped name + target = dest_dir / extract_root.name + if target.exists(): + # Keep it simple and ensure fresh state + import shutil + + shutil.rmtree(target) + target.mkdir(parents=True, exist_ok=True) + + if archive_path.suffix == ".zip": + with zipfile.ZipFile(archive_path, "r") as zf: + zf.extractall(target) + elif archive_path.name.endswith(".tar.gz") or archive_path.suffix == ".tgz": + with tarfile.open(archive_path, "r:gz") as tf: + tf.extractall(target) + else: + raise ValueError( + f"Unsupported archive type for `{archive_path.name}`. Expected .tar.gz, .tgz, or .zip." + ) + return target diff --git a/src/datacollective/progress_bar.py b/src/datacollective/progress_bar.py new file mode 100644 index 0000000..1a52516 --- /dev/null +++ b/src/datacollective/progress_bar.py @@ -0,0 +1,111 @@ +import sys +import time + +class ProgressBar: + """A custom progress bar with a fox emoji that moves across the bar""" + + def __init__(self, total_size: int, bar_length: int = 50): + self.total_size = total_size + self.downloaded = 0 + self.bar_length = bar_length + self.start_time = time.time() + self.last_update_time = 0.0 + self.update_interval = 0.1 # Update every 100ms max + + def update(self, chunk_size: int) -> None: + """Update the progress bar with new downloaded data""" + self.downloaded += chunk_size + + # Only update display if enough time has passed + current_time = time.time() + if current_time - self.last_update_time >= self.update_interval: + self._display() + self.last_update_time = current_time + + def _display(self) -> None: + """Display the current progress bar""" + if self.total_size <= 0: + # If we don't know the total size, show a spinning fox + spinner = ["๐ŸฆŠ", "๐ŸฆŠ", "๐ŸฆŠ", "๐ŸฆŠ"] + spin_char = spinner[int(time.time() * 2) % len(spinner)] + sys.stdout.write( + f"\r{spin_char} Downloading... {self._format_bytes(self.downloaded)}" + ) + sys.stdout.flush() + return + + # Calculate percentage and bar position + percentage = min(100.0, (self.downloaded / self.total_size) * 100) + filled_length = int(self.bar_length * self.downloaded // self.total_size) + + # Create the progress bar - always show fox, even at 0% + bar = "โ–ˆ" * filled_length + "โ–‘" * (self.bar_length - filled_length) + + # Position the fox emoji - always visible at position 0 or current progress + if filled_length == 0: + # Fox at the start when no progress yet + bar = "๐ŸฆŠ" + bar[1:] + else: + # Fox at the leading edge of progress + fox_position = min(filled_length, self.bar_length - 1) + bar = bar[:fox_position] + "๐ŸฆŠ" + bar[fox_position + 1 :] + + # Calculate speed and ETA + elapsed_time = time.time() - self.start_time + if elapsed_time > 0 and self.downloaded > 0: + speed = self.downloaded / elapsed_time + eta = (self.total_size - self.downloaded) / speed if speed > 0 else 0 + speed_str = f"{self._format_bytes(speed)}/s" + eta_str = f"ETA: {self._format_time(eta)}" + else: + speed_str = "0 B/s" + eta_str = "ETA: --:--" + + # Display the progress bar + sys.stdout.write( + f"\r{bar} {percentage:.1f}% " + f"({self._format_bytes(self.downloaded)}/{self._format_bytes(self.total_size)}) " + f"{speed_str} {eta_str}" + ) + sys.stdout.flush() + + def finish(self) -> None: + """Complete the progress bar and move to next line""" + if self.total_size > 0: + # Show completed bar with fox at the end + bar = "โ–ˆ" * (self.bar_length - 1) + "๐ŸฆŠ" + elapsed_time = time.time() - self.start_time + avg_speed = self.downloaded / elapsed_time if elapsed_time > 0 else 0 + sys.stdout.write( + f"\r{bar} 100.0% " + f"({self._format_bytes(self.downloaded)}/{self._format_bytes(self.total_size)}) " + f"Average: {self._format_bytes(avg_speed)}/s " + f"Total time: {self._format_time(elapsed_time)}\n" + ) + else: + elapsed_time = time.time() - self.start_time + avg_speed = self.downloaded / elapsed_time if elapsed_time > 0 else 0 + sys.stdout.write( + f"\n๐ŸฆŠ Download complete! {self._format_bytes(self.downloaded)} " + f"in {self._format_time(elapsed_time)} " + f"(avg: {self._format_bytes(avg_speed)}/s)\n" + ) + sys.stdout.flush() + + @staticmethod + def _format_bytes(bytes_val: float) -> str: + """Format bytes into human readable format""" + for unit in ["B", "KB", "MB", "GB", "TB"]: + if bytes_val < 1024.0: + return f"{bytes_val:.1f} {unit}" + bytes_val /= 1024.0 + return f"{bytes_val:.1f} PB" + + @staticmethod + def _format_time(seconds: float) -> str: + """Format seconds into MM:SS format""" + if seconds < 0: + return "--:--" + mins, secs = divmod(int(seconds), 60) + return f"{mins:02d}:{secs:02d}" + diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 8ae4dd1..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,37 +0,0 @@ -import os - -import pytest - - -@pytest.fixture(autouse=True) -def clean_environment(): - """Automatically clean environment variables before each test.""" - # Store original environment - original_env = os.environ.copy() - - # Clear MDC-related variables - for key in ["MDC_API_KEY", "MDC_API_URL", "ENVIRONMENT"]: - os.environ.pop(key, None) - - yield - - # Restore original environment - os.environ.clear() - os.environ.update(original_env) - - -@pytest.fixture -def mock_env_file(tmp_path): - """Create a temporary directory with mock .env files.""" - # Create temporary .env files - env_file = tmp_path / ".env" - env_file.write_text( - "MDC_API_KEY=prod-test-key\nMDC_API_URL=https://prod.test.url\n" - ) - - dev_env_file = tmp_path / ".env.development" - dev_env_file.write_text( - "MDC_API_KEY=dev-test-key\nMDC_API_URL=https://dev.test.url\n" - ) - - return tmp_path diff --git a/tests/test_client.py b/tests/test_client.py deleted file mode 100644 index f28427d..0000000 --- a/tests/test_client.py +++ /dev/null @@ -1,261 +0,0 @@ -import os -import tempfile -from unittest.mock import patch - -import pytest -import requests - -from datacollective import DataCollective - - -class TestDataCollective: - """Test suite for DataCollective client.""" - - def test_init_with_api_key_parameter(self): - """Test initialization with API key passed as parameter.""" - client = DataCollective(api_key="test-api-key-123") - assert client.api_key == "test-api-key-123" - assert client.api_url == "https://datacollective.mozillafoundation.org/api/" - - def test_init_with_env_variable(self): - """Test initialization with API key from environment variable.""" - with patch.dict(os.environ, {"MDC_API_KEY": "env-api-key-456"}): - client = DataCollective() - assert client.api_key == "env-api-key-456" - - def test_init_missing_api_key_raises_error(self): - """Test that missing API key raises ValueError.""" - with patch.dict(os.environ, {}, clear=True): - with patch( - "datacollective.client.load_dotenv" - ): # Mock to prevent loading from file - with pytest.raises(ValueError) as exc_info: - DataCollective() - assert "API key missing" in str(exc_info.value) - - def test_custom_api_url_from_env(self): - """Test custom API URL from environment variable.""" - with patch.dict( - os.environ, - {"MDC_API_KEY": "test-key", "MDC_API_URL": "https://custom.api.url"}, - ): - client = DataCollective() - assert client.api_url == "https://custom.api.url/" - - def test_default_api_url_when_env_not_set(self): - """Test default API URL is used when env variable not set.""" - with patch.dict(os.environ, {"MDC_API_KEY": "test-key"}): - # Ensure MDC_API_URL is not set - os.environ.pop("MDC_API_URL", None) - client = DataCollective() - assert client.api_url == "https://datacollective.mozillafoundation.org/api/" - - def test_environment_parameter_loads_correct_env_file(self): - """Test that different environment parameter loads correct .env file.""" - # Create temporary .env files - with tempfile.TemporaryDirectory() as tmpdir: - # Create .env.development file - dev_env_file = os.path.join(tmpdir, ".env.development") - with open(dev_env_file, "w") as f: - f.write("MDC_API_KEY=dev-key-789\n") - f.write("MDC_API_URL=https://dev.api.url\n") - - # Change to temp directory and create client - original_cwd = os.getcwd() - try: - os.chdir(tmpdir) - client = DataCollective(environment="development") - assert client.api_key == "dev-key-789" - assert client.api_url == "https://dev.api.url/" - finally: - os.chdir(original_cwd) - - def test_production_environment_loads_default_env(self): - """Test that production environment loads default .env file.""" - with tempfile.TemporaryDirectory() as tmpdir: - # Create default .env file - env_file = os.path.join(tmpdir, ".env") - with open(env_file, "w") as f: - f.write("MDC_API_KEY=prod-key-999\n") - - original_cwd = os.getcwd() - try: - os.chdir(tmpdir) - client = DataCollective(environment="production") - assert client.api_key == "prod-key-999" - finally: - os.chdir(original_cwd) - - def test_parameter_overrides_env_variable(self): - """Test that parameter takes precedence over environment variable.""" - with patch.dict(os.environ, {"MDC_API_KEY": "env-key"}): - client = DataCollective(api_key="param-key") - assert client.api_key == "param-key" - - @patch("datacollective.client.requests.post") - def test_get_dataset_handles_http_error(self, mock_post): - """Test that get_dataset handles HTTP errors properly.""" - # Mock a 403 Forbidden response - mock_response = mock_post.return_value - mock_response.status_code = 403 - - # Create a proper HTTPError with response - http_error = requests.exceptions.HTTPError("403 Client Error: Forbidden") - http_error.response = mock_response - mock_response.raise_for_status.side_effect = http_error - - client = DataCollective(api_key="test-key") - result = client.get_dataset("test-dataset") - - assert result is None - mock_post.assert_called_once() - - -class TestDataCollectiveWithMocking: - """Tests using mocking for isolation.""" - - @patch("datacollective.client.load_dotenv") - def test_load_dotenv_called_for_development(self, mock_load_dotenv): - """Test that load_dotenv is called with correct path for development.""" - with patch.dict(os.environ, {"MDC_API_KEY": "test-key"}): - with patch("os.path.exists", return_value=True): - DataCollective(environment="development") - mock_load_dotenv.assert_called_once_with(dotenv_path=".env.development") - - @patch("datacollective.client.load_dotenv") - def test_load_dotenv_fallback_when_env_file_missing(self, mock_load_dotenv): - """Test that load_dotenv falls back to default when env file doesn't exist.""" - with patch.dict(os.environ, {"MDC_API_KEY": "test-key"}): - with patch("os.path.exists", return_value=False): - DataCollective(environment="staging") - mock_load_dotenv.assert_called_once_with() - - -# Fixtures for shared test data -@pytest.fixture -def api_key(): - """Fixture providing a test API key.""" - return "test-api-key-fixture" - - -@pytest.fixture -def client(api_key): - """Fixture providing a DataCollective client.""" - return DataCollective(api_key=api_key) - - -class TestDataCollectiveWithFixtures: - """Tests using fixtures for common setup.""" - - def test_client_fixture_has_api_key(self, client, api_key): - """Test that fixture-provided client has correct API key.""" - assert client.api_key == api_key - - def test_client_fixture_has_default_url(self, client): - """Test that fixture-provided client has default URL.""" - assert client.api_url == "https://datacollective.mozillafoundation.org/api/" - - -class TestGetDatasetDetails: - """Tests for get_dataset_details.""" - - @patch("datacollective.client.requests.get") - def test_get_dataset_details_success(self, mock_get, api_key): - mock_resp = mock_get.return_value - mock_resp.status_code = 200 - mock_resp.json.return_value = {"id": "abc123", "name": "Example Dataset"} - - client = DataCollective(api_key=api_key) - result = client.get_dataset_details("abc123") - - assert result == {"id": "abc123", "name": "Example Dataset"} - - # Verify URL and Authorization header - called_url = mock_get.call_args[0][0] - called_headers = mock_get.call_args[1]["headers"] - assert called_url == client.api_url + "datasets/abc123" - assert called_headers["Authorization"] == f"Bearer {api_key}" - - @pytest.mark.parametrize("bad_id", ["", " ", " "]) - @patch("datacollective.client.requests.get") - def test_get_dataset_details_empty_id_raises(self, mock_get, bad_id): - client = DataCollective(api_key="test-key") - with pytest.raises(ValueError, match="dataset_id is required"): - client.get_dataset_details(bad_id) - mock_get.assert_not_called() - - @patch("datacollective.client.requests.get") - def test_get_dataset_details_404_raises_file_not_found(self, mock_get): - mock_resp = mock_get.return_value - mock_resp.status_code = 404 - - client = DataCollective(api_key="test-key") - with pytest.raises(FileNotFoundError, match="Dataset not found"): - client.get_dataset_details("missing-dataset") - - @patch("datacollective.client.requests.get") - def test_get_dataset_details_403_raises_permission_error(self, mock_get): - mock_resp = mock_get.return_value - mock_resp.status_code = 403 - - client = DataCollective(api_key="test-key") - with pytest.raises( - PermissionError, - match=r"Access denied\. Private dataset requires organization membership", - ): - client.get_dataset_details("private-dataset") - - @patch("datacollective.client.requests.get") - def test_get_dataset_details_other_http_error_propagates(self, mock_get): - mock_resp = mock_get.return_value - mock_resp.status_code = 500 - http_err = requests.exceptions.HTTPError( - "500 Server Error: Internal Server Error" - ) - mock_resp.raise_for_status.side_effect = http_err - - client = DataCollective(api_key="test-key") - with pytest.raises(requests.exceptions.HTTPError): - client.get_dataset_details("abc123") - - -def test_get_dataset_details_live_roundtrip(): - from dotenv import load_dotenv - - load_dotenv() - api_key = os.getenv("MDC_API_KEY") - if not api_key: - pytest.skip("MDC_API_KEY not set; skipping live roundtrip test") - - dataset_id = ( - "cmflnuzw414x7bnapn6iycjnv" # Common Voice Scripted Speech 23.0 - Bengali - ) - - client = DataCollective(api_key=api_key) - details = client.get_dataset_details(dataset_id) - - assert isinstance(details, dict) - assert details.get("id") == dataset_id - assert isinstance(details.get("slug"), str) and details["slug"] - assert isinstance(details.get("name"), str) and details["name"] - assert isinstance(details.get("locale"), str) and details["locale"] - visibility = details.get("visibility") - if visibility is not None: - assert isinstance(visibility, str) - assert visibility in ("public", "private", "restricted") - assert isinstance(details.get("sizeBytes"), str) - assert isinstance(details.get("createdAt"), str) and details["createdAt"].endswith( - "Z" - ) - updated_at = details.get("updatedAt") - if updated_at is not None: - assert isinstance(updated_at, str) - assert updated_at.endswith("Z") - org = details.get("organization") - assert isinstance(org, dict) - assert isinstance(org.get("name"), str) and org["name"] - assert isinstance(org.get("slug"), str) and org["slug"] - expected_dataset_url = ( - client.api_url.replace("/api/", "/") + "datasets/" + dataset_id - ) - assert details.get("datasetUrl") == expected_dataset_url diff --git a/tests/test_dataset.py b/tests/test_dataset.py deleted file mode 100644 index e33222b..0000000 --- a/tests/test_dataset.py +++ /dev/null @@ -1,89 +0,0 @@ -import pandas as pd -import pytest - -from datacollective.dataset import SCRIPTED_SPEECH_SPLITS, Dataset - - -@pytest.fixture -def scripted_dataset_dir(tmp_path): - """Create a fake MCV scripted dataset directory with valid .tsv split files.""" - base_dir = tmp_path / "mcv-scripted-en" - base_dir.mkdir() - for split in ["train", "test", "validated"]: - df = pd.DataFrame({"text": [f"{split}_1", f"{split}_2"], "speaker": [1, 2]}) - file_path = base_dir / f"{split}.tsv" - df.to_csv(file_path, sep="\t", index=False) - return base_dir - - -@pytest.fixture -def spontaneous_dataset_dir(tmp_path): - """Create a fake MCV spontaneous dataset directory with one ss-corpus file.""" - base_dir = tmp_path / "mcv-spontaneous-en" - base_dir.mkdir() - df = pd.DataFrame({"utterance": ["hello", "world"], "speaker": [1, 2]}) - (base_dir / "ss-corpus-data.tsv").write_text(df.to_csv(sep="\t", index=False)) - return base_dir - - -def test_scripted_dataset_loads_correctly(scripted_dataset_dir): - ds = Dataset(str(scripted_dataset_dir)) - df = ds.to_pandas() - - # Should contain concatenated data from all splits - assert set(df["split"].unique()) == {"train", "test", "validated"} - assert all(col in df.columns for col in ["text", "speaker", "split"]) - assert len(df) == 6 # 3 splits ร— 2 rows each - - -def test_scripted_splits_property(scripted_dataset_dir): - ds = Dataset(str(scripted_dataset_dir)) - splits = ds.splits - assert sorted(splits) == ["test", "train", "validated"] - - -def test_spontaneous_dataset_loads_correctly(spontaneous_dataset_dir): - ds = Dataset(str(spontaneous_dataset_dir)) - df = ds.to_pandas() - - assert set(df.columns) == {"utterance", "speaker"} - assert len(df) == 2 - assert df.iloc[0]["utterance"] == "hello" - - -def test_spontaneous_dataset_missing_file_raises(tmp_path): - base_dir = tmp_path / "mcv-spontaneous-en" - base_dir.mkdir() - - ds = Dataset(str(base_dir)) - with pytest.raises(Exception, match="Could nof find dataset file in directory"): - ds.to_pandas() - - -def test_invalid_dataset_dir_raises(tmp_path): - base_dir = tmp_path / "some-random-dataset" - base_dir.mkdir() - ds = Dataset(str(base_dir)) - - with pytest.raises( - Exception, match="cannot be identified as MCV scripted or spontaneous" - ): - ds.to_pandas() - - -def test_get_scripted_speech_splits_filters_only_valid_names(tmp_path): - base_dir = tmp_path / "mcv-scripted-en" - base_dir.mkdir() - # valid and invalid split names - valid_file = base_dir / "train.tsv" - invalid_file = base_dir / "random.tsv" - - pd.DataFrame({"x": [1]}).to_csv(valid_file, sep="\t", index=False) - pd.DataFrame({"x": [1]}).to_csv(invalid_file, sep="\t", index=False) - - ds = Dataset(str(base_dir)) - df = ds._get_scripted_speech_data() - - assert "split" in df.columns - assert all(df["split"].isin(SCRIPTED_SPEECH_SPLITS)) - assert "random" not in df["split"].unique()