From 35089f4ec6cff428077f0772cf834f145e3f4b65 Mon Sep 17 00:00:00 2001 From: Kostis-S-Z Date: Tue, 4 Aug 2026 16:05:08 +0300 Subject: [PATCH 1/5] Add to model new fields around compensated datasets --- src/datacollective/models.py | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/datacollective/models.py b/src/datacollective/models.py index 80ad5a9..7b5d867 100644 --- a/src/datacollective/models.py +++ b/src/datacollective/models.py @@ -6,6 +6,11 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +# Price bounds enforced by the platform for compensated datasets in USD cents +MIN_DATASET_PRICE_CENTS = 10_000 # 100 USD +MAX_DATASET_PRICE_CENTS = 5_000_000 # 50,000 USD + + class UploadPart(BaseModel): """A single multipart upload part.""" @@ -164,6 +169,14 @@ class Dataset(BaseModel): None, description="Dataset visibility (e.g., `public`, `private`, `restricted`).", ) + isPaid: bool | None = Field( + None, + description="Whether the dataset is compensated, i.e. has a price. Defaults to `False` on the platform when left unset.", + ) + basePriceCents: int | None = Field( + None, + description="Price of the dataset in USD cents (e.g. `100_000` is $1,000.00). Required when `isPaid` is True.", + ) # Defined by the API and not user-editable id: str | None = Field( None, description="Unique identifier as returned by the API." @@ -208,6 +221,17 @@ class DatasetSubmission(NonEmptyStrModel, Dataset): None, description="Dataset visibility: `public`, `private`, or `restricted`.", ) + basePriceCents: int | None = Field( + None, + ge=MIN_DATASET_PRICE_CENTS, + le=MAX_DATASET_PRICE_CENTS, + description=( + "Price of the dataset in USD cents (e.g. `100_000` is $1,000.00). Required when " + f"`isPaid` is True and must be between {MIN_DATASET_PRICE_CENTS} " + f"({MIN_DATASET_PRICE_CENTS // 100} USD) and {MAX_DATASET_PRICE_CENTS} " + f"({MAX_DATASET_PRICE_CENTS // 100} USD) cents." + ), + ) # Submission-specific fields defined by the user createdByFullName: str | None = Field(None, description="Creator's name.") createdByEmail: str | None = Field(None, description="Creator's email.") @@ -254,6 +278,20 @@ def _validate_license_details(self) -> DatasetSubmission: ) return self + @model_validator(mode="after") + def _validate_pricing(self) -> DatasetSubmission: + if self.isPaid and self.basePriceCents is None: + raise ValueError( + "`basePriceCents` is required when `isPaid` is True and must be between " + f"{MIN_DATASET_PRICE_CENTS} and {MAX_DATASET_PRICE_CENTS} USD cents" + ) + if self.basePriceCents is not None and not self.isPaid: + raise ValueError( + "`isPaid` must be True when providing `basePriceCents`, " + "otherwise the dataset stays uncompensated and the price is ignored" + ) + return self + class DatasetDetails(Dataset): """ @@ -345,6 +383,8 @@ def get(self, key: str, default: Any = None) -> Any: "showContactInfo", "visibility", "exclusivityOptOut", + "isPaid", + "basePriceCents", } SUBMIT_FIELDS = {"agreeToSubmit"} From 00020cfd9fe8212d541d70608bd5f5d9cc68cd0d Mon Sep 17 00:00:00 2001 From: Kostis-S-Z Date: Tue, 4 Aug 2026 16:05:23 +0300 Subject: [PATCH 2/5] Update demo code --- docs/demo_upload.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/demo_upload.py b/docs/demo_upload.py index d0d08a4..a81e1e5 100644 --- a/docs/demo_upload.py +++ b/docs/demo_upload.py @@ -36,6 +36,8 @@ "or considerations related to data collection and usage.", showContactInfo=False, # Whether to publicly display the contact information above visibility=Visibility.PUBLIC, # public | private | restricted + isPaid=False, # If True then compensated dataset and requires basePriceCents, If False (default) = free dataset + # basePriceCents=100_000, # Required if isPaid=True. Price in USD cents between 10_000 = $100 and 5_000_000 = $50,000. exclusivityOptOut=True, # True = dataset is not exclusive to Data Collective (can be found elsewhere), # False = dataset is exclusively shared in Mozilla Data Collective agreeToSubmit=True, # True = You confirm that you have the right to submit this dataset and that all information provided in the datasheet is accurate. Required to be true to complete the submission From 879b276f96026f33bfd2d5058ff5455d6b120d2e Mon Sep 17 00:00:00 2001 From: Kostis-S-Z Date: Tue, 4 Aug 2026 16:05:30 +0300 Subject: [PATCH 3/5] Add unit tests --- tests/test_models.py | 58 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/tests/test_models.py b/tests/test_models.py index e6986aa..5f5182f 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -2,6 +2,8 @@ from pydantic import ValidationError from datacollective.models import ( + MAX_DATASET_PRICE_CENTS, + MIN_DATASET_PRICE_CENTS, DatasetDetails, DatasetSubmission, License, @@ -154,6 +156,62 @@ def test_show_contact_info_accepts_boolean() -> None: assert DatasetSubmission(showContactInfo=False).showContactInfo is False +def test_pricing_defaults_to_unset_free_dataset() -> None: + model = DatasetSubmission(name="Dataset Name") + assert model.isPaid is None + assert model.basePriceCents is None + # Left out of the payload entirely, so the platform default (False) applies + assert "isPaid" not in model.model_dump(exclude_none=True) + + +def test_paid_dataset_accepts_price_within_platform_bounds() -> None: + model = DatasetSubmission(isPaid=True, basePriceCents=25_000) + assert model.isPaid is True + assert model.basePriceCents == 25_000 + + +def test_paid_dataset_accepts_price_bounds() -> None: + assert ( + DatasetSubmission( + isPaid=True, basePriceCents=MIN_DATASET_PRICE_CENTS + ).basePriceCents + == MIN_DATASET_PRICE_CENTS + ) + assert ( + DatasetSubmission( + isPaid=True, basePriceCents=MAX_DATASET_PRICE_CENTS + ).basePriceCents + == MAX_DATASET_PRICE_CENTS + ) + + +def test_paid_dataset_rejects_price_outside_platform_bounds() -> None: + with pytest.raises(ValidationError): + DatasetSubmission(isPaid=True, basePriceCents=MIN_DATASET_PRICE_CENTS - 1) + with pytest.raises(ValidationError): + DatasetSubmission(isPaid=True, basePriceCents=MAX_DATASET_PRICE_CENTS + 1) + with pytest.raises(ValidationError): + DatasetSubmission(isPaid=True, basePriceCents=0) + with pytest.raises(ValidationError): + DatasetSubmission(isPaid=True, basePriceCents=-25_000) + + +def test_paid_dataset_requires_price() -> None: + with pytest.raises(ValidationError) as exc_info: + DatasetSubmission(isPaid=True) + assert "`basePriceCents` is required when `isPaid` is True" in str(exc_info.value) + + +def test_price_requires_paid_dataset() -> None: + with pytest.raises(ValidationError) as exc_info: + DatasetSubmission(basePriceCents=25_000) + assert "`isPaid` must be True when providing `basePriceCents`" in str( + exc_info.value + ) + with pytest.raises(ValidationError): + DatasetSubmission(isPaid=False, basePriceCents=25_000) + + def test_dataset_details_requires_id() -> None: with pytest.raises(ValidationError): DatasetDetails.model_validate({"filename": "dataset.tar.gz"}) From a5d64642079bbfb275fd9101af580e16757e6eb2 Mon Sep 17 00:00:00 2001 From: Kostis-S-Z Date: Tue, 4 Aug 2026 16:05:38 +0300 Subject: [PATCH 4/5] Update docs --- README.md | 4 ++++ docs/index.md | 4 ++++ docs/upload.md | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) diff --git a/README.md b/README.md index b9c7b75..1d419fa 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,8 @@ submission = DatasetSubmission( ethicalReviewProcess="Describe the ethical review process that was " "followed for this dataset, including any approvals " "or considerations related to data collection and usage.", + isPaid=False, # True = the dataset is compensated and requires `basePriceCents`, + # False (default) = the dataset is free to access exclusivityOptOut=False, # True = This dataset is non-exclusive to Mozilla Data Collective, # False = Dataset is exclusively hosted in Mozilla Data Collective agreeToSubmit=True, # True = You confirm that you have the right to submit this dataset and @@ -158,6 +160,8 @@ print(response) For predefined licenses, pass `licenseAbbreviation=License.` and leave `licenseUrl` and `license` unset. For custom licenses, pass a custom string to `license` and optionally include `licenseUrl` and `licenseAbbreviation`. +To publish a compensated dataset, set `isPaid=True` and a `basePriceCents` price in **USD cents** (US Dollars), e.g. `basePriceCents=100_000` for $1,000.00. The platform only accepts prices between `10_000` ($100) and `5_000_000` ($50,000), and the SDK validates the range locally before any request is sent. + > [!TIP] > To upload a new `.tar.gz` version to an already approved dataset, call `upload_dataset_file(file_path=..., submission_id=...)` directly. Find the submission under **Profile → Uploads**, open the approved dataset, and copy the value after `/profile/submissions/` in the URL. Note that this value is the submission ID, which is different from the public dataset ID. diff --git a/docs/index.md b/docs/index.md index 19eabb3..98dc64f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -157,6 +157,8 @@ submission = DatasetSubmission( "or considerations related to data collection and usage.", showContactInfo=False, # Whether to publicly display the contact information above visibility=Visibility.PUBLIC, # public | private | restricted + isPaid=False, # True = the dataset is compensated and requires `basePriceCents`, + # False (default) = the dataset is free to access exclusivityOptOut=False, # True = This dataset is non-exclusive to Mozilla Data Collective, # False = Dataset is exclusively hosted in Mozilla Data Collective agreeToSubmit=True, # True = You confirm that you have the right to submit this dataset and @@ -174,6 +176,8 @@ print(response) For predefined licenses, pass `licenseAbbreviation=License.` and leave `licenseUrl` and `license` unset. For a custom license, pass a custom string to `license` and optionally include `licenseUrl` and `licenseAbbreviation`. +To publish a compensated dataset instead of a free one, set `isPaid=True` and a `basePriceCents` price in **USD cents** (between `10_000` = $100 and `5_000_000` = $50,000). See [Pricing](upload.md#pricing) for details. + > [!TIP] > To upload a new `.tar.gz` version to an already approved and published dataset, call `upload_dataset_file(file_path=..., submission_id=...)` directly. Get the submission ID from **Profile → Uploads** by opening the approved dataset and copying the value after `/profile/submissions/` in the URL. This submission ID is different from the dataset ID. diff --git a/docs/upload.md b/docs/upload.md index b96e25f..12ffa12 100644 --- a/docs/upload.md +++ b/docs/upload.md @@ -81,6 +81,8 @@ submission = DatasetSubmission( "or considerations related to data collection and usage.", showContactInfo=False, # Whether to publicly display the contact information above visibility=Visibility.PUBLIC, # public | private | restricted + isPaid=False, # True = the dataset is compensated and requires `basePriceCents`, + # False (default) = the dataset is free to access exclusivityOptOut=False, # True = This dataset is non-exclusive to Mozilla Data Collective, # False = Dataset is exclusively hosted in Mozilla Data Collective agreeToSubmit=True, # True = You confirm that you have the right to submit this dataset and @@ -108,6 +110,55 @@ For predefined licenses, pass `licenseAbbreviation=License.` and leave `l | `Visibility.PRIVATE` | Everyone | Your organization & Approved requesters only | | `Visibility.RESTRICTED` | Your organization | Your organization (via SDK) | +### Pricing + +Datasets are free by default (`isPaid` defaults to `False` on the platform when left unset). To +publish a **compensated** dataset, set `isPaid=True` and provide a price in `basePriceCents`: + +```python +from datacollective import DatasetSubmission + +submission = DatasetSubmission( + name="Dataset Name", + # ... other metadata fields ... + isPaid=True, + basePriceCents=100_000, # $1,000.00 +) +``` + +> [!IMPORTANT] +> `basePriceCents` is expressed in **USD cents** (US Dollars), not in dollars. +> For example, `basePriceCents=100_000` sets the price to **$1,000.00 USD**. + +The platform only accepts prices within the following range: + +| Bound | Cents | USD | +|---------|-------------|------------| +| Minimum | `10_000` | $100 | +| Maximum | `5_000_000` | $50,000 | + +Both bounds are available as `MIN_DATASET_PRICE_CENTS` and `MAX_DATASET_PRICE_CENTS` in +`datacollective.models`. + +The two fields are validated together as soon as the `DatasetSubmission` model is constructed, +so invalid pricing raises a `ValidationError` locally, before any request reaches the API: + +- `isPaid=True` requires `basePriceCents` to be set. +- `basePriceCents` outside the range above is rejected. +- `basePriceCents` cannot be set unless `isPaid=True`, since the price would otherwise be + ignored and the dataset would stay uncompensated. + +To change the price of an existing submission, pass both fields to `update_submission`: + +```python +from datacollective import DatasetSubmission, update_submission + +update_submission( + submission_id=submission_id, + submission=DatasetSubmission(isPaid=True, basePriceCents=250_000), # $2,500.00 +) +``` + ## Upload a New File Version to an Approved Dataset Use `upload_dataset_file` when the dataset already exists on the platform and is already in the **Published / Approved** state. @@ -156,6 +207,8 @@ To complete the submission process, the submission **must** include at least all - `visibility` - `agreeToSubmit=True` +Pricing (`isPaid` and `basePriceCents`) is optional — see [Pricing](#pricing). Datasets are free unless `isPaid=True`. + A completed file upload must also be attached to the submission before it can be submitted for review. The uploaded archive is linked to the submission automatically when the multipart upload completes (the upload is started with the submission's ID). ## Step-by-Step Upload From 3a13fd9ed388dd8820f2e021e9883db7833bc21b Mon Sep 17 00:00:00 2001 From: Kostis-S-Z Date: Tue, 4 Aug 2026 16:47:44 +0300 Subject: [PATCH 5/5] Remove value validation of price --- README.md | 3 +-- docs/demo_upload.py | 2 +- docs/index.md | 2 +- docs/upload.md | 16 +++------------- src/datacollective/models.py | 26 +++++++------------------- tests/test_models.py | 28 ---------------------------- 6 files changed, 13 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 1d419fa..5b3791c 100644 --- a/README.md +++ b/README.md @@ -160,8 +160,7 @@ print(response) For predefined licenses, pass `licenseAbbreviation=License.` and leave `licenseUrl` and `license` unset. For custom licenses, pass a custom string to `license` and optionally include `licenseUrl` and `licenseAbbreviation`. -To publish a compensated dataset, set `isPaid=True` and a `basePriceCents` price in **USD cents** (US Dollars), e.g. `basePriceCents=100_000` for $1,000.00. The platform only accepts prices between `10_000` ($100) and `5_000_000` ($50,000), and the SDK validates the range locally before any request is sent. - +To publish a compensated dataset, set `isPaid=True` and a `basePriceCents` price in **USD cents** (US Dollars), e.g. `basePriceCents=100_000` for $1,000.00. > [!TIP] > To upload a new `.tar.gz` version to an already approved dataset, call `upload_dataset_file(file_path=..., submission_id=...)` directly. Find the submission under **Profile → Uploads**, open the approved dataset, and copy the value after `/profile/submissions/` in the URL. Note that this value is the submission ID, which is different from the public dataset ID. diff --git a/docs/demo_upload.py b/docs/demo_upload.py index a81e1e5..228155c 100644 --- a/docs/demo_upload.py +++ b/docs/demo_upload.py @@ -37,7 +37,7 @@ showContactInfo=False, # Whether to publicly display the contact information above visibility=Visibility.PUBLIC, # public | private | restricted isPaid=False, # If True then compensated dataset and requires basePriceCents, If False (default) = free dataset - # basePriceCents=100_000, # Required if isPaid=True. Price in USD cents between 10_000 = $100 and 5_000_000 = $50,000. + # basePriceCents=100_000, # Required if isPaid=True. Price in USD cents (in this example: $1,000.00). exclusivityOptOut=True, # True = dataset is not exclusive to Data Collective (can be found elsewhere), # False = dataset is exclusively shared in Mozilla Data Collective agreeToSubmit=True, # True = You confirm that you have the right to submit this dataset and that all information provided in the datasheet is accurate. Required to be true to complete the submission diff --git a/docs/index.md b/docs/index.md index 98dc64f..2c6402f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -176,7 +176,7 @@ print(response) For predefined licenses, pass `licenseAbbreviation=License.` and leave `licenseUrl` and `license` unset. For a custom license, pass a custom string to `license` and optionally include `licenseUrl` and `licenseAbbreviation`. -To publish a compensated dataset instead of a free one, set `isPaid=True` and a `basePriceCents` price in **USD cents** (between `10_000` = $100 and `5_000_000` = $50,000). See [Pricing](upload.md#pricing) for details. +To publish a compensated dataset instead of a free one, set `isPaid=True` and a `basePriceCents` price in **USD cents** (US Dollars). See [Pricing](upload.md#pricing) for details. > [!TIP] > To upload a new `.tar.gz` version to an already approved and published dataset, call `upload_dataset_file(file_path=..., submission_id=...)` directly. Get the submission ID from **Profile → Uploads** by opening the approved dataset and copying the value after `/profile/submissions/` in the URL. This submission ID is different from the dataset ID. diff --git a/docs/upload.md b/docs/upload.md index 12ffa12..0264cb0 100644 --- a/docs/upload.md +++ b/docs/upload.md @@ -130,21 +130,11 @@ submission = DatasetSubmission( > `basePriceCents` is expressed in **USD cents** (US Dollars), not in dollars. > For example, `basePriceCents=100_000` sets the price to **$1,000.00 USD**. -The platform only accepts prices within the following range: - -| Bound | Cents | USD | -|---------|-------------|------------| -| Minimum | `10_000` | $100 | -| Maximum | `5_000_000` | $50,000 | - -Both bounds are available as `MIN_DATASET_PRICE_CENTS` and `MAX_DATASET_PRICE_CENTS` in -`datacollective.models`. - -The two fields are validated together as soon as the `DatasetSubmission` model is constructed, -so invalid pricing raises a `ValidationError` locally, before any request reaches the API: +> [!NOTE] +> The platform validates that the price falls within an acceptable range and rejects the +> submission otherwise. - `isPaid=True` requires `basePriceCents` to be set. -- `basePriceCents` outside the range above is rejected. - `basePriceCents` cannot be set unless `isPaid=True`, since the price would otherwise be ignored and the dataset would stay uncompensated. diff --git a/src/datacollective/models.py b/src/datacollective/models.py index 7b5d867..056e25e 100644 --- a/src/datacollective/models.py +++ b/src/datacollective/models.py @@ -6,11 +6,6 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -# Price bounds enforced by the platform for compensated datasets in USD cents -MIN_DATASET_PRICE_CENTS = 10_000 # 100 USD -MAX_DATASET_PRICE_CENTS = 5_000_000 # 50,000 USD - - class UploadPart(BaseModel): """A single multipart upload part.""" @@ -175,7 +170,11 @@ class Dataset(BaseModel): ) basePriceCents: int | None = Field( None, - description="Price of the dataset in USD cents (e.g. `100_000` is $1,000.00). Required when `isPaid` is True.", + description=( + "Price of the dataset in USD cents (e.g. `100_000` is $1,000.00). Required when " + "`isPaid` is True. The platform validates that the price is within an acceptable " + "range and rejects the submission otherwise." + ), ) # Defined by the API and not user-editable id: str | None = Field( @@ -221,17 +220,6 @@ class DatasetSubmission(NonEmptyStrModel, Dataset): None, description="Dataset visibility: `public`, `private`, or `restricted`.", ) - basePriceCents: int | None = Field( - None, - ge=MIN_DATASET_PRICE_CENTS, - le=MAX_DATASET_PRICE_CENTS, - description=( - "Price of the dataset in USD cents (e.g. `100_000` is $1,000.00). Required when " - f"`isPaid` is True and must be between {MIN_DATASET_PRICE_CENTS} " - f"({MIN_DATASET_PRICE_CENTS // 100} USD) and {MAX_DATASET_PRICE_CENTS} " - f"({MAX_DATASET_PRICE_CENTS // 100} USD) cents." - ), - ) # Submission-specific fields defined by the user createdByFullName: str | None = Field(None, description="Creator's name.") createdByEmail: str | None = Field(None, description="Creator's email.") @@ -282,8 +270,8 @@ def _validate_license_details(self) -> DatasetSubmission: def _validate_pricing(self) -> DatasetSubmission: if self.isPaid and self.basePriceCents is None: raise ValueError( - "`basePriceCents` is required when `isPaid` is True and must be between " - f"{MIN_DATASET_PRICE_CENTS} and {MAX_DATASET_PRICE_CENTS} USD cents" + "`basePriceCents` is required when `isPaid` is True. The platform only " + "accepts prices within its allowed range, in USD cents" ) if self.basePriceCents is not None and not self.isPaid: raise ValueError( diff --git a/tests/test_models.py b/tests/test_models.py index 5f5182f..56f7d90 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -2,8 +2,6 @@ from pydantic import ValidationError from datacollective.models import ( - MAX_DATASET_PRICE_CENTS, - MIN_DATASET_PRICE_CENTS, DatasetDetails, DatasetSubmission, License, @@ -170,32 +168,6 @@ def test_paid_dataset_accepts_price_within_platform_bounds() -> None: assert model.basePriceCents == 25_000 -def test_paid_dataset_accepts_price_bounds() -> None: - assert ( - DatasetSubmission( - isPaid=True, basePriceCents=MIN_DATASET_PRICE_CENTS - ).basePriceCents - == MIN_DATASET_PRICE_CENTS - ) - assert ( - DatasetSubmission( - isPaid=True, basePriceCents=MAX_DATASET_PRICE_CENTS - ).basePriceCents - == MAX_DATASET_PRICE_CENTS - ) - - -def test_paid_dataset_rejects_price_outside_platform_bounds() -> None: - with pytest.raises(ValidationError): - DatasetSubmission(isPaid=True, basePriceCents=MIN_DATASET_PRICE_CENTS - 1) - with pytest.raises(ValidationError): - DatasetSubmission(isPaid=True, basePriceCents=MAX_DATASET_PRICE_CENTS + 1) - with pytest.raises(ValidationError): - DatasetSubmission(isPaid=True, basePriceCents=0) - with pytest.raises(ValidationError): - DatasetSubmission(isPaid=True, basePriceCents=-25_000) - - def test_paid_dataset_requires_price() -> None: with pytest.raises(ValidationError) as exc_info: DatasetSubmission(isPaid=True)