diff --git a/docs/MIGRATION_LOG.md b/docs/MIGRATION_LOG.md index 50b590d..6585585 100644 --- a/docs/MIGRATION_LOG.md +++ b/docs/MIGRATION_LOG.md @@ -113,8 +113,8 @@ Source: code exploration 2026-06-24 on `deploy-test` @ `93f6f62`. Version **0.9. - [x] `.yarnrc.yml`: tracked + age gate + preapproved scopes (`packageManager` already pinned `yarn@4.10.3`) - [x] Container verified locally: amd64 `docker build` OK; `app.handler` imports inside the image; `uvicorn` TestClient `{ about }` byte-matches legacy. Removed `branches:` filter (Trap #14) so the stack gets CI. -### Phase 2 โ€” Data layer + schema migration -- [ ] `BinaryLargeObjectModel` (PynamoDB) โ†’ pydantic + boto3 CRUD; **preserve `BinaryLargeObject` wrapper API**; keep `migrate()`/`drop_tables()` +### Phase 2 โ€” Data layer + schema migration ๐ŸŸก +- [x] `BinaryLargeObjectModel` (PynamoDB) โ†’ `pydantic` + `boto3` CRUD behind the **unchanged** `BinaryLargeObject` wrapper; `migrate()`/`drop_tables()` kept; **`pynamodb` dependency removed** - [ ] Port ~25 graphene types across the 8 modules to Strawberry at SDL + runtime parity (relay node parity, nullability, `UNSET` args) - [ ] Keep `cli` / `cli_ab_test` imports working - [ ] SDL parity gate green @@ -165,4 +165,12 @@ Source: code exploration 2026-06-24 on `deploy-test` @ `93f6f62`. Version **0.9. - **CI:** removed the `dev.yml` `branches:` filter so stacked PRs get CI (Trap #14); added `.yarnrc.yml` age gate (ยง4.6). - **Next:** Phase 2 โ€” the headline work: `BinaryLargeObjectModel` (PynamoDB) โ†’ pydantic + boto3 behind the `BinaryLargeObject` wrapper, then port the ~25 graphene types to Strawberry at SDL parity. +### 2026-06-24 โ€” Phase 2a: PynamoDB โ†’ pydantic + boto3 (data layer) +- Converted the **only** PynamoDB usage โ€” `BinaryLargeObjectModel(Model)` โ†’ a `pydantic.BaseModel` (`BinaryLargeObjectItem`) + thin `boto3` DynamoDB CRUD (`describe`/`create`/`delete`/`put`/`get_item`). The public **`BinaryLargeObject` wrapper API is unchanged** (`get`/`save`/`exists`/`create_table`/`delete_table`/`to_json`/`set_s3_client_args`/`object_*` props), so `cli` and `composite_solution.cached` are untouched. `tables`/`migrate()`/`drop_tables()` preserved (`create_table(wait=True)` โ†’ boto3 `table_exists` waiter). +- **Faithful to `JSONAttribute`:** `object_meta` is stored as a JSON string (not a native DynamoDB Map), so values round-trip as ints โ€” avoids the `Decimal` skew a native Map would introduce, and keeps `to_json()` byte-equal. +- **Did not "improve" the wrapper** โ€” kept the TODO-flagged `get()` shape as-is (no contract change mid-migration). +- **`pynamodb` dependency dropped** from `pyproject.toml`; `uv lock`/`sync` (164 pkgs); no lingering imports. +- Verified: the moto contract `data_store/test/test_model.py` **4/4**; full suite **77 passed / 10 skipped**; ruff + mypy clean. Container entry/Dockerfile untouched, so the Phase 1 build proof still holds. +- **Next:** Phase 2b โ€” port the ~25 graphene types (8 modules) to Strawberry at SDL parity; re-point the parity gate + corpus replay at the new schema. + diff --git a/pyproject.toml b/pyproject.toml index 0cff075..b0c064c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,6 @@ dependencies = [ "matplotlib>=3.10.7,<4.0.0", "nzshm-common>=0.9.0,<1.0", "nzshm-model>=0.14.0,<1.0", - "pynamodb>=6.0.0", "pyyaml>=6.0.1", "solvis>=1.2.0,<2.0", ] diff --git a/solvis_graphql_api/data_store/model.py b/solvis_graphql_api/data_store/model.py index b62106d..97c0cfb 100644 --- a/solvis_graphql_api/data_store/model.py +++ b/solvis_graphql_api/data_store/model.py @@ -1,17 +1,31 @@ +"""DynamoDB + S3 blob store for the CompositeSolution archive. + +Migration note: the DynamoDB item was a PynamoDB ``Model``; it is now a ``pydantic`` +model + thin ``boto3`` CRUD. The public ``BinaryLargeObject`` wrapper API is unchanged +(``get`` / ``save`` / ``exists`` / ``create_table`` / ``delete_table`` / ``to_json`` / +``set_s3_client_args`` / the ``object_*`` properties), so ``cli`` and the resolvers +(``composite_solution.cached``) are untouched. ``object_meta`` is stored as a JSON string +to reproduce PynamoDB's ``JSONAttribute`` exactly (values round-trip as ints, not Decimals). +""" + import io +import json import logging from collections.abc import Sequence from typing import Any import boto3 import botocore -from pynamodb.attributes import JSONAttribute, UnicodeAttribute -from pynamodb.models import Model # Condition +from pydantic import BaseModel -from .config import DEPLOYMENT_STAGE, IS_OFFLINE, REGION, S3_BUCKET_NAME, TESTING +from .config import DB_ENDPOINT, DEPLOYMENT_STAGE, IS_OFFLINE, REGION, S3_BUCKET_NAME, TESTING log = logging.getLogger(__name__) +TABLE_NAME = f"SGI-BinaryLargeObject-{DEPLOYMENT_STAGE}" + +log.info(f"configuring BinaryLargeObject store with IS_OFFLINE: {IS_OFFLINE} TESTING: {TESTING}") + S3_CLIENT_ARGS = ( dict( aws_access_key_id="S3RVER", @@ -23,32 +37,39 @@ ) -class BinaryLargeObjectModel(Model): - class Meta: - billing_mode = "PAY_PER_REQUEST" - table_name = f"SGI-BinaryLargeObject-{DEPLOYMENT_STAGE}" - region = REGION - log.info(f"congiguring BinaryLargeObjectModel with IS_OFFLINE: {IS_OFFLINE} TESTING: {TESTING}") - if IS_OFFLINE and not TESTING: - host = "http://localhost:8000" - log.info(f"set dynamodb host: {host}") +def _dynamodb_resource() -> Any: + kwargs: dict[str, Any] = dict(region_name=REGION) + if DB_ENDPOINT: # set when running offline (serverless-dynamodb local) + kwargs["endpoint_url"] = DB_ENDPOINT + return boto3.resource("dynamodb", **kwargs) + + +class BinaryLargeObjectItem(BaseModel): + """The DynamoDB item shape (was the PynamoDB model attributes).""" + + hash_key: str + range_key: str + object_id: str + object_type: str + object_meta: dict - hash_key = UnicodeAttribute(hash_key=True) - range_key = UnicodeAttribute(range_key=True) - object_id = UnicodeAttribute() - object_type = UnicodeAttribute() - object_meta = JSONAttribute() + def to_simple_dict(self) -> dict[str, Any]: + # name/shape preserved from the PynamoDB Model.to_simple_dict() it replaces + return self.model_dump() class BinaryLargeObject: """ - A class wrapping BinaryLargeObjectModel so that we can intercept the save/get operations + A class wrapping the DynamoDB item so that we can intercept the save/get operations + and carry the S3 blob alongside the item. - TODO: maybe we can use Model directly but we have issues with the get() classmethod + TODO: maybe we can use the item model directly but we have issues with the get() classmethod """ - def __init__(self, object_id, object_type, object_meta, object_blob, client_args=None): - self._model_instance = BinaryLargeObjectModel( + def __init__( + self, object_id, object_type, object_meta, object_blob, client_args=None + ): + self._item = BinaryLargeObjectItem( hash_key=f"{object_type}:{object_id}", range_key=f"{object_type}:{object_id}", object_id=object_id, @@ -72,7 +93,9 @@ def set_s3_client_args(self, client_args: dict) -> "BinaryLargeObject": @property def s3_client(self): if not self._s3_client: - self._s3_client = boto3.client("s3", **self._aws_client_args, region_name=REGION) + self._s3_client = boto3.client( + "s3", **self._aws_client_args, region_name=REGION + ) return self._s3_client @property @@ -85,20 +108,22 @@ def s3_connection(self): @property def s3_bucket(self): if not self._s3_bucket: - self._s3_bucket = self.s3_connection.Bucket(self._bucket_name, client=self.s3_client) + self._s3_bucket = self.s3_connection.Bucket( + self._bucket_name, client=self.s3_client + ) return self._s3_bucket @property def object_id(self): - return self._model_instance.object_id + return self._item.object_id @property def object_type(self): - return self._model_instance.object_type + return self._item.object_type @property def object_meta(self): - return self._model_instance.object_meta + return self._item.object_meta @property def object_blob(self): @@ -108,7 +133,9 @@ def object_blob(self): log.info(f"get object_blob from bucket {self}") try: file_object = io.BytesIO() - self.s3_bucket.download_fileobj(f"{self.object_type}/{self.object_id}", file_object) + self.s3_bucket.download_fileobj( + f"{self.object_type}/{self.object_id}", file_object + ) file_object.seek(0) self._object_blob = file_object.read() except botocore.exceptions.ClientError as err: @@ -119,22 +146,43 @@ def object_blob(self): return self._object_blob def to_json(self): - mijson = self._model_instance.to_simple_dict() + mijson = self._item.to_simple_dict() print(mijson) mijson["object_blob"] = self._object_blob return mijson @classmethod def exists(cls) -> bool: - return BinaryLargeObjectModel.exists() + client = _dynamodb_resource().meta.client + try: + client.describe_table(TableName=TABLE_NAME) + return True + except client.exceptions.ResourceNotFoundException: + return False @classmethod - def create_table(cls) -> dict[str, Any]: - return BinaryLargeObjectModel.create_table() + def create_table(cls, wait: bool = False) -> dict[str, Any]: + client = _dynamodb_resource().meta.client + resp = client.create_table( + TableName=TABLE_NAME, + KeySchema=[ + {"AttributeName": "hash_key", "KeyType": "HASH"}, + {"AttributeName": "range_key", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "hash_key", "AttributeType": "S"}, + {"AttributeName": "range_key", "AttributeType": "S"}, + ], + BillingMode="PAY_PER_REQUEST", + ) + if wait: + client.get_waiter("table_exists").wait(TableName=TABLE_NAME) + return resp @classmethod def delete_table(cls) -> dict[str, Any]: - return BinaryLargeObjectModel.delete_table() + client = _dynamodb_resource().meta.client + return client.delete_table(TableName=TABLE_NAME) def save(self) -> dict[str, Any]: if self._object_blob: @@ -143,7 +191,10 @@ def save(self) -> dict[str, Any]: Key=f"{self.object_type}/{self.object_id}", Body=io.BytesIO(self._object_blob), ) - return self._model_instance.save() + table = _dynamodb_resource().Table(TABLE_NAME) + item = self._item.model_dump() + item["object_meta"] = json.dumps(item["object_meta"]) # JSONAttribute parity + return table.put_item(Item=item) @classmethod def get( @@ -156,21 +207,29 @@ def get( ) -> Any: log.info(f"{cls}.get() called") hash_key = f"{object_type}:{object_id}" - model_instance = BinaryLargeObjectModel.get(hash_key, hash_key, consistent_read, attributes_to_get) - instance = cls( - model_instance.object_id, - model_instance.object_type, - model_instance.object_meta, + table = _dynamodb_resource().Table(TABLE_NAME) + response = table.get_item( + Key={"hash_key": hash_key, "range_key": hash_key}, + ConsistentRead=consistent_read, + ) + if "Item" not in response: + raise KeyError(f"BinaryLargeObject {hash_key} does not exist") + record = response["Item"] + return cls( + record["object_id"], + record["object_type"], + json.loads(record["object_meta"]), None, ) - return instance -tables = [BinaryLargeObjectModel] +tables = [BinaryLargeObject] def migrate(): - log.info(f"migrate() stage: {DEPLOYMENT_STAGE} offline: {IS_OFFLINE} region: {REGION} testing: {TESTING}") + log.info( + f"migrate() stage: {DEPLOYMENT_STAGE} offline: {IS_OFFLINE} region: {REGION} testing: {TESTING}" + ) for table in tables: if not table.exists(): table.create_table(wait=True) diff --git a/uv.lock b/uv.lock index 1ab8a31..3d776bc 100644 --- a/uv.lock +++ b/uv.lock @@ -2357,18 +2357,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] -[[package]] -name = "pynamodb" -version = "6.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "botocore" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/af/0b/35d24cdfe532fd9c774c184fbd09118b0a8c94eb51ca9598c0c5ef21174d/pynamodb-6.1.0.tar.gz", hash = "sha256:c7d09700ace9f428e67ecf73fa867633c632de3acf402b2c7c136be695afda79", size = 96283, upload-time = "2025-06-02T17:32:59.367Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d4/86/71e355d232f28adb50c89b806b018548c095a6cf3d631b645918f415212a/pynamodb-6.1.0-py3-none-any.whl", hash = "sha256:9c0f1a0f177208640b2336ed56c557c5187b0012d356e9c7399c3923c5f93c7f", size = 61685, upload-time = "2025-06-02T17:32:57.488Z" }, -] - [[package]] name = "pyogrio" version = "0.12.1" @@ -3003,7 +2991,6 @@ dependencies = [ { name = "nzshm-common" }, { name = "nzshm-model" }, { name = "pydantic" }, - { name = "pynamodb" }, { name = "pyyaml" }, { name = "serverless-wsgi" }, { name = "solvis" }, @@ -3048,7 +3035,6 @@ requires-dist = [ { name = "nzshm-common", specifier = ">=0.9.0,<1.0" }, { name = "nzshm-model", specifier = ">=0.14.0,<1.0" }, { name = "pydantic", specifier = ">=2.0" }, - { name = "pynamodb", specifier = ">=6.0.0" }, { name = "pyyaml", specifier = ">=6.0.1" }, { name = "serverless-wsgi", specifier = ">=3.0" }, { name = "solvis", specifier = ">=1.2.0,<2.0" },