From 95462ba4e56ac09b42109cfdb636516df2b1a966 Mon Sep 17 00:00:00 2001 From: Maksim Semenov Date: Fri, 23 Jan 2026 12:29:42 +0000 Subject: [PATCH] Update validation test to follow documentation. Expect soft-fail when supported following https://ucp.dev/specification/checkout/#error-handling Use correct error message format for hard fails https://ucp.dev/specification/checkout-rest/#error-responses --- .cspell/custom-words.txt | 78 ++++++++++++----------- idempotency_test.py | 12 ++-- integration_test_utils.py | 49 +++++++++++++++ protocol_test.py | 7 ++- pyproject.toml | 2 + validation_test.py | 127 ++++++++++++++++++++++---------------- 6 files changed, 176 insertions(+), 99 deletions(-) diff --git a/.cspell/custom-words.txt b/.cspell/custom-words.txt index 16af9a4..af99016 100644 --- a/.cspell/custom-words.txt +++ b/.cspell/custom-words.txt @@ -1,81 +1,87 @@ # cspell-specific custom words related to UCP +absl +absltest +adyen Adyen +agentic Alam Amex Ant Anytown +atok +backorder Backordered Braintree Carrefour Centricity +checkout Chewy Commerce -Credentialless -Depot -EWALLET -Etsy -Flipkart -Gap -GitHub -Google -Gpay -Kroger -Lowe's -Macy's -Mastercard -Paymentech -Paypal -Preorders -Queensway -Sephora -Shopify -Shopee -Stripe -Target -UCP -Ulta -Visa -Wayfair -Worldpay -Zalando -absl -absltest -adyen -agentic -atok -backorder -checkout -credentialless credentialization +credentialless +Credentialless≈ +cust datamodel +Depot dpan +Etsy ewallet +EWALLET +Flipkart fontawesome fpan fulfillable +Gap +GitHub +Google gpay +Gpay healthz ingestions inlinehilite +Kroger linenums llmstxt +Lowe's +Macy's +Malform mastercard +Mastercard mkdocs mtok openapi openrpc +Paymentech paypal +Paypal permissionless preorders +Preorders proto protobuf pymdownx +Queensway renderable repudiable schemas sdjwt +Sephora +Shopee shopify +Shopify +Smallville +Stripe superfences +Target +UCP +Ulta +Villagetown +Visa vulnz +Wayfair +Worldpay +wumpus +Wumpus yaml yml +Zalando diff --git a/idempotency_test.py b/idempotency_test.py index ec62332..ee6acea 100644 --- a/idempotency_test.py +++ b/idempotency_test.py @@ -110,13 +110,11 @@ def test_idempotency_update(self) -> None: # We construct the update payload same as helper does line_items_req = [] for li in checkout_obj.line_items: - line_items_req.append( - { - "item": {"id": li.item.id, "title": li.item.title}, - "quantity": 2, # Change quantity - "id": li.id, - } - ) + line_items_req.append({ + "item": {"id": li.item.id, "title": li.item.title}, + "quantity": 2, # Change quantity + "id": li.id, + }) payment_req = { "selected_instrument_id": checkout_obj.payment.selected_instrument_id, diff --git a/integration_test_utils.py b/integration_test_utils.py index 92a15ea..bde3f28 100644 --- a/integration_test_utils.py +++ b/integration_test_utils.py @@ -558,6 +558,55 @@ def assert_response_status( ), ) + def assert_checkout_status( + self, + checkout_data: dict[str, Any], + expected_status: str = "incomplete", + valid_path_matchers: list[str] | None = None, + model_class: Any = None, + ) -> Any: + """Assert checkout status and optionally verify error paths using Pydantic.""" + if model_class is None: + # Use a more permissive model that only requires status and messages for errors + from pydantic import BaseModel + from ucp_sdk.models.schemas.shopping.types import message + + class BasicErrorResponse(BaseModel): + status: str + messages: list[message.Message] | None = None + + model_class = BasicErrorResponse + + checkout_obj = model_class(**checkout_data) + + self.assertEqual( + checkout_obj.status, + expected_status, + msg=f"Expected checkout status '{expected_status}', got '{checkout_obj.status}'.", + ) + + if valid_path_matchers: + # Extract error messages from the Pydantic model + error_messages = [ + msg.root + for msg in (checkout_obj.messages or []) + if getattr(msg.root, "type", None) == "error" + ] + + error_paths = [getattr(m, "path", "") for m in error_messages] + + matches = any( + any(path == matcher for matcher in valid_path_matchers) + for path in error_paths + ) + + self.assertTrue( + matches, + f"Expected error path matching one of {valid_path_matchers}. Paths found: {error_paths}", + ) + + return checkout_obj + def create_checkout_session( self, quantity: int = 1, diff --git a/protocol_test.py b/protocol_test.py index 98c91b1..bef9fa1 100644 --- a/protocol_test.py +++ b/protocol_test.py @@ -56,9 +56,10 @@ def _extract_document_urls( if service.mcp and service.mcp.schema_: urls.add((f"{base_path}.mcp.schema", str(service.mcp.schema_))) if service.embedded and service.embedded.schema_: - urls.add( - (f"{base_path}.embedded.schema", str(service.embedded.schema_)) - ) + urls.add(( + f"{base_path}.embedded.schema", + str(service.embedded.schema_), + )) # 2. Capabilities for i, cap in enumerate(profile.ucp.capabilities): diff --git a/pyproject.toml b/pyproject.toml index b5193b8..a6fed18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,8 @@ docstring-code-format = true [tool.ruff.lint] select = ["E", "F", "W", "B", "C4", "SIM", "N", "UP", "D", "PTH", "T20"] ignore = ["D203", "D213"] +# Don't fail on lines ruff format can't fix +ignore = ["E501"] [tool.ruff.lint.isort] combine-as-imports = true diff --git a/validation_test.py b/validation_test.py index 566089f..becbab2 100644 --- a/validation_test.py +++ b/validation_test.py @@ -42,10 +42,12 @@ class ValidationTest(integration_test_utils.IntegrationTestBase): def test_out_of_stock(self) -> None: """Test validation for out-of-stock items. + Reference: https://ucp.dev/specification/checkout/#error-handling + Given a product with 0 inventory, When a checkout creation request is made for this item, - Then the server should return a 400 Bad Request error indicating - insufficient stock. + Then the server should return a successful response (201) with status + 'incomplete' and an error message indicating insufficient stock. """ # Get out of stock item from config out_of_stock_item = self.conformance_config.get( @@ -66,20 +68,30 @@ def test_out_of_stock(self) -> None: headers=integration_test_utils.get_headers(), ) - self.assert_response_status(response, 400) - self.assertIn( - "Insufficient stock", - response.text, - msg="Expected 'Insufficient stock' message", + # Verify HTTP status and then using centralized utility with + # exact path matching + self.assert_response_status(response, [200, 201]) + checkout_data = response.json() + li_id = checkout_data["line_items"][0]["id"] + self.assert_checkout_status( + checkout_data, + expected_status="incomplete", + valid_path_matchers=[ + "$.line_items[0]", + f"$.line_items[?(@.id=='{li_id}')]", + ], + model_class=checkout.Checkout, ) def test_update_inventory_validation(self) -> None: """Test that inventory validation is enforced on update. + Reference: https://ucp.dev/specification/checkout/#error-handling + Given an existing checkout session with a valid quantity, When the line item quantity is updated to exceed available stock, - Then the server should return a 400 Bad Request error indicating - insufficient stock. + Then the server should return a successful response (200/201) with + status 'incomplete' and an error message indicating insufficient stock. """ response_json = self.create_checkout_session() checkout_obj = checkout.Checkout(**response_json) @@ -119,9 +131,19 @@ def test_update_inventory_validation(self) -> None: headers=integration_test_utils.get_headers(), ) - self.assert_response_status(response, 400) - self.assertIn( - "stock", response.text.lower(), msg="Expected 'stock' message" + # Verify HTTP status and then using centralized utility with + # exact path matching + self.assert_response_status(response, [200, 201]) + checkout_data = response.json() + li_id = checkout_data["line_items"][0]["id"] + self.assert_checkout_status( + checkout_data, + expected_status="incomplete", + valid_path_matchers=[ + "$.line_items[0]", + f"$.line_items[?(@.id=='{li_id}')]", + ], + model_class=checkout.Checkout, ) def test_product_not_found(self) -> None: @@ -180,63 +202,62 @@ def test_payment_failure(self) -> None: self.assert_response_status(response, 402) - def test_complete_without_fulfillment(self) -> None: - """Test completion rejection when fulfillment is missing. + def test_update_without_fulfillment(self) -> None: + """Test Soft-Fail when fulfillment info is missing during update. + + Reference: https://ucp.dev/specification/checkout/#error-handling - Given a newly created checkout session without fulfillment details, - When a completion request is submitted, - Then the server should return a 400 Bad Request error. + Given an existing checkout session, + When an update request is sent with an empty fulfillment method, + Then the server should return a 200 OK status with 'incomplete' status + and an error message indicating missing fulfillment info. """ response_json = self.create_checkout_session(select_fulfillment=False) - checkout_id = response_json["id"] - - payment_payload = integration_test_utils.get_valid_payment_payload() + checkout_obj = checkout.Checkout(**response_json) - response = self.client.post( - self.get_shopping_url(f"/checkout-sessions/{checkout_id}/complete"), - json=payment_payload, - headers=integration_test_utils.get_headers(), + # Update with empty fulfillment using the helper to ensure valid structure + response_json = self.update_checkout_session( + checkout_obj, fulfillment={"methods": [{"type": "shipping"}]} ) - self.assert_response_status(response, 400) - self.assertIn( - "Fulfillment address and option must be selected", - response.text, - msg="Expected error message for missing fulfillment", + # Get the method ID from the response for precise path matching + method_id = response_json["fulfillment"]["methods"][0]["id"] + + self.assert_checkout_status( + response_json, + expected_status="incomplete", + valid_path_matchers=[ + "$.fulfillment", + "$.fulfillment.methods", + f"$.fulfillment.methods[?(@.id=='{method_id}')].destinations", + ], + model_class=checkout.Checkout, ) def test_structured_error_messages(self) -> None: - """Test that error responses conform to the Message schema. + """Test that error responses conform to the UCP ErrorResponse schema. - Given a request that triggers an error (e.g., out of stock), - When the server responds with an error code (400), - Then the response body should contain a structured 'detail' field describing - the error. - """ - # Get out of stock item from config - out_of_stock_item = self.conformance_config.get( - "out_of_stock_item", - {"id": "out_of_stock_item_1", "title": "Out of Stock Item"}, - ) + Reference: https://ucp.dev/specification/checkout-rest/#error-responses - create_payload = self.create_checkout_payload( - item_id=out_of_stock_item["id"], - ) + Given a request for a non-existent checkout ID, + When the server responds with a 404 Not Found error, + Then the response body should contain a structured UCP response with + status 'requires_escalation' and a descriptive error message. + """ + non_existent_id = "non-existent-session-id" - response = self.client.post( - self.get_shopping_url("/checkout-sessions"), - json=create_payload.model_dump( - mode="json", by_alias=True, exclude_none=True - ), + response = self.client.get( + self.get_shopping_url(f"/checkout-sessions/{non_existent_id}"), headers=integration_test_utils.get_headers(), ) - self.assert_response_status(response, 400) + # Verify HTTP status 404 + self.assert_response_status(response, 404) - # Check for structured error - data = response.json() - self.assertTrue(data.get("detail"), "Error response missing 'detail' field") - self.assertIn("Insufficient stock", data["detail"]) + # Verify using centralized utility with 'requires_escalation' status + self.assert_checkout_status( + response.json(), expected_status="requires_escalation" + ) if __name__ == "__main__":