From 25369b72fad3650545d47f2d8628bfbd8b7f7715 Mon Sep 17 00:00:00 2001 From: tywalch Date: Tue, 23 May 2023 11:55:00 -0400 Subject: [PATCH 1/4] Smarter limiting --- src/entity.js | 43 +++++++++++++++++++++++-- test/ts_connected.crud.spec.ts | 32 ++++++++++-------- www/src/pages/en/queries/pagination.mdx | 6 +++- 3 files changed, 65 insertions(+), 16 deletions(-) diff --git a/src/entity.js b/src/entity.js index 352ac505..11c7c7eb 100644 --- a/src/entity.js +++ b/src/entity.js @@ -474,15 +474,24 @@ class Entity { } let pages = this._normalizePagesValue(config.pages); let max = this._normalizeLimitValue(config.limit); + let providedLimit = parameters.Limit; + delete parameters.Limit; let iterations = 0; let count = 0; let hydratedUnprocessed = []; const shouldHydrate = config.hydrate && method === MethodTypes.query; do { let limit = max === undefined - ? parameters.Limit + ? providedLimit : max - count; - let response = await this._exec(method, { ExclusiveStartKey, ...parameters, Limit: limit }, config); + let response = await this._exec(method, { ExclusiveStartKey, ...parameters }, config); + response = this._maybeApplyArtificialLimit({ + response, + limit, + indexName: parameters.IndexName + }); + // console.log('RESPONSE', JSON.stringify({ response }, null, 4)); + ExclusiveStartKey = response.LastEvaluatedKey; response = this.formatResponse(response, parameters.IndexName, { ...config, @@ -655,6 +664,36 @@ class Entity { } } + _maybeApplyArtificialLimit({response, limit, indexName = TableIndex}) { + let Items = []; + for (let i = 0; i < limit; i++) { + const item = response.Items[i]; + Items.push(item); + } + let LastEvaluatedKey = response.LastEvaluatedKey; + if (Array.isArray(response.Items) && response.Items.length > limit) { + const itemAtLimit = Items[Items.length - 1]; + const indexFields = this.model.translations.keys[indexName]; + const tableIndexFields = this.model.translations.keys[TableIndex]; + LastEvaluatedKey = { + [indexFields.pk]: itemAtLimit[indexFields.pk], + [tableIndexFields.pk]: itemAtLimit[tableIndexFields.pk], + } + if (indexFields.sk && itemAtLimit[indexFields.sk]) { + LastEvaluatedKey[indexFields.sk] = itemAtLimit[indexFields.sk] + } + if (tableIndexFields.sk && itemAtLimit[tableIndexFields.sk]) { + LastEvaluatedKey[tableIndexFields.sk] = itemAtLimit[tableIndexFields.sk] + } + } + + return { + ...response, + Items, + LastEvaluatedKey, + }; + } + formatResponse(response, index, config = {}) { let stackTrace; if (!config.originalErr) { diff --git a/test/ts_connected.crud.spec.ts b/test/ts_connected.crud.spec.ts index c3e73a54..d066f2be 100644 --- a/test/ts_connected.crud.spec.ts +++ b/test/ts_connected.crud.spec.ts @@ -4656,21 +4656,24 @@ describe('query limit', () => { table, client }); - const accountId = uuid(); - const createItem = () => { - return { - accountId, - id: uuid(), - name: uuid(), - description: uuid(), - } - } + // const accountId = uuid(); + // const createItem = () => { + // return { + // accountId, + // id: uuid(), + // name: uuid(), + // description: uuid(), + // } + // } const limit = 10; const itemCount = 100; - - const items = new Array(itemCount).fill({}).map(createItem); + const items = [{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"9285ca74-c726-4843-b5ce-c9d8b95c5321","name":"350dd74e-0269-4212-a157-284dd8af3163","description":"9f501f00-176e-45a7-b585-b59ecd4dfb98"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"b71f67c9-af08-4c3a-b1a0-1f365fef3fdf","name":"65197748-af8b-4db6-ad3e-da4618ef08f1","description":"2c848057-3e17-42ae-be20-49b3dc702ea4"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"163f9c9a-2e6f-4250-bce0-249c3ff383ca","name":"21c22079-4267-427a-8b8a-70b41fdb53ee","description":"5b8b20bd-cf12-4082-9c14-c545b2661f79"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"942fea6c-e3d2-4cb0-bdeb-aad8bc9ac90c","name":"e46d67db-22f4-4ba1-9ffc-d939a535822d","description":"cfbc25fb-b231-471c-bb89-7c76b19fcc9c"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"b6781bbb-12b6-4528-9da0-f867c168e89b","name":"fdf65da2-8833-4afc-934c-3c282cfbb080","description":"890558b8-4ed9-4617-9078-1d2d05ad7836"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"ea1d4c70-bf39-4e7c-ba54-0d20d48c9b6d","name":"57938032-c0ad-4565-8b0a-53e95531b256","description":"60c2c2f5-2a3c-43c4-8bda-8b30a293f744"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"25c6cbd5-863d-45be-a237-793a0a48206d","name":"dfd49c67-5554-4605-873f-0d69f8f48ad6","description":"6dc691cf-640e-4c70-8573-ae1789a89c35"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"2a50954f-f2fd-4668-9f02-a074198f70df","name":"9accd38a-557b-4956-98cf-483de6cb1450","description":"0417755f-b1b5-442f-9a57-d15c888e0bf0"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"f20ec780-405c-4b9b-ac2f-7818f0d00712","name":"81d236fd-5ecd-4560-86cc-6743372d66cc","description":"a46c0f02-a547-4526-a053-f4b3e02c74ef"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"f92dda61-855c-4d00-a99e-eadb8b4b1df0","name":"a05113bc-7e32-4524-9dc7-d6d3d029610d","description":"7ec403b4-67eb-4a87-9a26-6169552e3d23"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"05705694-ea52-42d5-a4f6-818fbb727816","name":"ded13657-a9bc-4885-b12c-0c06318f74aa","description":"ced6b30e-c4f1-4f93-8661-51a2e0b1ec3d"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"7624396d-8dea-4a08-8357-02b39aa5eb32","name":"b5e0a922-e76d-4962-8079-bd98ddb8902b","description":"b3713f7e-101c-467d-85d8-1e79d1f56fe7"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"fd5ab144-48b9-431c-819c-814ad5221471","name":"227fdf6b-18f6-4db2-b652-61ee518a6c1f","description":"07c0ca8b-b407-426f-b88f-2fc8939bdfce"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"cc71765e-92cd-44b2-9020-281eca6e1133","name":"586c29d8-0b2b-42d9-9dbc-eb3f992c25fb","description":"a5a474b7-399f-4382-924f-2999584e5b06"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"2a49d1bb-0ade-4578-aed7-c338ed564df2","name":"35dba9e2-3d8b-48b3-8e3c-941747d9bbb6","description":"e1c2b309-62d4-41d9-9b72-c5beec77ed39"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"273f5bb8-892a-451f-9418-cec75bab7bdd","name":"e08c9382-9214-425e-a942-386697eaeabb","description":"088d091e-cd2f-4b0c-9d47-53fc032cc532"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"c58c20e8-506f-4c96-8aa4-f10eadfa8464","name":"9c1ae16f-07b9-4f4b-9546-79bdf64cce87","description":"9147a6f4-5a88-4f6b-bf16-50c4eda8e337"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"ab5e6d14-1704-4699-99ef-f5cdf8d15cae","name":"ef9c9fc9-1761-49cb-b60d-564198e24f3f","description":"c09dfd83-a025-4701-b856-e2e367da1ee3"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"77295160-a2af-480c-b672-7d19452f5f98","name":"ca938fe1-c0a0-46b5-a4c7-c2e68ad87a0b","description":"010939aa-7eb4-485c-ba9b-4ad428ae145b"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"887de7ae-396c-4203-82fa-a13b2a26f61d","name":"6e09a922-faf5-40b5-97a8-ffc7207f9422","description":"84f462b1-80d5-46c7-add7-76a5d77c3f61"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"1c350017-421e-451a-80af-8f5115dbcb8a","name":"56f114bb-e660-4d59-8174-3d4ac28cb97d","description":"d57d5beb-4dfa-4d53-864b-372bca570d38"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"7bd908ec-12bc-4c61-9e01-8c7db8992560","name":"62e5f333-0b8b-4e59-aae7-923414ecb48e","description":"3399cada-d48c-4247-aab1-924c0b6ed723"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"71978d28-022a-45ce-951b-58006356e9af","name":"dae48ac9-cc77-45bc-ab1a-8b68747d101e","description":"63a54f8e-a3cb-499c-8998-c03f18dab50e"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"d1123e7a-a6d9-4583-bbef-8d86741abb53","name":"34c19579-5a64-4014-9a1c-76f051911ae4","description":"695373f3-6a24-41f9-9989-c43f6ceec643"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"054cc007-0ad5-40cb-9c1f-f63c1e27db85","name":"0a81608d-4046-4b75-b256-d8c24e2c7b21","description":"9d4ad35f-ef22-48a4-b2b6-eed7e56d2bca"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"95428a2c-4d8a-4da7-b922-c244fca2e571","name":"1cf80c5b-4e53-4dcd-b760-0297d9e4740b","description":"aec50d8a-f9a7-4736-8cba-498ea1f92ed8"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"42ed2e4f-5152-47f2-9306-9cd8316c2fde","name":"2c134481-e48c-4aa7-987b-fee16b57c8b2","description":"d4520bce-aa2e-4114-a1a3-79afee68b4b5"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"124541e1-619c-49ed-bf30-ce064aa1c85a","name":"2c2445fd-599b-4a90-874b-18840df75ea3","description":"03dbf75f-ab15-4841-9338-e22ccd935eee"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"e31a967c-e139-46f9-abba-48a8f0b46a25","name":"07dd68c5-51f1-475a-a08f-ca738ab4a6ad","description":"6d217d00-d572-4bb7-a7f5-2c8d80076194"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"4486a6b5-a19a-4709-a273-07f1dc9c6f33","name":"056fd004-3fd7-4dd1-9950-bde4d349c8d3","description":"e8d3f2b5-f8ca-43dd-9028-3406eee233b8"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"8f49efe9-6033-4c02-bf73-b702bc91f7d2","name":"bc730646-40f3-4f41-936c-33669c895209","description":"1c929671-9d8c-46e2-8dcd-4205e7a53288"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"18f413e3-e133-492a-99aa-34566f81dc8c","name":"efab6cd6-8894-449d-8444-4da95d288a42","description":"7d96ac9b-882c-4321-b4fd-71ab1e87bcb4"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"6f05e97a-bed2-4a7e-a057-5b836497e0b3","name":"92740971-247a-4fbc-bdd1-3df43a712265","description":"d5022ce6-f80c-45f6-a8f5-6718cf71590d"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"49d1674a-7744-49d2-b26d-a0ddf73ab199","name":"abe66f33-2321-42ee-80af-707da3a57443","description":"2d70b61e-9492-4dd4-a240-8de73c0b9698"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"0cf5c473-bdcb-45df-a53d-3d127e6cada9","name":"6cb5922a-19db-4c4c-a176-2d959bbc7e50","description":"7d5d5924-d29a-460c-9ca9-9ea8915db305"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"db1838cc-f196-498d-8c6e-342f06c0e475","name":"84293b70-dc5a-48f1-bd89-fb0807a7818b","description":"308bf638-fdbd-4782-b164-072f7b23ade0"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"a7827224-7f62-4025-bf19-31b41a7d0711","name":"c12202e7-47ff-498e-890f-332c2e74b85d","description":"6333cf19-70f5-424d-b75b-25edb2304f29"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"e41e97ba-392d-46a3-9cab-5e11c7c28788","name":"833bbad0-f89c-4620-890a-42d7812243df","description":"116406f1-3f15-4380-87c1-9bde7fdd52d7"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"84404876-75dd-4412-9554-0aeb37f6866e","name":"e29eb07d-48ce-463e-bf1d-15d8fc2bd24a","description":"ef6e2a13-a09e-4757-aafe-3036a82b714c"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"c98f5877-5aa2-4261-be5c-ea3503fb4799","name":"6ec5c4e4-a257-437d-ba44-a3f10f49a795","description":"05afddcd-f3e1-4135-9992-d3db653e2ed0"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"aab35584-7716-47c1-b739-427a3a678450","name":"bd7a9197-6382-4001-9b71-bd7e9e36fc8e","description":"d65bdc00-671d-4dbf-8102-3df2568e8a31"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"00a613ac-6664-411c-b134-5a0fb8e34c3e","name":"6433a85f-c0e5-401b-b29f-6045ea4432b7","description":"3488ebe3-c6f7-4545-b1bf-965f5b38c34d"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"fc448a17-954f-42c0-8e6e-d0b1974886c3","name":"8114245f-7eaa-4ea7-a75e-1fd2b52eeeb3","description":"2717d2b9-29f6-490c-a63f-b50185d7eb03"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"68a98739-f79e-48a3-8613-1d0b2508c519","name":"9a395854-5b3f-4546-b304-f4666bacf80b","description":"f762a41d-6dec-4638-ab34-dd114b7ed9a1"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"816f12c8-d7c5-4a39-ba81-c6daac5ea380","name":"623f5823-4077-48ec-b245-84784b46abd3","description":"93b4b37a-c896-4333-bc6a-5ee4109a41cf"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"1047d2b4-b6c6-4413-94c1-adae1a367917","name":"de6bcfdb-a501-498d-8554-cf075ef3f791","description":"02358778-78d0-4155-a236-e836c28aa410"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"b1c7e3dc-ab49-4297-9b71-a07544e5223e","name":"535c6e38-b3e7-49de-bfda-5544a68e40f8","description":"a05861e8-fbea-4e0e-8930-7df9a9dbd6a6"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"3392d3f2-f93c-48be-9a88-4b988c5fd8f2","name":"98780ac7-69ba-4a57-80aa-e9f6b86a95da","description":"32414088-80a8-4582-aefe-ef1051b4cf93"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"d32061b3-b738-4f4c-9ac8-46f6764a1973","name":"e90e6ea9-fa2b-4f8d-8e4d-bb206c0eb79a","description":"75cb76da-bd0b-4783-b8b5-ecf70e7dc8c6"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"4e86be6b-2cb6-49b7-9129-1a4288fd303a","name":"f7c62002-6fdc-481c-87fa-5c4383b60cb8","description":"39da81a3-e844-4f38-8199-efead6787971"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"37e24ea2-e594-41c7-bd62-02eb156ec982","name":"0a3bc46b-3d0a-40bb-a80a-ed6caf3bdcf5","description":"09489f8a-f1d8-4529-a389-88277bea9427"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"5688424d-29bf-433a-8022-cf2c70d6dd37","name":"19cfab03-c223-426f-8c05-f2b1e8ff4e20","description":"40ce7461-848e-4352-a22f-12929d63ac38"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"36904b64-b0dc-4220-bae6-96178d2b02ac","name":"468e68ed-0677-4592-965c-7062d47dbddd","description":"bc70ce95-dca4-44c7-9a6d-e1720b0c830f"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"bee13085-a600-4084-9ba2-5c740b158d8b","name":"67c9ea59-4b8e-427f-8eaa-009a44bde7b9","description":"93c21563-8e70-4a77-a47a-7178a9234eb9"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"1f244c2e-af58-4d3b-a0ce-6ccb0d57ad63","name":"73908a75-ed06-4f17-8267-45a2ab5195a6","description":"a4e78079-cfb3-4e9c-b0f4-83893de0e4dc"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"6cbfe482-a848-4c19-9fae-a987557f0407","name":"bb79fa10-d98c-4ae0-bf38-32c0a5395080","description":"b4289b45-d58f-4472-925b-3b1ecccff932"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"78f6b720-2ad1-4ae3-b905-f324a836b06b","name":"ca130470-5738-4f4d-83ce-f2b1547dd60e","description":"3f24243e-7fec-4046-8b00-00d20a1d6c0f"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"9b9786c3-7b7a-4408-a56a-b8698eeeee16","name":"19089526-8303-413c-9c84-75dc626b6b58","description":"663fceb4-c488-4997-a459-da152a2295b9"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"f39ca98d-f7ea-482b-8cfb-873e197b6ae0","name":"4fd9202c-f491-4f8c-87c7-21daed0fd9ae","description":"6ef8cfb8-4ac2-4e7c-9e17-2b1a30b42a68"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"bc7fc587-c61f-42b8-901e-1f39b746aa55","name":"25847768-3e79-4a01-9445-5e6b66dabf83","description":"4781b8e9-e9ce-4487-b93c-d878880a3fa4"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"e7fae49d-8019-401b-b55b-73218f12db34","name":"4041f4f8-6344-457b-941f-518b9cd2bbce","description":"5e8dc5e2-1e6f-41b1-afff-5d9bcaa522f9"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"60035f6e-54f8-4a5c-b372-3a977eb7a813","name":"70b5a681-d972-413f-8c75-1d770c2edc10","description":"5dd00ea2-6878-498a-871a-e3d7f69ba02a"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"459234ac-6615-4ad7-bd1d-3250efd9d714","name":"13ad7c24-c68f-4bf5-ac41-3edcb150df29","description":"2b6a4014-6833-49e9-b31a-d5eb58bb6563"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"9270ef65-d149-47ce-9dd5-9412db96872e","name":"0c5c9f98-a747-48c6-96ae-68d33d3e64e7","description":"074bddd6-90fa-4e34-b297-bac20ff414cf"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"4feb9c25-7889-4f1d-acc8-debb4afa4bb6","name":"2be4c924-41d0-4a2c-8e91-0ab352731f77","description":"f47bc17f-91d7-4632-a989-ab12a0009c52"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"abe43b04-d00d-4cef-a48b-2b57bd847e4c","name":"87dd51fa-98e0-43b8-9ad7-fb5f1004bda2","description":"c02c48e3-40da-4450-acf5-e21625ea29a3"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"4951ba55-9f2d-4776-9e9c-a5f0b5137998","name":"4132353c-9af6-4ca9-8969-bc594f822704","description":"a5c7c87a-892e-49b2-b33a-00fcc64d7d80"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"107c0ddc-0915-403f-9478-a89619af01b8","name":"43c22112-ca6a-4b84-8822-8b27170dcbdd","description":"3cb7dc1f-d567-49cb-a9a6-fdb035f3c38c"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"1a9f4115-3c35-4de0-83bb-faff7e8905ec","name":"a7d6c166-aaf7-459f-90da-83659657ee89","description":"0f80cd82-6aee-48cd-80c6-684eab22d8e1"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"66bb1e32-5e74-41b5-874b-6cc0bf863bd5","name":"3f2152c0-390e-4373-bc97-8ecccd67cebb","description":"271e6fde-05d9-491d-9f3c-ff24ae6d055d"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"581258ea-1d86-43e7-9590-edc9dc91df8a","name":"e837dadf-06af-4aa6-98d9-02978b1b9ad2","description":"eb1925b2-02c2-4553-b5e0-50c20e424d82"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"82523f11-d06b-486a-acd2-2728a828ec7c","name":"535ac3f3-f36c-4914-b766-5900bf34050c","description":"06bb692f-35c1-4056-9e7f-2907ebd24d90"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"a5f50a60-84ab-4cbf-b4b1-f5c82b3a124d","name":"1db3dc4a-8ef8-4336-9ae5-71a58e83b495","description":"3ee85f15-b2a4-4b7d-ad1c-cece1b21cbc0"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"462029d1-d6ec-40dd-86be-e73a1a5a9372","name":"134a35d1-e231-4f0f-83b4-b752245dad84","description":"1f488d80-5f17-47e4-b9b8-cde5190a5322"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"39377706-732a-47b3-8ada-1dd720caf83e","name":"de99834b-c309-4267-ae60-5a679e271c00","description":"06867b6b-904a-4644-a494-e4b444dc2ec3"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"2c37b0a6-6b8e-4dc3-9472-dcdcdf3e89cd","name":"202c46d1-9e04-42b0-ade5-bf13c10ba853","description":"727d1d22-adf7-4ada-b550-9cebeea0086f"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"22b7b79c-ddb1-463f-8664-4e41e49418ce","name":"3bf6df1c-be03-4848-be60-a868e6721d31","description":"3f431200-4ff3-4ae6-b5a1-f60990f84e4b"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"96b39da9-8077-432f-835e-5af263d8fb37","name":"58bd6080-97d7-40c8-8c7e-202b70608f4a","description":"d84b97b2-49f6-4e2a-a1a3-30bf8a234565"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"c16ce1cb-6098-4981-9567-909de05ef1b8","name":"f74c67b4-73f2-478a-ad5e-886fec6771aa","description":"2567eaf5-a9d9-4e67-8627-f8c392524831"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"58b0bf78-1ca1-4904-a60b-4fb727bb8b19","name":"b08f38eb-0ac8-45b5-b7e8-cb78ce0fc3a6","description":"f72daedd-68c2-483c-8ee5-65f114f99133"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"0f29c54d-ddea-48e6-95a0-58ce8e78b37d","name":"a5baabdc-3e23-4fbd-b2f2-c98531dfbc8e","description":"3716353e-90a3-4b09-b502-ed18f51701ab"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"c1f78f1c-191a-4217-868d-1f78364e2442","name":"e8cbe7eb-7f64-4520-ab6d-8dde2689b874","description":"cae949be-71e6-4c41-8f98-0ce70f7b3a38"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"97006bb1-b179-436b-8589-9d8d3f0fb68f","name":"1b4432fd-87d3-4d8a-a929-7eaa7770991f","description":"b85595f4-c795-41ef-b39f-efe0743aecf5"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"3345b507-c0a0-480f-9a1b-0dc1c123e489","name":"367ed46d-ac1b-4b6f-95b1-07897016a16d","description":"edee985b-9853-4f32-9638-378ed6e14a5a"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"65188feb-c1f6-488e-9f8f-18a0f97c9fc8","name":"f6971a62-c8ae-4d0f-9d3c-924db009fa5b","description":"680e4ea9-e7e5-4d96-8d41-9a035750eb0c"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"e4bbad13-2657-4fab-b0be-7a510ef8eebd","name":"ca7f38a1-7228-4961-bd63-76097148de48","description":"9ff18207-7e2a-4574-afbf-0600d5b35fd4"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"0f34dbe6-5303-47f7-a2e7-3214a206a48f","name":"7eaac3be-0b52-4a28-81fb-c73e259aed20","description":"e4114c41-acf7-4c7f-b062-9dab3c06a9f2"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"ffc86e32-89a0-4594-9a6c-56d75bae6e50","name":"bad9ad70-a7d1-44e2-bbd4-309082ca2f97","description":"eeafbda0-7d15-4aab-a267-1be1be705863"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"06170be7-8644-465b-ae20-339f331e5563","name":"65cdaf62-b5f0-4ab2-8ea6-27df5d6978f3","description":"13bca9cb-2f00-4f21-a02d-8e92cbf76eb5"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"fcfd9857-5f11-4c45-920f-80448deaed8b","name":"13970d97-637a-4867-82af-83887afb1435","description":"d16dd260-25b0-4233-8091-0155dfb7b41d"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"d85cfa7c-9a24-4ca5-856c-29f44e12b6ba","name":"59ca0e43-11bc-40ad-a67d-cc1332131a8d","description":"58563f39-b1c7-4563-913e-05ba75026c61"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"8e7b3c9e-c9cf-4f75-a268-49e87ffc3155","name":"62acb0bc-9c64-4164-980c-d91b666b574d","description":"df56a0ac-29cb-447c-a2eb-18a9b6027ef6"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"6d682821-fece-4842-b2b3-153b78dbce12","name":"4f6755de-7065-49de-9de5-ff7dbaa62fdc","description":"b817b1ff-82f5-4a69-9afc-f2ad0e2e82a9"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"29769801-ca5b-44ee-942e-ae372cc4a466","name":"16734380-df21-4062-b084-ca375f90a48d","description":"81ff9c16-3fc9-446b-8afb-d725e21b8f24"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"be1ee9c0-94a0-49b2-b868-41761cf4fde9","name":"dd95d38d-af02-4771-9dce-e17005849ac2","description":"d718dfff-cc55-4adc-bc8a-394be0d5ae8f"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"6c794b77-04c4-4ba4-818c-078c668727cb","name":"bdcfe2f2-a034-4e90-aa5e-46e86e9567fa","description":"aa630450-aee2-4d7e-88f8-bf121ce5204e"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"b269e1e6-c56c-4872-9c17-d5930dd4d3eb","name":"1da0703b-5200-4b93-bd83-5ee6d5738dbd","description":"61f36f56-3a07-4525-954c-22c2b41567a5"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"f3665aac-44fb-4c88-b031-b0b0d843d37f","name":"4c93da9d-6ff0-4c70-84f7-d56dc37ec5cd","description":"eeb185fb-d07c-47db-9811-f327766a75bd"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"dffb04b3-2adf-4776-9a02-b0bae605c646","name":"971d6019-38a5-452b-9853-fdac05f78953","description":"ce249d3a-7d4d-4ab5-b9da-dc002ab2b245"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"41b2aa59-318d-4b4f-9737-42ed441618fb","name":"fe924026-0521-4081-be6e-16c9af185ef7","description":"2b45b979-9041-4e61-9a74-661311dbd118"}]; + const accountId = items[0].accountId; + // const items = new Array(itemCount) + // .fill({}) + // .map(createItem); + // await entity.put(items).go(); - let iterations = 0; let cursor: string | null = null; let results: EntityItem[] = []; @@ -4682,7 +4685,10 @@ describe('query limit', () => { cursor = response.cursor; iterations++; } while (cursor); - + // console.log(JSON.stringify({ + // results: results.sort((a, z) => a.id.localeCompare(z.id)), + // items: items.sort((a, z) => a.id.localeCompare(z.id)) + // }, null, 4)); expect( items.sort((a, z) => a.id.localeCompare(z.id)) ).to.deep.equal( diff --git a/www/src/pages/en/queries/pagination.mdx b/www/src/pages/en/queries/pagination.mdx index b0d54f5b..76e61c46 100644 --- a/www/src/pages/en/queries/pagination.mdx +++ b/www/src/pages/en/queries/pagination.mdx @@ -22,7 +22,11 @@ All ElectroDB `query` and `scan` operations return a `cursor`, which is a string The terminal method `go()` accepts a `cursor` when executing a `query` or `scan` to continue paginating for more results. Pass the cursor from the previous query to your next query and ElectroDB will continue its pagination where it left off. -> To limit the number of items ElectroDB will retrieve, read more about the [Query Options](/en/core-concepts/execution-queries) `pages` and `limit`. +> To limit the number of items ElectroDB will retrieve, read more about the [Execution Options](/en/core-concepts/execution-queries) `pages` and `limit`. + +## Limit + +The execution option `limit` allows you to specify a number of items you'd like to receive. If the number of items returned in a single request to DynamoDB is above the `limit` provided, ElectroDB will paginate until the limit is reached. When using `limit` with `.params()` the DynamoDB `Limit` parameter will be applied. ### Entities From 99d2dc7756ec67fbb806be1e1885c4e4e030cdb0 Mon Sep 17 00:00:00 2001 From: tywalch Date: Tue, 23 May 2023 12:00:02 -0400 Subject: [PATCH 2/4] Changelog update --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d9c2f64..ed29179a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -325,5 +325,9 @@ All notable changes to this project will be documented in this file. Breaking ch ### Added - Adds new query execution option `hydrate`. When a GSI uses a `KEYS_ONLY` projection, the `hydrate` option will perform the query and then a subsequent `batchGet` to "hydrate" the results. +### Changed +- The execution option `limit` is now better optimized for cases where filters might result in index "misses". The `limit` option used to use the `Limit` parameter, but this could result in requiring more requests when user applied filters caused no items from being returned in a single request. + ### Fixed -- A common issue amongst new users, was unexpected errors when using a terminal method twice on a query chain. This would often come up when a user called `.params()` to log out parameters and then call `.go()` on the same chain. The fix was to prevent duplicative side effects from occurring on each subsequent terminal method call. \ No newline at end of file +- A common issue amongst new users, was unexpected errors when using a terminal method twice on a query chain. This would often come up when a user called `.params()` to log out parameters and then call `.go()` on the same chain. The fix was to prevent duplicative side effects from occurring on each subsequent terminal method call. Addresses gh issue #239. + From ba4a8a6e520e86a95a03ef859fdc07d58119b4cb Mon Sep 17 00:00:00 2001 From: tywalch Date: Tue, 23 May 2023 12:01:16 -0400 Subject: [PATCH 3/4] Restores tests --- test/ts_connected.crud.spec.ts | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/test/ts_connected.crud.spec.ts b/test/ts_connected.crud.spec.ts index d066f2be..966936f3 100644 --- a/test/ts_connected.crud.spec.ts +++ b/test/ts_connected.crud.spec.ts @@ -4656,23 +4656,21 @@ describe('query limit', () => { table, client }); - // const accountId = uuid(); - // const createItem = () => { - // return { - // accountId, - // id: uuid(), - // name: uuid(), - // description: uuid(), - // } - // } + const accountId = uuid(); + const createItem = () => { + return { + accountId, + id: uuid(), + name: uuid(), + description: uuid(), + } + } const limit = 10; const itemCount = 100; - const items = [{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"9285ca74-c726-4843-b5ce-c9d8b95c5321","name":"350dd74e-0269-4212-a157-284dd8af3163","description":"9f501f00-176e-45a7-b585-b59ecd4dfb98"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"b71f67c9-af08-4c3a-b1a0-1f365fef3fdf","name":"65197748-af8b-4db6-ad3e-da4618ef08f1","description":"2c848057-3e17-42ae-be20-49b3dc702ea4"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"163f9c9a-2e6f-4250-bce0-249c3ff383ca","name":"21c22079-4267-427a-8b8a-70b41fdb53ee","description":"5b8b20bd-cf12-4082-9c14-c545b2661f79"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"942fea6c-e3d2-4cb0-bdeb-aad8bc9ac90c","name":"e46d67db-22f4-4ba1-9ffc-d939a535822d","description":"cfbc25fb-b231-471c-bb89-7c76b19fcc9c"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"b6781bbb-12b6-4528-9da0-f867c168e89b","name":"fdf65da2-8833-4afc-934c-3c282cfbb080","description":"890558b8-4ed9-4617-9078-1d2d05ad7836"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"ea1d4c70-bf39-4e7c-ba54-0d20d48c9b6d","name":"57938032-c0ad-4565-8b0a-53e95531b256","description":"60c2c2f5-2a3c-43c4-8bda-8b30a293f744"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"25c6cbd5-863d-45be-a237-793a0a48206d","name":"dfd49c67-5554-4605-873f-0d69f8f48ad6","description":"6dc691cf-640e-4c70-8573-ae1789a89c35"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"2a50954f-f2fd-4668-9f02-a074198f70df","name":"9accd38a-557b-4956-98cf-483de6cb1450","description":"0417755f-b1b5-442f-9a57-d15c888e0bf0"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"f20ec780-405c-4b9b-ac2f-7818f0d00712","name":"81d236fd-5ecd-4560-86cc-6743372d66cc","description":"a46c0f02-a547-4526-a053-f4b3e02c74ef"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"f92dda61-855c-4d00-a99e-eadb8b4b1df0","name":"a05113bc-7e32-4524-9dc7-d6d3d029610d","description":"7ec403b4-67eb-4a87-9a26-6169552e3d23"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"05705694-ea52-42d5-a4f6-818fbb727816","name":"ded13657-a9bc-4885-b12c-0c06318f74aa","description":"ced6b30e-c4f1-4f93-8661-51a2e0b1ec3d"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"7624396d-8dea-4a08-8357-02b39aa5eb32","name":"b5e0a922-e76d-4962-8079-bd98ddb8902b","description":"b3713f7e-101c-467d-85d8-1e79d1f56fe7"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"fd5ab144-48b9-431c-819c-814ad5221471","name":"227fdf6b-18f6-4db2-b652-61ee518a6c1f","description":"07c0ca8b-b407-426f-b88f-2fc8939bdfce"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"cc71765e-92cd-44b2-9020-281eca6e1133","name":"586c29d8-0b2b-42d9-9dbc-eb3f992c25fb","description":"a5a474b7-399f-4382-924f-2999584e5b06"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"2a49d1bb-0ade-4578-aed7-c338ed564df2","name":"35dba9e2-3d8b-48b3-8e3c-941747d9bbb6","description":"e1c2b309-62d4-41d9-9b72-c5beec77ed39"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"273f5bb8-892a-451f-9418-cec75bab7bdd","name":"e08c9382-9214-425e-a942-386697eaeabb","description":"088d091e-cd2f-4b0c-9d47-53fc032cc532"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"c58c20e8-506f-4c96-8aa4-f10eadfa8464","name":"9c1ae16f-07b9-4f4b-9546-79bdf64cce87","description":"9147a6f4-5a88-4f6b-bf16-50c4eda8e337"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"ab5e6d14-1704-4699-99ef-f5cdf8d15cae","name":"ef9c9fc9-1761-49cb-b60d-564198e24f3f","description":"c09dfd83-a025-4701-b856-e2e367da1ee3"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"77295160-a2af-480c-b672-7d19452f5f98","name":"ca938fe1-c0a0-46b5-a4c7-c2e68ad87a0b","description":"010939aa-7eb4-485c-ba9b-4ad428ae145b"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"887de7ae-396c-4203-82fa-a13b2a26f61d","name":"6e09a922-faf5-40b5-97a8-ffc7207f9422","description":"84f462b1-80d5-46c7-add7-76a5d77c3f61"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"1c350017-421e-451a-80af-8f5115dbcb8a","name":"56f114bb-e660-4d59-8174-3d4ac28cb97d","description":"d57d5beb-4dfa-4d53-864b-372bca570d38"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"7bd908ec-12bc-4c61-9e01-8c7db8992560","name":"62e5f333-0b8b-4e59-aae7-923414ecb48e","description":"3399cada-d48c-4247-aab1-924c0b6ed723"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"71978d28-022a-45ce-951b-58006356e9af","name":"dae48ac9-cc77-45bc-ab1a-8b68747d101e","description":"63a54f8e-a3cb-499c-8998-c03f18dab50e"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"d1123e7a-a6d9-4583-bbef-8d86741abb53","name":"34c19579-5a64-4014-9a1c-76f051911ae4","description":"695373f3-6a24-41f9-9989-c43f6ceec643"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"054cc007-0ad5-40cb-9c1f-f63c1e27db85","name":"0a81608d-4046-4b75-b256-d8c24e2c7b21","description":"9d4ad35f-ef22-48a4-b2b6-eed7e56d2bca"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"95428a2c-4d8a-4da7-b922-c244fca2e571","name":"1cf80c5b-4e53-4dcd-b760-0297d9e4740b","description":"aec50d8a-f9a7-4736-8cba-498ea1f92ed8"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"42ed2e4f-5152-47f2-9306-9cd8316c2fde","name":"2c134481-e48c-4aa7-987b-fee16b57c8b2","description":"d4520bce-aa2e-4114-a1a3-79afee68b4b5"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"124541e1-619c-49ed-bf30-ce064aa1c85a","name":"2c2445fd-599b-4a90-874b-18840df75ea3","description":"03dbf75f-ab15-4841-9338-e22ccd935eee"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"e31a967c-e139-46f9-abba-48a8f0b46a25","name":"07dd68c5-51f1-475a-a08f-ca738ab4a6ad","description":"6d217d00-d572-4bb7-a7f5-2c8d80076194"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"4486a6b5-a19a-4709-a273-07f1dc9c6f33","name":"056fd004-3fd7-4dd1-9950-bde4d349c8d3","description":"e8d3f2b5-f8ca-43dd-9028-3406eee233b8"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"8f49efe9-6033-4c02-bf73-b702bc91f7d2","name":"bc730646-40f3-4f41-936c-33669c895209","description":"1c929671-9d8c-46e2-8dcd-4205e7a53288"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"18f413e3-e133-492a-99aa-34566f81dc8c","name":"efab6cd6-8894-449d-8444-4da95d288a42","description":"7d96ac9b-882c-4321-b4fd-71ab1e87bcb4"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"6f05e97a-bed2-4a7e-a057-5b836497e0b3","name":"92740971-247a-4fbc-bdd1-3df43a712265","description":"d5022ce6-f80c-45f6-a8f5-6718cf71590d"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"49d1674a-7744-49d2-b26d-a0ddf73ab199","name":"abe66f33-2321-42ee-80af-707da3a57443","description":"2d70b61e-9492-4dd4-a240-8de73c0b9698"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"0cf5c473-bdcb-45df-a53d-3d127e6cada9","name":"6cb5922a-19db-4c4c-a176-2d959bbc7e50","description":"7d5d5924-d29a-460c-9ca9-9ea8915db305"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"db1838cc-f196-498d-8c6e-342f06c0e475","name":"84293b70-dc5a-48f1-bd89-fb0807a7818b","description":"308bf638-fdbd-4782-b164-072f7b23ade0"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"a7827224-7f62-4025-bf19-31b41a7d0711","name":"c12202e7-47ff-498e-890f-332c2e74b85d","description":"6333cf19-70f5-424d-b75b-25edb2304f29"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"e41e97ba-392d-46a3-9cab-5e11c7c28788","name":"833bbad0-f89c-4620-890a-42d7812243df","description":"116406f1-3f15-4380-87c1-9bde7fdd52d7"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"84404876-75dd-4412-9554-0aeb37f6866e","name":"e29eb07d-48ce-463e-bf1d-15d8fc2bd24a","description":"ef6e2a13-a09e-4757-aafe-3036a82b714c"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"c98f5877-5aa2-4261-be5c-ea3503fb4799","name":"6ec5c4e4-a257-437d-ba44-a3f10f49a795","description":"05afddcd-f3e1-4135-9992-d3db653e2ed0"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"aab35584-7716-47c1-b739-427a3a678450","name":"bd7a9197-6382-4001-9b71-bd7e9e36fc8e","description":"d65bdc00-671d-4dbf-8102-3df2568e8a31"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"00a613ac-6664-411c-b134-5a0fb8e34c3e","name":"6433a85f-c0e5-401b-b29f-6045ea4432b7","description":"3488ebe3-c6f7-4545-b1bf-965f5b38c34d"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"fc448a17-954f-42c0-8e6e-d0b1974886c3","name":"8114245f-7eaa-4ea7-a75e-1fd2b52eeeb3","description":"2717d2b9-29f6-490c-a63f-b50185d7eb03"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"68a98739-f79e-48a3-8613-1d0b2508c519","name":"9a395854-5b3f-4546-b304-f4666bacf80b","description":"f762a41d-6dec-4638-ab34-dd114b7ed9a1"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"816f12c8-d7c5-4a39-ba81-c6daac5ea380","name":"623f5823-4077-48ec-b245-84784b46abd3","description":"93b4b37a-c896-4333-bc6a-5ee4109a41cf"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"1047d2b4-b6c6-4413-94c1-adae1a367917","name":"de6bcfdb-a501-498d-8554-cf075ef3f791","description":"02358778-78d0-4155-a236-e836c28aa410"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"b1c7e3dc-ab49-4297-9b71-a07544e5223e","name":"535c6e38-b3e7-49de-bfda-5544a68e40f8","description":"a05861e8-fbea-4e0e-8930-7df9a9dbd6a6"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"3392d3f2-f93c-48be-9a88-4b988c5fd8f2","name":"98780ac7-69ba-4a57-80aa-e9f6b86a95da","description":"32414088-80a8-4582-aefe-ef1051b4cf93"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"d32061b3-b738-4f4c-9ac8-46f6764a1973","name":"e90e6ea9-fa2b-4f8d-8e4d-bb206c0eb79a","description":"75cb76da-bd0b-4783-b8b5-ecf70e7dc8c6"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"4e86be6b-2cb6-49b7-9129-1a4288fd303a","name":"f7c62002-6fdc-481c-87fa-5c4383b60cb8","description":"39da81a3-e844-4f38-8199-efead6787971"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"37e24ea2-e594-41c7-bd62-02eb156ec982","name":"0a3bc46b-3d0a-40bb-a80a-ed6caf3bdcf5","description":"09489f8a-f1d8-4529-a389-88277bea9427"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"5688424d-29bf-433a-8022-cf2c70d6dd37","name":"19cfab03-c223-426f-8c05-f2b1e8ff4e20","description":"40ce7461-848e-4352-a22f-12929d63ac38"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"36904b64-b0dc-4220-bae6-96178d2b02ac","name":"468e68ed-0677-4592-965c-7062d47dbddd","description":"bc70ce95-dca4-44c7-9a6d-e1720b0c830f"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"bee13085-a600-4084-9ba2-5c740b158d8b","name":"67c9ea59-4b8e-427f-8eaa-009a44bde7b9","description":"93c21563-8e70-4a77-a47a-7178a9234eb9"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"1f244c2e-af58-4d3b-a0ce-6ccb0d57ad63","name":"73908a75-ed06-4f17-8267-45a2ab5195a6","description":"a4e78079-cfb3-4e9c-b0f4-83893de0e4dc"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"6cbfe482-a848-4c19-9fae-a987557f0407","name":"bb79fa10-d98c-4ae0-bf38-32c0a5395080","description":"b4289b45-d58f-4472-925b-3b1ecccff932"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"78f6b720-2ad1-4ae3-b905-f324a836b06b","name":"ca130470-5738-4f4d-83ce-f2b1547dd60e","description":"3f24243e-7fec-4046-8b00-00d20a1d6c0f"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"9b9786c3-7b7a-4408-a56a-b8698eeeee16","name":"19089526-8303-413c-9c84-75dc626b6b58","description":"663fceb4-c488-4997-a459-da152a2295b9"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"f39ca98d-f7ea-482b-8cfb-873e197b6ae0","name":"4fd9202c-f491-4f8c-87c7-21daed0fd9ae","description":"6ef8cfb8-4ac2-4e7c-9e17-2b1a30b42a68"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"bc7fc587-c61f-42b8-901e-1f39b746aa55","name":"25847768-3e79-4a01-9445-5e6b66dabf83","description":"4781b8e9-e9ce-4487-b93c-d878880a3fa4"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"e7fae49d-8019-401b-b55b-73218f12db34","name":"4041f4f8-6344-457b-941f-518b9cd2bbce","description":"5e8dc5e2-1e6f-41b1-afff-5d9bcaa522f9"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"60035f6e-54f8-4a5c-b372-3a977eb7a813","name":"70b5a681-d972-413f-8c75-1d770c2edc10","description":"5dd00ea2-6878-498a-871a-e3d7f69ba02a"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"459234ac-6615-4ad7-bd1d-3250efd9d714","name":"13ad7c24-c68f-4bf5-ac41-3edcb150df29","description":"2b6a4014-6833-49e9-b31a-d5eb58bb6563"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"9270ef65-d149-47ce-9dd5-9412db96872e","name":"0c5c9f98-a747-48c6-96ae-68d33d3e64e7","description":"074bddd6-90fa-4e34-b297-bac20ff414cf"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"4feb9c25-7889-4f1d-acc8-debb4afa4bb6","name":"2be4c924-41d0-4a2c-8e91-0ab352731f77","description":"f47bc17f-91d7-4632-a989-ab12a0009c52"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"abe43b04-d00d-4cef-a48b-2b57bd847e4c","name":"87dd51fa-98e0-43b8-9ad7-fb5f1004bda2","description":"c02c48e3-40da-4450-acf5-e21625ea29a3"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"4951ba55-9f2d-4776-9e9c-a5f0b5137998","name":"4132353c-9af6-4ca9-8969-bc594f822704","description":"a5c7c87a-892e-49b2-b33a-00fcc64d7d80"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"107c0ddc-0915-403f-9478-a89619af01b8","name":"43c22112-ca6a-4b84-8822-8b27170dcbdd","description":"3cb7dc1f-d567-49cb-a9a6-fdb035f3c38c"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"1a9f4115-3c35-4de0-83bb-faff7e8905ec","name":"a7d6c166-aaf7-459f-90da-83659657ee89","description":"0f80cd82-6aee-48cd-80c6-684eab22d8e1"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"66bb1e32-5e74-41b5-874b-6cc0bf863bd5","name":"3f2152c0-390e-4373-bc97-8ecccd67cebb","description":"271e6fde-05d9-491d-9f3c-ff24ae6d055d"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"581258ea-1d86-43e7-9590-edc9dc91df8a","name":"e837dadf-06af-4aa6-98d9-02978b1b9ad2","description":"eb1925b2-02c2-4553-b5e0-50c20e424d82"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"82523f11-d06b-486a-acd2-2728a828ec7c","name":"535ac3f3-f36c-4914-b766-5900bf34050c","description":"06bb692f-35c1-4056-9e7f-2907ebd24d90"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"a5f50a60-84ab-4cbf-b4b1-f5c82b3a124d","name":"1db3dc4a-8ef8-4336-9ae5-71a58e83b495","description":"3ee85f15-b2a4-4b7d-ad1c-cece1b21cbc0"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"462029d1-d6ec-40dd-86be-e73a1a5a9372","name":"134a35d1-e231-4f0f-83b4-b752245dad84","description":"1f488d80-5f17-47e4-b9b8-cde5190a5322"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"39377706-732a-47b3-8ada-1dd720caf83e","name":"de99834b-c309-4267-ae60-5a679e271c00","description":"06867b6b-904a-4644-a494-e4b444dc2ec3"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"2c37b0a6-6b8e-4dc3-9472-dcdcdf3e89cd","name":"202c46d1-9e04-42b0-ade5-bf13c10ba853","description":"727d1d22-adf7-4ada-b550-9cebeea0086f"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"22b7b79c-ddb1-463f-8664-4e41e49418ce","name":"3bf6df1c-be03-4848-be60-a868e6721d31","description":"3f431200-4ff3-4ae6-b5a1-f60990f84e4b"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"96b39da9-8077-432f-835e-5af263d8fb37","name":"58bd6080-97d7-40c8-8c7e-202b70608f4a","description":"d84b97b2-49f6-4e2a-a1a3-30bf8a234565"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"c16ce1cb-6098-4981-9567-909de05ef1b8","name":"f74c67b4-73f2-478a-ad5e-886fec6771aa","description":"2567eaf5-a9d9-4e67-8627-f8c392524831"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"58b0bf78-1ca1-4904-a60b-4fb727bb8b19","name":"b08f38eb-0ac8-45b5-b7e8-cb78ce0fc3a6","description":"f72daedd-68c2-483c-8ee5-65f114f99133"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"0f29c54d-ddea-48e6-95a0-58ce8e78b37d","name":"a5baabdc-3e23-4fbd-b2f2-c98531dfbc8e","description":"3716353e-90a3-4b09-b502-ed18f51701ab"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"c1f78f1c-191a-4217-868d-1f78364e2442","name":"e8cbe7eb-7f64-4520-ab6d-8dde2689b874","description":"cae949be-71e6-4c41-8f98-0ce70f7b3a38"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"97006bb1-b179-436b-8589-9d8d3f0fb68f","name":"1b4432fd-87d3-4d8a-a929-7eaa7770991f","description":"b85595f4-c795-41ef-b39f-efe0743aecf5"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"3345b507-c0a0-480f-9a1b-0dc1c123e489","name":"367ed46d-ac1b-4b6f-95b1-07897016a16d","description":"edee985b-9853-4f32-9638-378ed6e14a5a"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"65188feb-c1f6-488e-9f8f-18a0f97c9fc8","name":"f6971a62-c8ae-4d0f-9d3c-924db009fa5b","description":"680e4ea9-e7e5-4d96-8d41-9a035750eb0c"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"e4bbad13-2657-4fab-b0be-7a510ef8eebd","name":"ca7f38a1-7228-4961-bd63-76097148de48","description":"9ff18207-7e2a-4574-afbf-0600d5b35fd4"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"0f34dbe6-5303-47f7-a2e7-3214a206a48f","name":"7eaac3be-0b52-4a28-81fb-c73e259aed20","description":"e4114c41-acf7-4c7f-b062-9dab3c06a9f2"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"ffc86e32-89a0-4594-9a6c-56d75bae6e50","name":"bad9ad70-a7d1-44e2-bbd4-309082ca2f97","description":"eeafbda0-7d15-4aab-a267-1be1be705863"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"06170be7-8644-465b-ae20-339f331e5563","name":"65cdaf62-b5f0-4ab2-8ea6-27df5d6978f3","description":"13bca9cb-2f00-4f21-a02d-8e92cbf76eb5"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"fcfd9857-5f11-4c45-920f-80448deaed8b","name":"13970d97-637a-4867-82af-83887afb1435","description":"d16dd260-25b0-4233-8091-0155dfb7b41d"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"d85cfa7c-9a24-4ca5-856c-29f44e12b6ba","name":"59ca0e43-11bc-40ad-a67d-cc1332131a8d","description":"58563f39-b1c7-4563-913e-05ba75026c61"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"8e7b3c9e-c9cf-4f75-a268-49e87ffc3155","name":"62acb0bc-9c64-4164-980c-d91b666b574d","description":"df56a0ac-29cb-447c-a2eb-18a9b6027ef6"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"6d682821-fece-4842-b2b3-153b78dbce12","name":"4f6755de-7065-49de-9de5-ff7dbaa62fdc","description":"b817b1ff-82f5-4a69-9afc-f2ad0e2e82a9"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"29769801-ca5b-44ee-942e-ae372cc4a466","name":"16734380-df21-4062-b084-ca375f90a48d","description":"81ff9c16-3fc9-446b-8afb-d725e21b8f24"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"be1ee9c0-94a0-49b2-b868-41761cf4fde9","name":"dd95d38d-af02-4771-9dce-e17005849ac2","description":"d718dfff-cc55-4adc-bc8a-394be0d5ae8f"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"6c794b77-04c4-4ba4-818c-078c668727cb","name":"bdcfe2f2-a034-4e90-aa5e-46e86e9567fa","description":"aa630450-aee2-4d7e-88f8-bf121ce5204e"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"b269e1e6-c56c-4872-9c17-d5930dd4d3eb","name":"1da0703b-5200-4b93-bd83-5ee6d5738dbd","description":"61f36f56-3a07-4525-954c-22c2b41567a5"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"f3665aac-44fb-4c88-b031-b0b0d843d37f","name":"4c93da9d-6ff0-4c70-84f7-d56dc37ec5cd","description":"eeb185fb-d07c-47db-9811-f327766a75bd"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"dffb04b3-2adf-4776-9a02-b0bae605c646","name":"971d6019-38a5-452b-9853-fdac05f78953","description":"ce249d3a-7d4d-4ab5-b9da-dc002ab2b245"},{"accountId":"0a14c757-895b-4f6c-95ac-e3343142bd7f","id":"41b2aa59-318d-4b4f-9737-42ed441618fb","name":"fe924026-0521-4081-be6e-16c9af185ef7","description":"2b45b979-9041-4e61-9a74-661311dbd118"}]; - const accountId = items[0].accountId; - // const items = new Array(itemCount) - // .fill({}) - // .map(createItem); - // + const items = new Array(itemCount) + .fill({}) + .map(createItem); + await entity.put(items).go(); let iterations = 0; let cursor: string | null = null; @@ -4685,10 +4683,7 @@ describe('query limit', () => { cursor = response.cursor; iterations++; } while (cursor); - // console.log(JSON.stringify({ - // results: results.sort((a, z) => a.id.localeCompare(z.id)), - // items: items.sort((a, z) => a.id.localeCompare(z.id)) - // }, null, 4)); + expect( items.sort((a, z) => a.id.localeCompare(z.id)) ).to.deep.equal( From 1db0172abc18037e91817b73bff97acb17f36014 Mon Sep 17 00:00:00 2001 From: tywalch Date: Thu, 25 May 2023 10:52:09 -0400 Subject: [PATCH 4/4] Working tests --- src/entity.js | 194 ++++++++++----------- src/service.js | 39 ++++- src/transaction.js | 22 ++- test/connected.page.spec.js | 296 +++++++++++++++++++++++++++++++-- test/connected.service.spec.js | 10 +- test/ts_connected.crud.spec.ts | 8 +- 6 files changed, 427 insertions(+), 142 deletions(-) diff --git a/src/entity.js b/src/entity.js index 11c7c7eb..45f76b47 100644 --- a/src/entity.js +++ b/src/entity.js @@ -382,9 +382,7 @@ class Entity { let response = await this._exec(MethodTypes.batchWrite, params, config); if (validations.isFunction(config.parse)) { let parsed = config.parse(config, response); - if (parsed) { - results.push(parsed); - } + results.push(parsed.data); } else { let {unprocessed} = this.formatBulkWriteResponse(response, config); for (let u of unprocessed) { @@ -432,7 +430,8 @@ class Entity { await Promise.all(operation.map(async params => { let response = await this._exec(MethodTypes.batchGet, params, config); if (validations.isFunction(config.parse)) { - resultsAll.push(config.parse(config, response)); + const parsed = config.parse(config, response); + resultsAll.push(parsed.data); } else { this.applyBulkGetResponseFormatting({ orderMaintainer, @@ -473,32 +472,41 @@ class Entity { ExclusiveStartKey = undefined; } let pages = this._normalizePagesValue(config.pages); - let max = this._normalizeLimitValue(config.limit); - let providedLimit = parameters.Limit; - delete parameters.Limit; + let configLimit = this._normalizeNumberOptionsValue('limit', config.limit); + let configCount = this._normalizeNumberOptionsValue('count', config.count); + let max = this._safeMinimum(configLimit, configCount); let iterations = 0; let count = 0; let hydratedUnprocessed = []; const shouldHydrate = config.hydrate && method === MethodTypes.query; do { - let limit = max === undefined - ? providedLimit - : max - count; - let response = await this._exec(method, { ExclusiveStartKey, ...parameters }, config); - response = this._maybeApplyArtificialLimit({ - response, - limit, - indexName: parameters.IndexName - }); - // console.log('RESPONSE', JSON.stringify({ response }, null, 4)); + let remainingCount = configCount !== undefined + ? max - count + : undefined; + + let limit = configLimit !== undefined + ? max - count + : undefined; + + let params = { ExclusiveStartKey, ...parameters }; + + if (config.raw || (limit !== undefined && remainingCount === undefined)) { + params.Limit = limit; + } + + let response = await this._exec(method, params, config); - ExclusiveStartKey = response.LastEvaluatedKey; response = this.formatResponse(response, parameters.IndexName, { ...config, + count: remainingCount, includeKeys: shouldHydrate || config.includeKeys, ignoreOwnership: shouldHydrate || config.ignoreOwnership, + _returnLastEvaluatedKeyRaw: true, }); + ExclusiveStartKey = response.lastEvaluatedKey; + delete response.lastEvaluatedKey; + if (config.raw) { return response; } else if (config._isCollectionQuery) { @@ -664,34 +672,21 @@ class Entity { } } - _maybeApplyArtificialLimit({response, limit, indexName = TableIndex}) { - let Items = []; - for (let i = 0; i < limit; i++) { - const item = response.Items[i]; - Items.push(item); + _getLastEvaluatedKeyFromItem({indexName = TableIndex, item}) { + const indexFields = this.model.translations.keys[indexName]; + const tableIndexFields = this.model.translations.keys[TableIndex]; + const lastEvaluatedKey = { + [indexFields.pk]: item[indexFields.pk], + [tableIndexFields.pk]: item[tableIndexFields.pk], } - let LastEvaluatedKey = response.LastEvaluatedKey; - if (Array.isArray(response.Items) && response.Items.length > limit) { - const itemAtLimit = Items[Items.length - 1]; - const indexFields = this.model.translations.keys[indexName]; - const tableIndexFields = this.model.translations.keys[TableIndex]; - LastEvaluatedKey = { - [indexFields.pk]: itemAtLimit[indexFields.pk], - [tableIndexFields.pk]: itemAtLimit[tableIndexFields.pk], - } - if (indexFields.sk && itemAtLimit[indexFields.sk]) { - LastEvaluatedKey[indexFields.sk] = itemAtLimit[indexFields.sk] - } - if (tableIndexFields.sk && itemAtLimit[tableIndexFields.sk]) { - LastEvaluatedKey[tableIndexFields.sk] = itemAtLimit[tableIndexFields.sk] - } + if (indexFields.sk && item[indexFields.sk]) { + lastEvaluatedKey[indexFields.sk] = item[indexFields.sk] + } + if (tableIndexFields.sk && item[tableIndexFields.sk]) { + lastEvaluatedKey[tableIndexFields.sk] = item[tableIndexFields.sk] } - return { - ...response, - Items, - LastEvaluatedKey, - }; + return lastEvaluatedKey; } formatResponse(response, index, config = {}) { @@ -699,10 +694,13 @@ class Entity { if (!config.originalErr) { stackTrace = new e.ElectroError(e.ErrorCodes.AWSError); } + let lastEvaluatedKey = response.LastEvaluatedKey; try { let results = {}; if (validations.isFunction(config.parse)) { - results = config.parse(config, response); + const parsed = config.parse(config, response); + results = parsed.data; + lastEvaluatedKey = parsed.lastEvaluatedKey; } else if (config.raw && !config._isPagination) { if (response.TableName) { results = {}; @@ -722,12 +720,27 @@ class Entity { results = null; } } else if (response.Items) { + let size = typeof config.count === 'number' ? config.count : response.Items.length; + let count = 0; + let lastItem; results = []; - for (let item of response.Items) { + for (let i = 0; i < response.Items.length; i++) { + const item = { ...response.Items[i] }; if (config.ignoreOwnership || this.ownsItem(item)) { let record = this.model.schema.formatItemForRetrieval(item, config); if (Object.keys(record).length > 0) { + count = count + 1; + if (count > size) { + if (lastItem) { + lastEvaluatedKey = this._getLastEvaluatedKeyFromItem({ + indexName: index, + item: lastItem, + }); + } + break; + } results.push(record); + lastItem = response.Items[i]; } } } @@ -743,8 +756,11 @@ class Entity { } } - if (config._isPagination || response.LastEvaluatedKey) { - const nextPage = this._formatReturnPager(config, response.LastEvaluatedKey); + if (config._isPagination || lastEvaluatedKey) { + const nextPage = this._formatReturnPager(config, lastEvaluatedKey); + if (config._returnLastEvaluatedKeyRaw) { + return { cursor: nextPage || null, data: results, lastEvaluatedKey }; + } return { cursor: nextPage || null, data: results }; } @@ -857,16 +873,31 @@ class Entity { return value; } - _normalizeLimitValue(value) { + _normalizeNumberOptionsValue(option, value) { if (value !== undefined) { value = parseInt(value); if (isNaN(value) || value < 1) { - throw new e.ElectroError(e.ErrorCodes.InvalidLimitOption, "Query option 'limit' must be of type 'number' and greater than zero."); + throw new e.ElectroError(e.ErrorCodes.InvalidLimitOption, `Query option '${option}' must be of type 'number' and greater than zero.`); } } return value; } + _safeMinimum(...values) { + let eligibleNumbers = []; + for (let value of values) { + if (typeof value === 'number') { + eligibleNumbers.push(value); + } + } + + if (eligibleNumbers.length) { + return Math.min(...eligibleNumbers); + } + + return undefined; + } + _createKeyDeconstructor(prefixes = {}, labels = [], attributes = {}) { let {prefix, isCustom, postfix} = prefixes; let names = []; @@ -921,65 +952,6 @@ class Entity { } } - // _deconstructKeys(index, keyType, key, backupFacets = {}) { - // if (typeof key !== "string" || key.length === 0) { - // return null; - // } - // - // let accessPattern = this.model.translations.indexes.fromIndexToAccessPattern[index]; - // let {prefix, isCustom} = this.model.prefixes[index][keyType]; - // let {facets} = this.model.indexes[accessPattern][keyType]; - // let names = []; - // let types = []; - // let pattern = `^${this._regexpEscape(prefix)}`; - // let labels = this.model.facets.labels[index][keyType] || []; - // for (let {name, label} of labels) { - // let attr = this.model.schema.attributes[name]; - // if (attr) { - // if (isCustom) { - // pattern += `${this._regexpEscape(label === undefined ? "" : label)}(.+)`; - // } else { - // pattern += `#${this._regexpEscape(label === undefined ? name : label)}_(.+)`; - // } - // names.push(name); - // types.push(attr.type); - // } - // } - // pattern += "$"; - // let regex = new RegExp(pattern, "i"); - // let match = key.match(regex); - // let results = {}; - // if (match) { - // for (let i = 0; i < names.length; i++) { - // let key = names[i]; - // let value = match[i+1]; - // let type = types[i]; - // switch (type) { - // case "number": - // value = parseFloat(value); - // break; - // case "boolean": - // value = value === "true"; - // break; - // } - // results[key] = value; - // } - // } else { - // if (Object.keys(backupFacets || {}).length === 0) { - // // this can occur when a scan is performed but returns no results given the current filters or record timing - // return {}; - // } - // for (let facet of facets) { - // if (backupFacets[facet] === undefined) { - // throw new e.ElectroError(e.ErrorCodes.LastEvaluatedKey, 'LastEvaluatedKey contains entity that does not match the entity used to query. Use {pager: "raw"} query option.'); - // } else { - // results[facet] = backupFacets[facet]; - // } - // } - // } - // return results; - // } - _deconstructIndex({index = TableIndex, keys = {}} = {}) { const hasIndex = !!this.model.translations.keys[index]; if (!hasIndex) { @@ -1198,6 +1170,10 @@ class Entity { config.params.Limit = option.limit; } + if (typeof option.count === 'number') { + config.count = option.count; + } + if (validations.isStringHasLength(option.table)) { config.params.TableName = option.table; config.table = option.table; @@ -1703,8 +1679,6 @@ class Entity { const { updatedKeys, setAttributes, indexKey } = this._getPutKeys(pk, sk && sk.facets, upsert.data); const upsertAttributes = this.model.schema.translateToFields(setAttributes); const keyNames = Object.keys(indexKey); - // update.set(this.identifiers.entity, this.getName()); - // update.set(this.identifiers.version, this.getVersion()); for (const field of [...Object.keys(upsertAttributes), ...Object.keys(updatedKeys)]) { const value = u.getFirstDefined(upsertAttributes[field], updatedKeys[field]); if (!keyNames.includes(field)) { diff --git a/src/service.js b/src/service.js index dc21f3da..7da42495 100644 --- a/src/service.js +++ b/src/service.js @@ -305,19 +305,26 @@ class Service { cleanseRetrievedData(index = TableIndex, entities, data = {}, config = {}) { if (config.raw) { - return data; + if (config._returnLastEvaluatedKeyRaw) { + return {data, lastEvaluatedKey: data.LastEvaluatedKey}; + } else { + return { data }; + } } const identifiers = getEntityIdentifiers(entities); data.Items = data.Items || []; const results = {}; + let size = typeof config.count === 'number' ? config.count : data.Items.length; + let count = 0; + let lastEvaluatedKey = data.LastEvaluatedKey; for (let {alias} of identifiers) { results[alias] = []; } for (let i = 0; i < data.Items.length; i++) { - const record = data.Items[i]; + const record = { ...data.Items[i] }; if (!record) { continue; @@ -333,7 +340,7 @@ class Service { let formatted; if (config.hydrate) { formatted = { - data: record // entities[entityAlias]._formatKeysToItem(index, record), + data: record }; } else { formatted = entities[entityAlias].formatResponse({Item: record}, index, { @@ -343,9 +350,29 @@ class Service { }); } - results[entityAlias].push(formatted.data); + if (formatted.data) { + count = count + 1; + if (count > size) { + lastEvaluatedKey = entities[entityAlias]._getLastEvaluatedKeyFromItem({ + indexName: index, + item: data.Items[i - 1], + }); + break; + } + results[entityAlias].push(formatted.data); + } + } + + if (config._returnLastEvaluatedKeyRaw) { + return { + data: results, + lastEvaluatedKey + }; + } else { + return { + data: results, + } } - return results; } findKeyOwner(lastEvaluatedKey) { @@ -409,7 +436,7 @@ class Service { // expressions, // DynamoDB doesnt return what I expect it would when provided with these entity filters parse: (options, data) => { if (options.raw) { - return data; + return { data }; } return this.cleanseRetrievedData(index, entities, data, options); }, diff --git a/src/transaction.js b/src/transaction.js index 0ee6df91..29eabd4e 100644 --- a/src/transaction.js +++ b/src/transaction.js @@ -119,29 +119,43 @@ function createTransaction(options) { ...options, parse: (options, data) => { if (options.raw) { - return data; + return { + data + }; } else if (data.canceled) { canceled = true; - return cleanseCanceledData(TableIndex, getEntities(), data, { + const cleansed = cleanseCanceledData(TableIndex, getEntities(), data, { ...options, _isTransaction: true, _paramItems: paramItems, }); + + return { + data: cleansed, + } } else if (data.Responses) { - return cleanseTransactionData(TableIndex, getEntities(), { + const cleansed = cleanseTransactionData(TableIndex, getEntities(), { Items: data.Responses.map(response => response.Item) }, { ...options, _isTransaction: true, _paramItems: paramItems, }); + + return { + data: cleansed, + } } else { - return new Array(paramItems ? paramItems.length : 0).fill({ + const items = new Array(paramItems ? paramItems.length : 0).fill({ item: null, code: 'None', rejected: false, message: undefined, }); + + return { + data: items, + } } } }); diff --git a/test/connected.page.spec.js b/test/connected.page.spec.js index 7975d8eb..25d1f223 100644 --- a/test/connected.page.spec.js +++ b/test/connected.page.spec.js @@ -305,7 +305,7 @@ describe("Page", () => { } ]; for (const test of paginationTests) { - it('should paginate through all records for a given query', async () => { + it(`should paginate through all records for a given ${test.type} operation using limit`, async () => { const pages = 'all'; const limit = 2; let results = []; @@ -322,9 +322,27 @@ describe("Page", () => { test.output, )).to.not.throw; }).timeout(10000); + + it(`should paginate through all records for a given ${test.type} operation using count`, async () => { + const pages = 'all'; + const count = 2; + let results = []; + let cursor = null; + do { + const response = test.type === 'query' + ? await tasks.query[test.input.index](test.input.key).go({cursor, count, pages}) + : await tasks.scan.go({cursor, count, pages}); + results = results.concat(response.data); + cursor = response.cursor; + } while (cursor !== null); + expect(() => Tasks.compareTasks( + results, + test.output, + )).to.not.throw; + }).timeout(50000); } - it("Paginate without overlapping values", async () => { + it("Paginate without overlapping values using limit", async () => { let limit = 30; let count = 0; let cursor = null; @@ -349,7 +367,32 @@ describe("Page", () => { expect(all).to.have.length(keys.size); }).timeout(10000); - it("Paginate without overlapping values with raw response", async () => { + it("Paginate without overlapping values using count", async () => { + let limit = 30; + let count = 0; + let cursor = null; + let all = []; + let keys = new Set(); + do { + count++; + let [next, items] = await tasks.query.assigned({employee: Tasks.employees[0]}) + .go({cursor, count: limit}) + .then(res => [res.cursor, res.data]); + if (next && count > 0) { + const deserialized = cursorFormatter.deserialize(next); + expect(deserialized).to.have.keys(["gsi2pk", "gsi2sk", "pk", "sk"]); + } + expect(items.length <= limit).to.be.true; + for (let item of items) { + keys.add(item.task + item.project + item.employee); + all.push(item); + } + cursor = next; + } while(cursor !== null); + expect(all).to.have.length(keys.size); + }).timeout(10000); + + it("Paginate without overlapping values with raw response using limit", async () => { let limit = 30; let count = 0; let cursor = null; @@ -375,7 +418,7 @@ describe("Page", () => { } while(cursor !== null); }).timeout(10000); - it("Paginate without overlapping values with pager='raw'", async () => { + it("Paginate without overlapping values with pager='raw' using limit", async () => { let limit = 30; let count = 0; let cursor = null; @@ -398,6 +441,29 @@ describe("Page", () => { } while(cursor !== null); }).timeout(10000); + it("Paginate without overlapping values with pager='raw' using count", async () => { + let limit = 30; + let count = 0; + let cursor = null; + let all = []; + + do { + count++; + let keys = new Set(); + let [next, items] = await tasks.query.projects({project: Tasks.projects[0]}).go({count: limit, cursor, pager: "raw"}).then(res => [res.cursor, res.data]); + if (next !== null && count > 1) { + expect(next).to.have.keys(["sk", "pk", "gsi1sk", "gsi1pk"]); + } + expect(items.length <= limit).to.be.true; + for (let item of items) { + keys.add(item.task + item.project + item.employee); + all.push(item); + } + expect(items.length).to.equal(keys.size); + cursor = next; + } while(cursor !== null); + }).timeout(10000); + // it("Should not accept incomplete page composite attributes", async () => { // let tests = [ // { @@ -444,7 +510,7 @@ describe("Page", () => { // }).timeout(10000); it("Should paginate and return raw results", async () => { - let results = await tasks.scan.go({raw: true}); + let results = await tasks.scan.go({ raw: true }); expect(results).to.have.keys(['cursor', 'data']); expect(results.data.Items).to.not.be.undefined expect(results.data.Items).to.be.an("array"); @@ -657,9 +723,54 @@ describe("Page", () => { } }); - it("entity query should only count entities belonging to the collection entities to fulfill 'limit' option requirements", async () => { + it("entity query should continue to query until 'count' option is reached", async () => { const ExclusiveStartKey = {key: 'hi'}; const [one, two, three, four, five, six] = tasks.data; + const {client, calls} = createClient({ + mockResponses: [ + { + Items: [one, two, three], + LastEvaluatedKey: ExclusiveStartKey, + }, + { + Items: [four, five, six], + LastEvaluatedKey: ExclusiveStartKey, + }, + { + Items: [], + LastEvaluatedKey: ExclusiveStartKey, + }, + { + Items: [], + LastEvaluatedKey: undefined, + }, + { + Items: [], + LastEvaluatedKey: ExclusiveStartKey, + } + ] + }); + const pages = 3; + const limit = 5; + const entity = new Tasks(TasksModel, {client, table}); + const results = await entity.query.task({task: "my_task"}).go({pages, count: limit}).then(res => res.data); + // this behaves differently + expect(results).to.be.an("array").with.length(limit); + expect(calls).to.have.length(2); + for (let i = 0; i < calls.length; i++) { + const call = calls[i]; + if (i === 0) { + expect(call.ExclusiveStartKey).to.be.undefined; + } else { + expect(call.ExclusiveStartKey.key).to.equal('hi'); + expect(call.ExclusiveStartKey.key === ExclusiveStartKey.key).to.be.true; + } + } + }); + + it("entity query should only count entities belonging to the collection entities to fulfill 'limit' option requirements", async () => { + const ExclusiveStartKey = { key: 'hi' }; + const [one, two, three, four, five, six] = tasks.data; const {client, calls} = createClient({ mockResponses: [ { @@ -701,6 +812,50 @@ describe("Page", () => { } }); + it("entity query should only count entities belonging to the collection entities to fulfill 'count' option requirements", async () => { + const ExclusiveStartKey = { key: 'hi' }; + const [one, two, three, four, five, six] = tasks.data; + const {client, calls} = createClient({ + mockResponses: [ + { + Items: [{}, {}, one, two, three, {}, {}], + LastEvaluatedKey: ExclusiveStartKey, + }, + { + Items: [four, five, six, {}], + LastEvaluatedKey: ExclusiveStartKey, + }, + { + Items: [], + LastEvaluatedKey: ExclusiveStartKey, + }, + { + Items: [], + LastEvaluatedKey: undefined, + }, + { + Items: [], + LastEvaluatedKey: ExclusiveStartKey, + } + ] + }); + const pages = 3; + const limit = 5; + const entity = new Tasks(TasksModel, {client, table}); + const results = await entity.query.task({task: "my_task"}).go({pages, count: limit}).then(res => res.data); + expect(results).to.be.an("array").with.length(limit); + expect(calls).to.have.length(2); + for (let i = 0; i < calls.length; i++) { + const call = calls[i]; + if (i === 0) { + expect(call.ExclusiveStartKey).to.be.undefined; + } else { + expect(call.ExclusiveStartKey.key).to.equal('hi'); + expect(call.ExclusiveStartKey.key === ExclusiveStartKey.key).to.be.true; + } + } + }); + it("collection query should continue to query until LastEvaluatedKey is not returned", async () => { const ExclusiveStartKey = {key: 'hi'}; const {client, calls} = createClient({ @@ -844,6 +999,58 @@ describe("Page", () => { } }); + it("collection query should continue to query until 'count' option is reached", async () => { + const ExclusiveStartKey = {key: 'hi'}; + const tasks = new Tasks(makeTasksModel(), {client, table}); + const tasks2 = new Tasks(makeTasksModel(), {client, table}); + + await tasks.load(10); + await tasks2.load(10); + const [one, two, three, four, five, six] = tasks.data; + const [seven, eight, nine, ten] = tasks2.data; + const created = createClient({ + mockResponses: [ + { + Items: [one, two, three, seven, eight, nine], + LastEvaluatedKey: ExclusiveStartKey + }, + { + Items: [four, five, six, ten], + LastEvaluatedKey: ExclusiveStartKey + }, + { + Items: [], + LastEvaluatedKey: ExclusiveStartKey + }, + { + Items: [], + LastEvaluatedKey: undefined + }, + { + Items: [], + LastEvaluatedKey: ExclusiveStartKey, + } + ] + }); + const service = new Service({tasks, tasks2}, {client: created.client, table}); + const pages = 3; + const limit = 9; + const employee = "my_employee"; + const results = await service.collections.assignments({employee}).go({pages, count: limit}).then(res => res.data); + expect(results.tasks).to.be.an("array").with.length(6); + expect(results.tasks2).to.be.an("array").with.length(3); + expect(created.calls).to.have.length(2); + for (let i = 0; i < created.calls.length; i++) { + const call = created.calls[i]; + if (i === 0) { + expect(call.ExclusiveStartKey).to.be.undefined; + } else { + expect(call.ExclusiveStartKey.key).to.equal('hi'); + expect(call.ExclusiveStartKey.key === ExclusiveStartKey.key).to.be.true; + } + } + }); + it("collection query should only count entities belonging to the collection entities to fulfill 'limit' option requirements", async () => { const ExclusiveStartKey = {key: 'hi'}; const tasks = new Tasks(makeTasksModel(), {client, table}); @@ -896,7 +1103,59 @@ describe("Page", () => { } }); - it("should automatically paginate all results with query", async () => { + it("collection query should only count entities belonging to the collection entities to fulfill 'count' option requirements", async () => { + const ExclusiveStartKey = {key: 'hi'}; + const tasks = new Tasks(makeTasksModel(), {client, table}); + const tasks2 = new Tasks(makeTasksModel(), {client, table}); + + await tasks.load(10); + await tasks2.load(10); + const [three, four, five, six] = tasks.data; + const [seven, eight, nine] = tasks2.data; + const created = createClient({ + mockResponses: [ + { + Items: [{}, {}, three, seven, eight, nine, {}, {}], + LastEvaluatedKey: ExclusiveStartKey + }, + { + Items: [four, five, six, {}], + LastEvaluatedKey: ExclusiveStartKey + }, + { + Items: [], + LastEvaluatedKey: ExclusiveStartKey + }, + { + Items: [], + LastEvaluatedKey: undefined + }, + { + Items: [], + LastEvaluatedKey: ExclusiveStartKey, + } + ] + }); + const service = new Service({tasks, tasks2}, {client: created.client, table}); + const pages = 3; + const limit = 6; + const employee = "my_employee"; + const results = await service.collections.assignments({employee}).go({pages, count: limit}).then(res => res.data); + expect(results.tasks).to.be.an("array").with.length(3); + expect(results.tasks2).to.be.an("array").with.length(3); + expect(created.calls).to.have.length(2); + for (let i = 0; i < created.calls.length; i++) { + const call = created.calls[i]; + if (i === 0) { + expect(call.ExclusiveStartKey).to.be.undefined; + } else { + expect(call.ExclusiveStartKey.key).to.equal('hi'); + expect(call.ExclusiveStartKey.key === ExclusiveStartKey.key).to.be.true; + } + } + }); + + it("should automatically paginate all results with query with 'limit'", async () => { const project = Tasks.projects[0]; const occurrences = tasks.occurrences.projects[project]; const overLimit = occurrences + 10; @@ -909,6 +1168,19 @@ describe("Page", () => { expect(limited).to.have.length(underLimit); }); + it("should automatically paginate all results with query with 'count'", async () => { + const project = Tasks.projects[0]; + const occurrences = tasks.occurrences.projects[project]; + const overLimit = occurrences + 10; + const underLimit = occurrences - 10; + const results = await tasks.query.projects({project}).go({count: overLimit}).then(res => res.data); + const limited = await tasks.query.projects({project}).go({count: underLimit}).then(res => res.data); + const loaded = tasks.filterLoaded({project}); + expect(() => Tasks.compareTasks(results, loaded)).to.not.throw; + expect(results).to.have.length(occurrences); + expect(limited).to.have.length(underLimit); + }); + // it("should automatically paginate all results with collection", async () => { // const employee = Tasks.employees[0]; // const limit1 = tasks.occurrences.employees[employee]; @@ -948,7 +1220,7 @@ describe("Page", () => { const occurrences = tasks.occurrences.employees[employee]; const pages = 2; const limit = Math.floor(occurrences / 4); - const results = await tasks.query.assigned({employee}).go({pages, params: {Limit: limit}}).then(res => res.data); + const results = await tasks.query.assigned({ employee }).go({ pages, params: { Limit: limit }}).then(res => res.data); expect(limit).to.be.greaterThan(0); expect(occurrences).to.be.greaterThan(limit * pages); expect(results).to.have.length(limit * pages); @@ -1069,7 +1341,7 @@ describe("Page", () => { it("should return the response received by options.parse if value is not array", async () => { let wasParsed = false; let parseArgs = {}; - const parserResponse = {value: true}; + const parserResponse = { value: 12345 }; const project = Tasks.projects[0]; const limit = 1; const results = await tasks.query @@ -1079,15 +1351,13 @@ describe("Page", () => { parse: (config, response) => { wasParsed = true; parseArgs = response; - return parserResponse; + return { data: parserResponse }; } }); expect(wasParsed).to.be.true; - expect(results.data === parserResponse).to.be.true; + expect(results.data).to.deep.equal(parserResponse); expect(parseArgs.Items).to.be.an("array"); expect(parseArgs.LastEvaluatedKey).to.have.keys("pk", "sk", "gsi1sk", "gsi1pk"); - expect(parseArgs.Count).to.equal(1); - expect(parseArgs.ScannedCount).to.equal(1); }); it("should not clobber a user defined ExclusiveStartKey", async () => { diff --git a/test/connected.service.spec.js b/test/connected.service.spec.js index cd9c3cfe..5c77592c 100644 --- a/test/connected.service.spec.js +++ b/test/connected.service.spec.js @@ -1,9 +1,8 @@ const sleep = async (ms) => new Promise((resolve) => setTimeout(resolve, ms)); process.env.AWS_NODEJS_CONNECTION_REUSE_ENABLED = 1; -const { Entity, clauses } = require("../src/entity"); +const { Entity } = require("../src/entity"); const { Service } = require("../src/service"); const { expect } = require("chai"); -const moment = require("moment"); const uuid = require("uuid").v4; const DynamoDB = require("aws-sdk/clients/dynamodb"); const table = "electro"; @@ -862,10 +861,11 @@ describe("Entities with custom identifiers and versions", () => { let collectionA = await service.collections .collectionA({prop1}) .where(({prop2}, {eq}) => eq(prop2, "prop2Value")) - .go({raw: true}) + .go({ raw: true }) .then(res => res.data) - .then((data) => ({success: true, data})) - .catch(err => ({success: false, err})); + .then((data) => ({ success: true, data })) + .catch(err => ({ success: false, err })); + expect(collectionA.success).to.be.true; expect(collectionA.data).to.be.deep.equal({ "Items": [ diff --git a/test/ts_connected.crud.spec.ts b/test/ts_connected.crud.spec.ts index 966936f3..d19d1069 100644 --- a/test/ts_connected.crud.spec.ts +++ b/test/ts_connected.crud.spec.ts @@ -4618,8 +4618,8 @@ describe('terminal methods', () => { }) }); -describe('query limit', () => { - it('adding a limit should not cause dropped items when paginating', async () => { +describe('query size', () => { + it('adding a size should not cause dropped items when paginating', async () => { const entity = new Entity({ model: { version: '1', @@ -4665,7 +4665,7 @@ describe('query limit', () => { description: uuid(), } } - const limit = 10; + const count = 10; const itemCount = 100; const items = new Array(itemCount) .fill({}) @@ -4678,7 +4678,7 @@ describe('query limit', () => { do { const response: QueryResponse = await entity.query .records({accountId}) - .go({ cursor, limit }); + .go({ cursor, count }); results = results.concat(response.data); cursor = response.cursor; iterations++;