From 5bbd5827b85d2a0dd4bea0904d818b136b2011c2 Mon Sep 17 00:00:00 2001 From: Seung Park Date: Mon, 10 Feb 2025 11:49:00 -0500 Subject: [PATCH 1/6] update persistence module to save facets --- modules/persistence/src/services/pages/index.ts | 9 +++++++++ modules/persistence/tests/services/pages.test.ts | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/modules/persistence/src/services/pages/index.ts b/modules/persistence/src/services/pages/index.ts index 81dd98d5d..a3ca50d98 100644 --- a/modules/persistence/src/services/pages/index.ts +++ b/modules/persistence/src/services/pages/index.ts @@ -17,12 +17,20 @@ interface PageAst { [key: string]: any; } +interface Facet { + category: string; + value: string; + display_name: string; + sub_facets?: { [key: string]: any }[]; +} + export interface Page { page_id: string; filename: string; ast: PageAst; static_assets: UpdatedAsset[]; github_username: string; + facets?: Facet[]; } export interface UpdatedPage extends Page { @@ -162,6 +170,7 @@ class UpdatedPagesManager { deleted: false, // Track the last build ID to update the content build_id: this.buildId, + facets: page.facets, }, $setOnInsert: { created_at: this.updateTime, diff --git a/modules/persistence/tests/services/pages.test.ts b/modules/persistence/tests/services/pages.test.ts index 03b059f2f..05dc5a64c 100644 --- a/modules/persistence/tests/services/pages.test.ts +++ b/modules/persistence/tests/services/pages.test.ts @@ -113,6 +113,16 @@ describe('pages module', () => { page_id: `${pagePrefix}/page1.txt`, filename: 'page1.txt', ast: { foo: 'foo', bar: { foo: 'baz' } }, + facets: [ + { + category: 'target_product', + value: 'atlas', + display_name: 'Atlas', + sub_facets: [ + { category: 'sub_product', value: 'kubernetes-operator', display_name: 'Kubernetes Operator' }, + ], + }, + ], static_assets: [], github_username: GH_USER, }, @@ -123,6 +133,7 @@ describe('pages module', () => { expect(res).toHaveLength(2); const updatedPage = res.find(({ filename }) => filename === 'page1.txt'); expect(updatedPage).toHaveProperty('ast.bar.foo', 'baz'); + expect(updatedPage).toHaveProperty('facets[0].category'); // Page documents should have different timestamps to denote different update times expect(res[0].updated_at !== res[1].updated_at).toBeTruthy(); }); From b410f24d4a0be0eeb59fa96cdebc75e4b381d6d1 Mon Sep 17 00:00:00 2001 From: Seung Park Date: Mon, 10 Feb 2025 11:57:24 -0500 Subject: [PATCH 2/6] update server version --- modules/persistence/jest-mongodb-config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/persistence/jest-mongodb-config.js b/modules/persistence/jest-mongodb-config.js index 87aa4fa4e..52a0ee683 100644 --- a/modules/persistence/jest-mongodb-config.js +++ b/modules/persistence/jest-mongodb-config.js @@ -1,7 +1,7 @@ module.exports = { mongodbMemoryServerOptions: { binary: { - version: '4.0.3', + version: '5.0.x', skipMD5: true, }, autoStart: false, From a8e1376367d505255301cadecbc865f953b7106a Mon Sep 17 00:00:00 2001 From: Seung Park Date: Mon, 10 Feb 2025 12:44:09 -0500 Subject: [PATCH 3/6] update server version --- modules/persistence/jest-mongodb-config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/persistence/jest-mongodb-config.js b/modules/persistence/jest-mongodb-config.js index 52a0ee683..b43047e48 100644 --- a/modules/persistence/jest-mongodb-config.js +++ b/modules/persistence/jest-mongodb-config.js @@ -1,7 +1,7 @@ module.exports = { mongodbMemoryServerOptions: { binary: { - version: '5.0.x', + version: '5.0.0', skipMD5: true, }, autoStart: false, From dd505084cc8bdd2e2bb511b99a9d2edc6422b32c Mon Sep 17 00:00:00 2001 From: Seung Park Date: Mon, 10 Feb 2025 12:48:30 -0500 Subject: [PATCH 4/6] update runner - lock in hardware version --- .github/workflows/test-persistence.yml | 4 ++-- modules/persistence/jest-mongodb-config.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-persistence.yml b/.github/workflows/test-persistence.yml index e8514efed..777f711c8 100644 --- a/.github/workflows/test-persistence.yml +++ b/.github/workflows/test-persistence.yml @@ -11,7 +11,7 @@ on: jobs: # Checks for changes to the modules/persistence directory changes: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 outputs: persistence: ${{ steps.filter.outputs.persistence }} steps: @@ -29,7 +29,7 @@ jobs: defaults: run: working-directory: 'modules/persistence' - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 - uses: actions/setup-node@v1 diff --git a/modules/persistence/jest-mongodb-config.js b/modules/persistence/jest-mongodb-config.js index b43047e48..87aa4fa4e 100644 --- a/modules/persistence/jest-mongodb-config.js +++ b/modules/persistence/jest-mongodb-config.js @@ -1,7 +1,7 @@ module.exports = { mongodbMemoryServerOptions: { binary: { - version: '5.0.0', + version: '4.0.3', skipMD5: true, }, autoStart: false, From a950aadd8ee3c8bf3b81d2addeca904486bcede3 Mon Sep 17 00:00:00 2001 From: Seung Park Date: Tue, 11 Feb 2025 18:39:36 -0500 Subject: [PATCH 5/6] update to react on facet change only. add tests --- .../persistence/src/services/pages/index.ts | 5 +- .../persistence/tests/services/pages.test.ts | 54 +++++++++++++++---- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/modules/persistence/src/services/pages/index.ts b/modules/persistence/src/services/pages/index.ts index a3ca50d98..bc1a54be3 100644 --- a/modules/persistence/src/services/pages/index.ts +++ b/modules/persistence/src/services/pages/index.ts @@ -43,6 +43,7 @@ interface PreviousPageMapping { [key: string]: { ast: PageAst; static_assets: StaticAsset[]; + facets?: Facet[]; }; } @@ -83,6 +84,7 @@ const findPrevPageDocs = async (pageIdPrefix: string, collection: string, github page_id: 1, ast: 1, static_assets: 1, + facets: 1, }; try { @@ -104,6 +106,7 @@ const createPageAstMapping = async (docsCursor: FindCursor) => { mapping[doc.page_id] = { ast: doc.ast, static_assets: doc.static_assets, + facets: doc.facets, }; pageIds.add(doc.page_id); } @@ -156,7 +159,7 @@ class UpdatedPagesManager { // Update the document if page's current AST is different from previous build's. // New pages should always count as having a "different" AST - if (!isEqual(page.ast, prevPageData?.ast)) { + if (!isEqual(page.ast, prevPageData?.ast) || !isEqual(page.facets, prevPageData?.facets)) { const operation = { updateOne: { filter: { page_id: currentPageId, github_username: page.github_username }, diff --git a/modules/persistence/tests/services/pages.test.ts b/modules/persistence/tests/services/pages.test.ts index 05dc5a64c..2bf6ca20e 100644 --- a/modules/persistence/tests/services/pages.test.ts +++ b/modules/persistence/tests/services/pages.test.ts @@ -113,16 +113,6 @@ describe('pages module', () => { page_id: `${pagePrefix}/page1.txt`, filename: 'page1.txt', ast: { foo: 'foo', bar: { foo: 'baz' } }, - facets: [ - { - category: 'target_product', - value: 'atlas', - display_name: 'Atlas', - sub_facets: [ - { category: 'sub_product', value: 'kubernetes-operator', display_name: 'Kubernetes Operator' }, - ], - }, - ], static_assets: [], github_username: GH_USER, }, @@ -133,11 +123,53 @@ describe('pages module', () => { expect(res).toHaveLength(2); const updatedPage = res.find(({ filename }) => filename === 'page1.txt'); expect(updatedPage).toHaveProperty('ast.bar.foo', 'baz'); - expect(updatedPage).toHaveProperty('facets[0].category'); // Page documents should have different timestamps to denote different update times expect(res[0].updated_at !== res[1].updated_at).toBeTruthy(); }); + it('should update pages with modified facets', async () => { + const pagePrefix = generatePagePrefix(); + const pages = generatePages(pagePrefix); + await _updatePages(pages, collection, GH_USER, new ObjectID()); + // Applying facet updates to both pages + const updatedPages = generatePages(pagePrefix); + updatedPages[0].facets = [ + { + category: 'target_product', + value: 'atlas', + display_name: 'Atlas', + }, + ]; + updatedPages[1].facets = [ + { + category: 'target_product', + value: 'atlas', + display_name: 'Atlas', + sub_facets: [{ category: 'sub_product', value: 'kubernetes-operator', display_name: 'Kubernetes Operator' }], + }, + ]; + + const findQuery = { + page_id: { $regex: new RegExp(`^${pagePrefix}`) }, + }; + await _updatePages(updatedPages, collection, GH_USER, new ObjectID()); + const res = await mockDb.collection(collection).find(findQuery).toArray(); + expect(res).toHaveLength(2); + expect(res.every(({ facets }) => facets && facets.length)).toBeTruthy(); + expect(res[1]).toHaveProperty('facets[0].sub_facets[0].value'); + + // removal of facet also propagates to update + const removedFacetUpdates = generatePages(pagePrefix).slice(0, 1); + delete removedFacetUpdates[0].facets; + + await _updatePages(removedFacetUpdates, collection, GH_USER, new ObjectID()); + const findRemovalQuery = { + page_id: { $regex: new RegExp(`^${pagePrefix}/page0.txt`) }, + }; + const removeFacetRes = await mockDb.collection(collection).find(findRemovalQuery).toArray(); + expect(removeFacetRes[0].facets).toBeNull(); + }); + it('should mark pages for deletion', async () => { const pagePrefix = generatePagePrefix(); const pages = generatePages(pagePrefix); From cd7ba034af4c273904a20bbe0af4f047e4fef7b0 Mon Sep 17 00:00:00 2001 From: Seung Park Date: Wed, 12 Feb 2025 10:48:25 -0500 Subject: [PATCH 6/6] add comment for locked ubuntu version --- .github/workflows/test-persistence.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-persistence.yml b/.github/workflows/test-persistence.yml index 777f711c8..582df4fe2 100644 --- a/.github/workflows/test-persistence.yml +++ b/.github/workflows/test-persistence.yml @@ -11,6 +11,7 @@ on: jobs: # Checks for changes to the modules/persistence directory changes: + # locking to ubuntu 22.04 to allow mongodb-memory-server to run runs-on: ubuntu-22.04 outputs: persistence: ${{ steps.filter.outputs.persistence }} @@ -29,6 +30,7 @@ jobs: defaults: run: working-directory: 'modules/persistence' + # locking to ubuntu 22.04 to allow mongodb-memory-server to run runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3