[♻] {PROD4POD-1931} Move zipfile mock-up and improve coverage (#1205)

JJ · web-flow · commit a580e09494fb · 2022-10-14T13:18:51.000+02:00
# ✍️ Description The mock for zip-files is more useful in the common `poly-import` module. That way it can also be used in the module itself to mock zipfiles, and imported from any module that needs to mock zips. While we were at it, boost a bit coverage of said test. ### 🏗️ Fixes PROD4POD-1931 It is a precondition for #1204, since `Storage` needs zip files to be tested. ## ℹ️ Other information Some minor refactoring also made. Might include some refactoring of tests, mainly to reduce boilerplate. Not for the time being. ### Additional functionality * ✅ adds tests in parts that were not covered, for instance, file name analysis in Facebook
diff --git a/feature-utils/poly-import/src/index.js b/feature-utils/poly-import/src/index.js
@@ -14,16 +14,27 @@ import {
 import DataAccount from "./entities/data-account";
 import { Status, statusTypes } from "../utils/status";
 import { Telemetry } from "../utils/performance-telemetry";
+import {
+    MINIMUM_FILE_SIZE,
+    ZipFileEntryMock,
+    ZipFileMock,
+} from "../utils/zipfile-mock";
+import { jsonStringifyWithUtfEscape } from "../utils/json-encoding";
+
 export {
+    MINIMUM_FILE_SIZE,
+    jsonStringifyWithUtfEscape,
     Status,
     Importer,
     runImporter,
     runImporters,
     runOutdatedImporter,
     runOutdatedImporters,
     ZipFile,
+    ZipFileMock,
     Telemetry,
     ZipFileEntry,
+    ZipFileEntryMock,
     FileImportError,
     RefreshFilesError,
     FileSelectionError,
diff --git a/feature-utils/poly-import/test/utf8-encoding-decoding.test.js b/feature-utils/poly-import/test/utf8-encoding-decoding.test.js
@@ -1,6 +1,6 @@
 "use strict";
 
-import { jsonStringifyWithUtfEscape } from "../src/model/importers/utils/json-encoding";
+import { jsonStringifyWithUtfEscape } from "../utils/json-encoding";
 
 const jsonData1 = "🦊";
 const jsonData2 = "🦊🤗😍";
@@ -22,6 +22,11 @@ describe("JSON encode", () => {
     it("Various characters", () => {
         expect(jsonStringifyWithUtfEscape(jsonData3)).toBe(decodedData3);
     });
+
+    it("passes through non-string data", () => {
+        const foo42 = { foo: 42 };
+        expect(jsonStringifyWithUtfEscape(foo42)).toBe(JSON.stringify(foo42));
+    });
 });
 
 describe("Decodes", () => {
diff --git a/feature-utils/poly-import/utils/json-encoding.js b/feature-utils/poly-import/utils/json-encoding.js
@@ -24,8 +24,7 @@ function jsonStringifyWithUtfEscape(jsonData) {
         }
         return value;
     });
-    const escapedString = jsonStringEscapeUtfCharacters(serializedString);
-    return escapedString;
+    return jsonStringEscapeUtfCharacters(serializedString);
 }
 
 export { jsonStringEscapeUtfCharacters, jsonStringifyWithUtfEscape };
diff --git a/feature-utils/poly-import/utils/zipfile-mock.js b/feature-utils/poly-import/utils/zipfile-mock.js
@@ -1,4 +1,4 @@
-import { jsonStringifyWithUtfEscape } from "../../src/model/importers/utils/json-encoding";
+import { jsonStringifyWithUtfEscape } from "./json-encoding";
 
 //The minimum size of a .ZIP file is 22 bytes
 export const MINIMUM_FILE_SIZE = 22;
diff --git a/features/facebookImport/test/datasets/messages-data.js b/features/facebookImport/test/datasets/messages-data.js
@@ -1,4 +1,4 @@
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { createMockedZip } from "../utils/data-creation";
 export const DATASET_EXPECTED_VALUES = {
     numberOfMessageThreads: 2,
diff --git a/features/facebookImport/test/datasets/posts-data.js b/features/facebookImport/test/datasets/posts-data.js
@@ -1,4 +1,4 @@
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { createMockedZip } from "../utils/data-creation";
 
 const POSTS_FILE_PATH = "posts/your_posts_X.json";
diff --git a/features/facebookImport/test/importers/ad-interests-importer.test.js b/features/facebookImport/test/importers/ad-interests-importer.test.js
@@ -5,7 +5,7 @@ import {
     DATASET_EXPECTED_VALUES,
     zipFileWithAdInterests,
 } from "../datasets/ad-interests-data";
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { zipWithWrongDatasetKey } from "../utils/data-creation";
 import { runAdInterestsImporter } from "../utils/data-importing";
 import {
diff --git a/features/facebookImport/test/importers/ad-views-importer.test.js b/features/facebookImport/test/importers/ad-views-importer.test.js
@@ -9,7 +9,7 @@ import {
     createEnglishDatasetWithMissingAdsCategory,
     createIncompleteEnglishAdViewsData,
 } from "../datasets/ad-views-data";
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { zipWithWrongDatasetKey } from "../utils/data-creation";
 import {
     runAdsImportForDataset,
diff --git a/features/facebookImport/test/importers/comments-importer.test.js b/features/facebookImport/test/importers/comments-importer.test.js
@@ -5,7 +5,7 @@ import {
     DATASET_EXPECTED_VALUES,
     zipFileWithComments,
 } from "../datasets/comments-data";
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { zipWithWrongDatasetKey } from "../utils/data-creation";
 import { runCommentsImporter } from "../utils/data-importing";
 import {
diff --git a/features/facebookImport/test/importers/connected-advertisers-all-types-importer.test.js b/features/facebookImport/test/importers/connected-advertisers-all-types-importer.test.js
@@ -6,7 +6,7 @@ import {
     DATASET_EXPECTED_VALUES,
     zipFileWithConnectedAdvertisersAllTypes,
 } from "../datasets/connected-advertisers-all-types-data";
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { zipWithWrongDatasetKey } from "../utils/data-creation";
 import { runConnectedAdvertisersWithAllTypesImporter } from "../utils/data-importing";
 import {
diff --git a/features/facebookImport/test/importers/connected-advertisers-importer.test.js b/features/facebookImport/test/importers/connected-advertisers-importer.test.js
@@ -6,7 +6,7 @@ import {
     DATASET_EXPECTED_VALUES,
     zipFileWithConnectedAdvertisers,
 } from "../datasets/connected-advertisers-data";
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { zipWithWrongDatasetKey } from "../utils/data-creation";
 import { runConnectedAdvertisersImporter } from "../utils/data-importing";
 import {
diff --git a/features/facebookImport/test/importers/interacted-with-advertisers-importer.test.js b/features/facebookImport/test/importers/interacted-with-advertisers-importer.test.js
@@ -5,7 +5,7 @@ import {
     DATASET_EXPECTED_VALUES,
     zipFileWithInteractedWithAdvertisers,
 } from "../datasets/interacted-with-advertisers-data";
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { zipWithWrongDatasetKey } from "../utils/data-creation";
 import { runInteractedWithAdvertisersImporter } from "../utils/data-importing";
 import {
diff --git a/features/facebookImport/test/importers/language-and-locale-importer.test.js b/features/facebookImport/test/importers/language-and-locale-importer.test.js
@@ -5,7 +5,7 @@ import {
     createLanguageSettingsData,
     createLocaleData,
 } from "../datasets/language-and-locale-data";
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { runLanguageAndLocaleImporter } from "../utils/data-importing";
 import {
     expectImportSuccess,
diff --git a/features/facebookImport/test/importers/messages-importer.test.js b/features/facebookImport/test/importers/messages-importer.test.js
@@ -10,7 +10,7 @@ import {
     zipFileWithMessageThreads,
     zipFileWithThreeFileErrors,
 } from "../datasets/messages-data.js";
-import { ZipFileMock } from "../mocks/zipfile-mock.js";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { runMessagesImporter } from "../utils/data-importing.js";
 import {
     expectError,
diff --git a/features/facebookImport/test/importers/name-importer.test.js b/features/facebookImport/test/importers/name-importer.test.js
@@ -1,7 +1,7 @@
 "use strict";
 
 import NameImporter from "../../src/model/importers/name-importer";
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { runNameImporter } from "../utils/data-importing";
 import {
     expectError,
diff --git a/features/facebookImport/test/importers/off-facebook-events-importer.test.js b/features/facebookImport/test/importers/off-facebook-events-importer.test.js
@@ -7,7 +7,7 @@ import {
     zipFileWithOffFacebookEvents,
     DATASET_EXPECTED_VALUES,
 } from "../datasets/off-facebook-events-data";
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { zipWithWrongDatasetKey } from "../utils/data-creation";
 import { runOffFacebookEventsImporter } from "../utils/data-importing";
 import {
diff --git a/features/facebookImport/test/importers/personal-data-importer.test.js b/features/facebookImport/test/importers/personal-data-importer.test.js
@@ -1,7 +1,7 @@
 import PersonalDataImporter, {
     PROFILE_INFORMATION_FILE_PATH,
 } from "../../src/model/importers/personal-data-importer";
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { runPersonalDataImporter } from "../utils/data-importing";
 import {
     expectError,
diff --git a/features/facebookImport/test/importers/post-reactions-importer.test.js b/features/facebookImport/test/importers/post-reactions-importer.test.js
@@ -5,7 +5,7 @@ import {
     DATASET_EXPECTED_VALUES,
     zipFileWithPostReactions,
 } from "../datasets/post-reactions-data";
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { zipWithWrongDatasetKey } from "../utils/data-creation";
 import { runPostReactionsImporter } from "../utils/data-importing";
 import {
diff --git a/features/facebookImport/test/importers/posts-importer.test.js b/features/facebookImport/test/importers/posts-importer.test.js
@@ -11,7 +11,7 @@ import {
     zipFileWithTwoFileErrors,
     zipFileWithTwoPostsFiles,
 } from "../datasets/posts-data";
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { runPostsImporter } from "../utils/data-importing";
 import {
     expectError,
diff --git a/features/facebookImport/test/importers/searches-importer.test.js b/features/facebookImport/test/importers/searches-importer.test.js
@@ -5,7 +5,7 @@ import {
     DATASET_EXPECTED_VALUES,
     zipFileWithSearches,
 } from "../datasets/searches-data";
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { zipWithWrongDatasetKey } from "../utils/data-creation";
 import { runSearchesImporter } from "../utils/data-importing";
 import {
diff --git a/features/facebookImport/test/importers/test-definition/importer-tests-definition.js b/features/facebookImport/test/importers/test-definition/importer-tests-definition.js
@@ -1,4 +1,4 @@
-import { ZipFileMock } from "../../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { zipWithWrongDatasetKey } from "../../utils/data-creation";
 import { runSingleOutdatedImporter } from "../../utils/data-importing";
 import {
diff --git a/features/facebookImport/test/ministory/about-pictures-data-analysis.test.js b/features/facebookImport/test/ministory/about-pictures-data-analysis.test.js
@@ -1,6 +1,6 @@
 import AboutPicturesDataAnalysis from "../../src/model/analyses/ministories/about-pictures-data-analysis";
 import AboutPicturesMinistory from "../../src/views/ministories/aboutPictures";
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { runAnalysisForExport } from "../utils/analyses-execution";
 import {
     expectInactiveAnalysis,
diff --git a/features/facebookImport/test/ministory/json-file-names-analysis.test.js b/features/facebookImport/test/ministory/json-file-names-analysis.test.js
@@ -0,0 +1,72 @@
+import JSONFileNamesAnalysis from "../../src/model/analyses/report/json-file-names-analysis";
+import commonStructure from "../../src/static/commonStructure";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
+import { runAnalysisForExport } from "../utils/analyses-execution";
+import { expectAnalysisSuccessStatus } from "../utils/analysis-assertions";
+
+const commonJsonFiles = commonStructure
+    .filter((path) => path.match(/\.json$/))
+    .map((jsonPath) => {
+        return jsonPath.substring(1);
+    });
+
+async function analyzeZipWithFiles(files) {
+    const zipFile = new ZipFileMock();
+    if (files.length > 0) {
+        files.forEach((jsonPath) => {
+            zipFile.addJsonEntry(jsonPath, { foo: "bar" });
+        });
+    }
+    const { analysisResult } = await runAnalysisForExport(
+        JSONFileNamesAnalysis,
+        zipFile
+    );
+    return analysisResult;
+}
+
+describe("JSON files analysis for empty zip", () => {
+    let status;
+    beforeAll(async () => {
+        ({ status } = await analyzeZipWithFiles([]));
+    });
+    it("has success status", async () => {
+        expectAnalysisSuccessStatus(status);
+    });
+});
+
+describe("JSON files analysis for non-empty zip", () => {
+    let status;
+    let analysis;
+
+    beforeAll(async () => {
+        ({ status, analysis } = await analyzeZipWithFiles(commonJsonFiles));
+    });
+
+    it("reports successful status", () => {
+        expect(status.isSuccess).toBe(true);
+    });
+
+    it("has the right name", () => {
+        expect(analysis.reportData).toStrictEqual(commonJsonFiles);
+    });
+
+    it("has the right title", () => {
+        expect(analysis.title).toBe("JSON file names");
+    });
+});
+
+describe("JSON files analysis with anonymized path", () => {
+    let analysis;
+    beforeAll(async () => {
+        commonJsonFiles.push(
+            "messages/archived_threads/facebookuser_gktxomrifg/message_1.json"
+        );
+        ({ analysis } = await analyzeZipWithFiles(commonJsonFiles));
+    });
+
+    it("has anonymized file", () => {
+        expect(analysis.reportData).toContain(
+            "messages/archived_threads/uniqueid_hash/message_1.json"
+        );
+    });
+});
diff --git a/features/facebookImport/test/ministory/messages-analysis.test.js b/features/facebookImport/test/ministory/messages-analysis.test.js
@@ -4,7 +4,7 @@ import {
     DATASET_EXPECTED_VALUES,
     zipFileWithMessageThreads,
 } from "../datasets/messages-data";
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { runAnalysisForExport } from "../utils/analyses-execution";
 import {
     expectActiveAnalysis,
diff --git a/features/facebookImport/test/ministory/missing-common-json-files.test.js b/features/facebookImport/test/ministory/missing-common-json-files.test.js
@@ -1,7 +1,7 @@
 import MissingCommonJSONFilesAnalysis from "../../src/model/analyses/report/missing-common-json-files";
 import commonStructure from "../../src/static/commonStructure";
 import MissingCommonJSONFilesReport from "../../src/views/ministories/missingCommonJsonFiles";
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { runAnalysisForExport } from "../utils/analyses-execution";
 import {
     expectActiveAnalysis,
diff --git a/features/facebookImport/test/ministory/post-reactions-types-analysis.test.js b/features/facebookImport/test/ministory/post-reactions-types-analysis.test.js
@@ -1,7 +1,7 @@
 import PostReactionsTypesAnalysis from "../../src/model/analyses/ministories/post-reactions-types-analysis";
 import PostReactionTypesMinistory from "../../src/views/ministories/postReactionsTypes";
 import { zipFileWithPostReactions } from "../datasets/post-reactions-data";
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { runAnalysisForExport } from "../utils/analyses-execution";
 import {
     expectActiveAnalysis,
diff --git a/features/facebookImport/test/ministory/report-metadata.test.js b/features/facebookImport/test/ministory/report-metadata.test.js
@@ -7,7 +7,7 @@ import ReportMetadataReport from "../../src/views/ministories/reportMetadata";
 import { createInteractedWithAdvertisersDataset } from "../datasets/interacted-with-advertisers-data";
 import { createLanguageSettingsData } from "../datasets/language-and-locale-data";
 import { createOffFacebookEventsSimpleData } from "../datasets/off-facebook-events-data";
-import { MINIMUM_FILE_SIZE } from "../mocks/zipfile-mock";
+import { MINIMUM_FILE_SIZE } from "@polypoly-eu/poly-import";
 import { runAnalysisForExport } from "../utils/analyses-execution";
 import {
     expectActiveAnalysis,
diff --git a/features/facebookImport/test/utils/data-creation.js b/features/facebookImport/test/utils/data-creation.js
@@ -1,4 +1,4 @@
-import { ZipFileMock } from "../mocks/zipfile-mock";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 
 export function createMockedZip(datasets) {
     let zipFile = new ZipFileMock();
diff --git a/features/facebookImport/test/utils/data-importing.js b/features/facebookImport/test/utils/data-importing.js
@@ -12,7 +12,7 @@ import RecentlyViewedAdsImporter, {
     RECENTLY_VIEWED_FILE_PATH,
 } from "../../src/model/importers/recently-viewed-ads-importer.js";
 import OffFacebookEventsImporter from "../../src/model/importers/off-facebook-events-importer.js";
-import { ZipFileMock } from "../mocks/zipfile-mock.js";
+import { ZipFileMock } from "@polypoly-eu/poly-import";
 import { MockPod } from "@polypoly-eu/api/dist/mock-pod";
 import LanguageAndLocaleImporter from "../../src/model/importers/language-and-locale-importer.js";
 import FriendsImporter from "../../src/model/importers/friends-importer.js";

Original file line number	Diff line number	Diff line change
`@@ -24,8 +24,7 @@ function jsonStringifyWithUtfEscape(jsonData) {`
`24`	`24`	`}`
`25`	`25`	`return value;`
`26`	`26`	`});`
`27`		`- const escapedString = jsonStringEscapeUtfCharacters(serializedString);`
`28`		`- return escapedString;`
	`27`	`+ return jsonStringEscapeUtfCharacters(serializedString);`
`29`	`28`	`}`
`30`	`29`
`31`	`30`	`export { jsonStringEscapeUtfCharacters, jsonStringifyWithUtfEscape };`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-import { jsonStringifyWithUtfEscape } from "../../src/model/importers/utils/json-encoding";`
	`1`	`+import { jsonStringifyWithUtfEscape } from "./json-encoding";`
`2`	`2`
`3`	`3`	`//The minimum size of a .ZIP file is 22 bytes`
`4`	`4`	`export const MINIMUM_FILE_SIZE = 22;`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-import { ZipFileMock } from "../mocks/zipfile-mock";`
	`1`	`+import { ZipFileMock } from "@polypoly-eu/poly-import";`
`2`	`2`	`import { createMockedZip } from "../utils/data-creation";`
`3`	`3`	`export const DATASET_EXPECTED_VALUES = {`
`4`	`4`	`numberOfMessageThreads: 2,`