Reduced package index fetch memory consumption

src/r2mm/manager/PackageDexieStore.ts

@@ -195,7 +195,7 @@ export async function resetCommunity(community: string) {
 }
 
 export async function upsertPackageListChunk(community: string, packageChunk: any[]) {
-    const newPackages: DexiePackage[] = packageChunk.map((pkg) => ({...pkg, community}));
+    const newPackages: DexiePackage[] = packageChunk.map((pkg) => Object.assign(pkg, {community}));
     const newSummaries: DexieSummary[] = packageChunk.map((pkg) => toSummary(community, pkg));
     await db.transaction('rw', db.packages, db.summaries, async () => {
         await db.packages.bulkPut(newPackages);

src/store/modules/TsModsModule.ts

@@ -23,7 +23,7 @@ interface State {
     activeGameCacheStatus: string|undefined;
     cache: Map<string, CachedMod>;
     deprecated: Map<string, boolean>;
-    exclusions: string[];
+    exclusions: Set<string>;
     isThunderstoreModListUpdateInProgress: boolean;
     mods: ThunderstoreMod[];
     modsLastUpdated?: Date;
@@ -63,7 +63,7 @@ export const TsModsModule = {
         cache: new Map<string, CachedMod>(),
         deprecated: new Map<string, boolean>(),
         /*** Packages available through API that should be ignored by the manager */
-        exclusions: [],
+        exclusions: new Set<string>(),
         /*** Mod list is updated from the API automatically and by user action */
         isThunderstoreModListUpdateInProgress: false,
         /*** All mods available through API for the current active game */
@@ -166,7 +166,7 @@ export const TsModsModule = {
         },
         setExclusions(state, payload: string|string[]) {
             const exclusions_ = Array.isArray(payload) ? payload : payload.split('\n');
-            state.exclusions = exclusions_.map((e) => e.trim()).filter(Boolean);
+            state.exclusions = new Set(exclusions_.map((e) => e.trim()).filter(Boolean));
         },
         setThunderstoreModListUpdateError(state, error: Error) {
             state.thunderstoreModListUpdateError = error instanceof Error ? error : new Error(error);
@@ -261,18 +261,21 @@ export const TsModsModule = {
             const packageIndexUrl = transformPackageUrl(rootState.activeGame.thunderstoreUrl);
             const indexUrl = CdnProvider.addCdnQueryParameter(packageIndexUrl);
             const options = {attempts: 5, interval: 2000, throwLastErrorAsIs: true};
-            const index = await retry(() => fetchAndProcessBlobFile(indexUrl), options);
+            const index = await retry(() => fetchAndProcessBlobFile(indexUrl, {computeHash: true}), options);
 
             if (!isStringArray(index.content)) {
                 throw new Error('Received invalid chunk index from API');
             }
             if (isEmptyArray(index.content)) {
                 throw new Error('Received empty chunk index from API');
             }
+            if (typeof index.hash !== 'string') {
+                throw new Error('Failed to compute hash for the chunk index');
+            }
 
             const community = rootState.activeGame.internalFolderName;
             const isLatest = await PackageDb.isLatestPackageListIndex(community, index.hash);
-            return {...index, isLatest};
+            return {content: index.content, hash: index.hash, isLatest};
         },
 
         async fetchAndCachePackageListChunks(
@@ -324,7 +327,7 @@ export const TsModsModule = {
                 throw new Error(`Received invalid chunk from URL "${url}"`);
             }
 
-            const filtered = chunk.filter((pkg) => !state.exclusions.includes(pkg.full_name));
+            const filtered = chunk.filter((pkg) => !state.exclusions.has(pkg.full_name));
             const community = rootState.activeGame.internalFolderName;
             await PackageDb.upsertPackageListChunk(community, filtered);
             return filtered.map((pkg) => pkg.full_name);

src/utils/GzipUtils.ts

@@ -1,7 +1,7 @@
 // Use third party library as Node's zlib is not available on TSMM.
 import { AsyncGunzip } from "fflate";
 
-export function decompressArrayBuffer(compressed: Buffer, encoding = 'utf-8'): Promise<string> {
+export function decompressArrayBuffer(compressed: Uint8Array, encoding = 'utf-8'): Promise<string> {
     return new Promise((resolve, reject) => {
         const gunzip = new AsyncGunzip((err, result) => {
             if (err) {
@@ -13,6 +13,6 @@ export function decompressArrayBuffer(compressed: Buffer, encoding = 'utf-8'): P
             return resolve(decodedString);
         });
 
-        gunzip.push(new Uint8Array(compressed), true);
+        gunzip.push(compressed, true);
     });
 }

src/utils/HttpUtils.ts

@@ -1,7 +1,6 @@
 import axios, { AxiosRequestConfig } from "axios";
 
 import { decompressArrayBuffer } from "./GzipUtils";
-import Buffer from "../providers/node/buffer/buffer";
 
 const newAbortSignal = (timeoutMs: number) => {
     const abortController = new AbortController();
@@ -104,24 +103,26 @@ export const makeLongRunningGetRequest = async (
     }
 }
 
+interface ProcessBlobOptions {
+    computeHash?: boolean;
+}
+
 /**
  * Download blob files containing gzip compressed JSON strings and
  * return them as objects. This is used for data that's shared by
  * all users and can be cached heavily on CDN level.
  */
-export const fetchAndProcessBlobFile = async (url: string) => {
-    const dateFetched = new Date();
+export const fetchAndProcessBlobFile = async (url: string, options: ProcessBlobOptions = {}) => {
     const response = await makeLongRunningGetRequest(url, {axiosConfig: {responseType: 'arraybuffer'}});
-    const buffer = Buffer.from(response.data);
-    const hash = await getSha256Hash(buffer);
-    const jsonString = await decompressArrayBuffer(buffer);
+    const compressed = new Uint8Array(response.data);
+    const hash = options.computeHash ? await getSha256Hash(compressed) : undefined;
+    const jsonString = await decompressArrayBuffer(compressed);
     const content = JSON.parse(jsonString);
-    return {content, hash, dateFetched};
+    return {content, hash};
 }
 
-async function getSha256Hash(buffer: Buffer): Promise<string> {
-    const arrayBuffer = new Uint8Array(buffer);
-    const hashBuffer = await crypto.subtle.digest('SHA-256', arrayBuffer);
+async function getSha256Hash(data: Uint8Array): Promise<string> {
+    const hashBuffer = await crypto.subtle.digest('SHA-256', data as BufferSource);
     const hashByteArray = Array.from(new Uint8Array(hashBuffer));
     const hexHash = hashByteArray.map(b => b.toString(16).padStart(2, '0')).join('');
     return hexHash;