Add support for > 2GB download and extraction

package.json

@@ -44,6 +44,7 @@
     "dot-prop": "^5.2.0",
     "electron-updater": "4.2.5",
     "elliptic": "^6.5.4",
+    "extract-zip": "^2.0.1",
     "fflate": "^0.8.2",
     "floating-vue": "5.2.2",
     "fs-extra": "^8.1.0",

src-electron/ipc/init-ipc.ts

@@ -2,6 +2,7 @@ import { BrowserWindow } from 'electron';
 import { hookPathIpc } from './node-path-impl';
 import { hookChildProcessIpc } from './node-child-process-impl';
 import { hookFsIpc } from 'app/src-electron/ipc/node-fs-impl';
+import { hookNetIpc } from 'app/src-electron/ipc/node-net-impl';
 import { hookZipIpc } from 'app/src-electron/ipc/zip-hook';
 import { hookElectronIpc } from 'app/src-electron/ipc/electron-hook';
 import {hookOsIpc} from "app/src-electron/ipc/node-os-impl";
@@ -10,6 +11,7 @@ export function hookIpc(browserWindow: BrowserWindow) {
     hookPathIpc(browserWindow);
     hookChildProcessIpc(browserWindow);
     hookFsIpc(browserWindow);
+    hookNetIpc(browserWindow);
     hookOsIpc(browserWindow);
     hookZipIpc(browserWindow);
     hookElectronIpc(browserWindow);

src-electron/ipc/node-net-impl.ts

@@ -0,0 +1,69 @@
+import { BrowserWindow, ipcMain, net } from 'electron';
+import fs from 'fs';
+import path from 'path';
+import { Readable } from 'stream';
+
+// Throttle progress events so a multi-GB download doesn't flood IPC.
+const PROGRESS_INTERVAL_MS = 200;
+
+export function hookNetIpc(browserWindow: BrowserWindow) {
+    ipcMain.handle('node:net:download', async (event, url: string, destPath: string, downloadId: number) => {
+        await fs.promises.mkdir(path.dirname(destPath), { recursive: true });
+
+        const sendProgress = (loaded: number) => {
+            if (!browserWindow.isDestroyed()) {
+                browserWindow.webContents.send('node:net:download-progress', { downloadId, loaded });
+            }
+        };
+
+        try {
+            await downloadToFile(url, destPath, sendProgress);
+        } catch (e) {
+            // Remove the partial file so isVersionAlreadyDownloaded doesn't later treat
+            // it as a complete download and feed a truncated zip to extraction.
+            await fs.promises.rm(destPath, { force: true }).catch(() => undefined);
+            throw e;
+        }
+    });
+}
+
+function downloadToFile(url: string, destPath: string, onProgress: (loaded: number) => void): Promise<void> {
+    return new Promise((resolve, reject) => {
+        const writeStream = fs.createWriteStream(destPath);
+        const fail = (error: Error) => {
+            writeStream.destroy();
+            reject(error);
+        };
+
+        const request = net.request(url);
+        request.on('redirect', () => request.followRedirect());
+        request.on('error', fail);
+        request.on('response', (response) => {
+            if (response.statusCode >= 400) {
+                request.abort();
+                fail(new Error(`Download failed with status ${response.statusCode}`));
+                return;
+            }
+
+            let loaded = 0;
+            let lastEmit = 0;
+            response.on('data', (chunk: Buffer) => {
+                loaded += chunk.length;
+                const now = Date.now();
+                if (now - lastEmit >= PROGRESS_INTERVAL_MS) {
+                    lastEmit = now;
+                    onProgress(loaded);
+                }
+            });
+            response.on('error', fail);
+            writeStream.on('error', fail);
+            writeStream.on('finish', () => {
+                onProgress(loaded);
+                resolve();
+            });
+            // Electron's IncomingMessage is a Readable at runtime but isn't typed as one.
+            (response as unknown as Readable).pipe(writeStream);
+        });
+        request.end();
+    });
+}

src-electron/ipc/zip-hook.ts

@@ -1,23 +1,14 @@
 import { BrowserWindow, ipcMain } from 'electron';
 import AdmZip from 'adm-zip';
+import extract from 'extract-zip';
 import path from 'path';
 
 let zipCreatorIdentifier = 0;
 const zipCreatorCache = new Map<number, AdmZip>();
 
 export function hookZipIpc(browserWindow: BrowserWindow) {
-    ipcMain.handle('zip:extractAllTo', (event, zip: string, outputFolder: string) => {
-        return new Promise((resolve, reject) => {
-            const adm = new AdmZip(zip);
-            outputFolder = outputFolder.replace(/\\/g, '/');
-            adm.extractAllToAsync(outputFolder, true, error => {
-                if (error) {
-                    reject(error);
-                } else {
-                    resolve(error);
-                }
-            });
-        });
+    ipcMain.handle('zip:extractAllTo', async (event, zip: string, outputFolder: string) => {
+        await extract(zip, { dir: path.resolve(outputFolder) });
     });
 
     ipcMain.handle('zip:readFile', (event, zip: string, fileName: string) => {

src-electron/preload/expose-preload.ts

@@ -2,6 +2,7 @@ import { contextBridge } from 'electron';
 import * as path from './node-path';
 import * as child_process from './node-child-process';
 import * as fs from './node-fs';
+import * as net from './node-net';
 import * as buffer from './node-buffer';
 import * as os from './node-os';
 import * as zip from './zip-preload';
@@ -12,6 +13,7 @@ contextBridge.exposeInMainWorld('node', {
     path: path,
     child_process: child_process,
     fs: fs,
+    net: net,
     buffer: buffer,
     os: os,
 });

src-electron/preload/node-net.ts

@@ -0,0 +1,21 @@
+import { ipcRenderer } from 'electron';
+
+let nextDownloadId = 0;
+
+export function download(
+    url: string,
+    destPath: string,
+    onProgress?: (downloadedBytes: number) => void
+): Promise<void> {
+    const downloadId = nextDownloadId++;
+
+    const listener = (_event: unknown, payload: { downloadId: number; loaded: number }) => {
+        if (payload.downloadId === downloadId && onProgress) {
+            onProgress(payload.loaded);
+        }
+    };
+    ipcRenderer.on('node:net:download-progress', listener);
+
+    return ipcRenderer.invoke('node:net:download', url, destPath, downloadId)
+        .finally(() => ipcRenderer.removeListener('node:net:download-progress', listener));
+}

src/env.d.ts

@@ -27,6 +27,9 @@ declare global {
             os: NodeOsProvider,
             child_process: NodeChildProcessProvider
             buffer: NodeBufferProvider
+            net: {
+                download: (url: string, destPath: string, onProgress?: (downloadedBytes: number) => void) => Promise<void>;
+            }
         },
         app: {
             checkForApplicationUpdates: () => Promise<void>;

src/providers/ror2/downloading/ThunderstoreDownloaderProvider.ts

@@ -23,13 +23,4 @@ export default abstract class ThunderstoreDownloaderProvider {
         totalProgressCallback: (downloadedSize: number, modName: string, status: DownloadStatusEnum, err: R2Error | null) => void
     ): Promise<void>;
 
-    /**
-     * Save the download buffer to a zip file in the cache.
-     *
-     * @param response  The download buffer.
-     * @param combo     The mod being downloaded.
-     * @param callback  Callback on if saving and extracting has been performed correctly. An error is provided if success is false.
-     */
-    public abstract saveToFile(response: Buffer, combo: ThunderstoreCombo, callback: (success: boolean, error?: R2Error) => void): void;
-
 }

src/r2mm/downloading/BetterThunderstoreDownloader.ts

@@ -1,16 +1,12 @@
-import axios, { AxiosResponse } from 'axios';
 import ThunderstoreCombo from '../../model/ThunderstoreCombo';
 import ZipExtract from '../installing/ZipExtract';
 import R2Error from '../../model/errors/R2Error';
 import PathResolver from '../../r2mm/manager/PathResolver';
 import FsProvider from '../../providers/generic/file/FsProvider';
-import FileWriteError from '../../model/errors/FileWriteError';
 import ThunderstoreDownloaderProvider from '../../providers/ror2/downloading/ThunderstoreDownloaderProvider';
-import ManagerInformation from '../../_managerinf/ManagerInformation';
 import * as DownloadUtils from '../../utils/DownloadUtils';
 import { DownloadStatusEnum } from '../../model/enums/DownloadStatusEnum';
 import path from '../../providers/node/path/path';
-import Buffer from '../../providers/node/buffer/buffer';
 
 export default class BetterThunderstoreDownloader extends ThunderstoreDownloaderProvider {
 
@@ -58,64 +54,43 @@ export default class BetterThunderstoreDownloader extends ThunderstoreDownloader
             }
 
             try {
-                const response = await this._downloadCombo(comboInProgress, singleModProgressCallback);
-                await this._saveDownloadResponse(response, comboInProgress, singleModProgressCallback);
+                await this._downloadAndExtract(comboInProgress, singleModProgressCallback);
                 await DownloadUtils.markAsDownloadedFromOnline(comboInProgress);
             } catch(e) {
                 throw R2Error.fromThrownValue(e, `Failed to download mod ${comboInProgress.getVersion().getFullName()}`);
             }
         }
     }
 
-    private async _downloadCombo(combo: ThunderstoreCombo, callback: (downloadedBytes: number, status: DownloadStatusEnum, err: R2Error | null) => void): Promise<AxiosResponse> {
-        return axios.get(combo.getVersion().getDownloadUrl(), {
-            onDownloadProgress: progress => callback(progress.loaded, DownloadStatusEnum.DOWNLOADING, null),
-            responseType: 'arraybuffer',
-            headers: {
-                'Content-Type': 'application/zip',
-                'Access-Control-Allow-Origin': '*'
-            }
-        });
-    }
+    private async _downloadAndExtract(combo: ThunderstoreCombo, callback: (downloadedBytes: number, status: DownloadStatusEnum, err: R2Error | null) => void): Promise<void> {
+        const fs = FsProvider.instance;
+        const modCacheDirectory = path.join(PathResolver.MOD_ROOT, 'cache', combo.getMod().getFullName());
+        if (! await fs.exists(modCacheDirectory)) {
+            await fs.mkdirs(modCacheDirectory);
+        }
+
+        const zipName = combo.getVersion().getVersionNumber().toString() + '.zip';
+
+        await window.node.net.download(
+            combo.getVersion().getDownloadUrl(),
+            path.join(modCacheDirectory, zipName),
+            (downloadedBytes) => callback(downloadedBytes, DownloadStatusEnum.DOWNLOADING, null)
+        );
 
-    private async _saveDownloadResponse(response: AxiosResponse, combo: ThunderstoreCombo, callback: (downloadedBytes: number, status: DownloadStatusEnum, err: R2Error | null) => void): Promise<void> {
-        const buf: Buffer = Buffer.from(response.data);
         const comboSize = combo.getVersion().getFileSize();
         callback(comboSize, DownloadStatusEnum.EXTRACTING, null);
-        await this.saveToFile(buf, combo, (success: boolean, error?: R2Error) => {
-            if (success) {
-                callback(comboSize, DownloadStatusEnum.EXTRACTED, null);
-            } else {
-                callback(comboSize, DownloadStatusEnum.FAILED, error || null);
+        await ZipExtract.extractAndDelete(
+            modCacheDirectory,
+            zipName,
+            combo.getVersion().getVersionNumber().toString(),
+            (success: boolean, error?: R2Error) => {
+                if (success) {
+                    callback(comboSize, DownloadStatusEnum.EXTRACTED, null);
+                } else {
+                    callback(comboSize, DownloadStatusEnum.FAILED, error || null);
+                }
             }
-        });
-    }
-
-    public async saveToFile(response: Buffer, combo: ThunderstoreCombo, callback: (success: boolean, error?: R2Error) => void) {
-        const fs = FsProvider.instance;
-        const cacheDirectory = path.join(PathResolver.MOD_ROOT, 'cache');
-        try {
-            if (! await fs.exists(path.join(cacheDirectory, combo.getMod().getFullName()))) {
-                await fs.mkdirs(path.join(cacheDirectory, combo.getMod().getFullName()));
-            }
-            await fs.writeFile(path.join(
-                cacheDirectory,
-                combo.getMod().getFullName(),
-                combo.getVersion().getVersionNumber().toString() + '.zip'
-            ), response);
-            await ZipExtract.extractAndDelete(
-                path.join(cacheDirectory, combo.getMod().getFullName()),
-                combo.getVersion().getVersionNumber().toString() + '.zip',
-                combo.getVersion().getVersionNumber().toString(),
-                callback
-            );
-        } catch(e) {
-            callback(false, new FileWriteError(
-                'File write error',
-                `Failed to write downloaded zip of ${combo.getMod().getFullName()} cache folder. \nReason: ${(e as Error).message}`,
-                `Try running ${ManagerInformation.APP_NAME} as an administrator`
-            ));
-        }
+        );
     }
 
 }

yarn.lock

@@ -9212,6 +9212,7 @@ __metadata:
     electron-packager: "npm:14.1.1"
     electron-updater: "npm:4.2.5"
     elliptic: "npm:^6.5.4"
+    extract-zip: "npm:^2.0.1"
     fflate: "npm:^0.8.2"
     floating-vue: "npm:5.2.2"
     fs-extra: "npm:^8.1.0"