From f13debc6b99d6e3afbe830ffd698e2d1ac1efbf4 Mon Sep 17 00:00:00 2001 From: Ethan Green Date: Sat, 6 Jun 2026 17:15:36 -0400 Subject: [PATCH 1/2] Fix extraction of mods larger than 2GB adm-zip loads the entire archive into a single Buffer via fs.readFileSync, which throws ERR_FS_FILE_TOO_LARGE for files over 2GB. Swap the zip:extractAllTo handler to extract-zip, which streams each entry through yauzl and supports ZIP64. The remaining adm-zip uses operate on small manifest and profile zips and are left untouched. --- package.json | 1 + src-electron/ipc/zip-hook.ts | 15 +++------------ yarn.lock | 1 + 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index 96ca4d33f..5e9a44048 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "dot-prop": "^5.2.0", "electron-updater": "4.2.5", "elliptic": "^6.5.4", + "extract-zip": "^2.0.1", "fflate": "^0.8.2", "floating-vue": "5.2.2", "fs-extra": "^8.1.0", diff --git a/src-electron/ipc/zip-hook.ts b/src-electron/ipc/zip-hook.ts index 95019c760..58e7e04fa 100644 --- a/src-electron/ipc/zip-hook.ts +++ b/src-electron/ipc/zip-hook.ts @@ -1,23 +1,14 @@ import { BrowserWindow, ipcMain } from 'electron'; import AdmZip from 'adm-zip'; +import extract from 'extract-zip'; import path from 'path'; let zipCreatorIdentifier = 0; const zipCreatorCache = new Map(); export function hookZipIpc(browserWindow: BrowserWindow) { - ipcMain.handle('zip:extractAllTo', (event, zip: string, outputFolder: string) => { - return new Promise((resolve, reject) => { - const adm = new AdmZip(zip); - outputFolder = outputFolder.replace(/\\/g, '/'); - adm.extractAllToAsync(outputFolder, true, error => { - if (error) { - reject(error); - } else { - resolve(error); - } - }); - }); + ipcMain.handle('zip:extractAllTo', async (event, zip: string, outputFolder: string) => { + await extract(zip, { dir: path.resolve(outputFolder) }); }); ipcMain.handle('zip:readFile', (event, zip: string, fileName: string) => { diff --git a/yarn.lock b/yarn.lock index 213d0a486..070bd4cf5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9212,6 +9212,7 @@ __metadata: electron-packager: "npm:14.1.1" electron-updater: "npm:4.2.5" elliptic: "npm:^6.5.4" + extract-zip: "npm:^2.0.1" fflate: "npm:^0.8.2" floating-vue: "npm:5.2.2" fs-extra: "npm:^8.1.0" From d9c63a207c4c9544fff86837ef8bbbe4c99a8a81 Mon Sep 17 00:00:00 2001 From: Ethan Green Date: Sat, 6 Jun 2026 17:15:37 -0400 Subject: [PATCH 2/2] Stream mod downloads to disk to support files over 2GB The renderer downloaded mods with axios responseType 'arraybuffer', which buffered the whole file into a single ArrayBuffer (null past ~2GB in the XHR adapter) and then wrote it to disk in one fs.writeFile (capped at 2147483647 bytes). Both failed for mods larger than 2GB. Move the download to the main process: a new node:net:download IPC streams the response straight to a file and reports progress back over webContents.send. It uses Electron's net module so downloads stay on Chromium's network stack, honouring the same proxy and TLS settings as the rest of the app. Partial files are removed on failure so a stalled download isn't mistaken for a completed one. --- src-electron/ipc/init-ipc.ts | 2 + src-electron/ipc/node-net-impl.ts | 69 +++++++++++++++++ src-electron/preload/expose-preload.ts | 2 + src-electron/preload/node-net.ts | 21 +++++ src/env.d.ts | 3 + .../ThunderstoreDownloaderProvider.ts | 9 --- .../BetterThunderstoreDownloader.ts | 77 +++++++------------ 7 files changed, 123 insertions(+), 60 deletions(-) create mode 100644 src-electron/ipc/node-net-impl.ts create mode 100644 src-electron/preload/node-net.ts diff --git a/src-electron/ipc/init-ipc.ts b/src-electron/ipc/init-ipc.ts index 67e802018..b007de44c 100644 --- a/src-electron/ipc/init-ipc.ts +++ b/src-electron/ipc/init-ipc.ts @@ -2,6 +2,7 @@ import { BrowserWindow } from 'electron'; import { hookPathIpc } from './node-path-impl'; import { hookChildProcessIpc } from './node-child-process-impl'; import { hookFsIpc } from 'app/src-electron/ipc/node-fs-impl'; +import { hookNetIpc } from 'app/src-electron/ipc/node-net-impl'; import { hookZipIpc } from 'app/src-electron/ipc/zip-hook'; import { hookElectronIpc } from 'app/src-electron/ipc/electron-hook'; import {hookOsIpc} from "app/src-electron/ipc/node-os-impl"; @@ -10,6 +11,7 @@ export function hookIpc(browserWindow: BrowserWindow) { hookPathIpc(browserWindow); hookChildProcessIpc(browserWindow); hookFsIpc(browserWindow); + hookNetIpc(browserWindow); hookOsIpc(browserWindow); hookZipIpc(browserWindow); hookElectronIpc(browserWindow); diff --git a/src-electron/ipc/node-net-impl.ts b/src-electron/ipc/node-net-impl.ts new file mode 100644 index 000000000..acf98f185 --- /dev/null +++ b/src-electron/ipc/node-net-impl.ts @@ -0,0 +1,69 @@ +import { BrowserWindow, ipcMain, net } from 'electron'; +import fs from 'fs'; +import path from 'path'; +import { Readable } from 'stream'; + +// Throttle progress events so a multi-GB download doesn't flood IPC. +const PROGRESS_INTERVAL_MS = 200; + +export function hookNetIpc(browserWindow: BrowserWindow) { + ipcMain.handle('node:net:download', async (event, url: string, destPath: string, downloadId: number) => { + await fs.promises.mkdir(path.dirname(destPath), { recursive: true }); + + const sendProgress = (loaded: number) => { + if (!browserWindow.isDestroyed()) { + browserWindow.webContents.send('node:net:download-progress', { downloadId, loaded }); + } + }; + + try { + await downloadToFile(url, destPath, sendProgress); + } catch (e) { + // Remove the partial file so isVersionAlreadyDownloaded doesn't later treat + // it as a complete download and feed a truncated zip to extraction. + await fs.promises.rm(destPath, { force: true }).catch(() => undefined); + throw e; + } + }); +} + +function downloadToFile(url: string, destPath: string, onProgress: (loaded: number) => void): Promise { + return new Promise((resolve, reject) => { + const writeStream = fs.createWriteStream(destPath); + const fail = (error: Error) => { + writeStream.destroy(); + reject(error); + }; + + const request = net.request(url); + request.on('redirect', () => request.followRedirect()); + request.on('error', fail); + request.on('response', (response) => { + if (response.statusCode >= 400) { + request.abort(); + fail(new Error(`Download failed with status ${response.statusCode}`)); + return; + } + + let loaded = 0; + let lastEmit = 0; + response.on('data', (chunk: Buffer) => { + loaded += chunk.length; + const now = Date.now(); + if (now - lastEmit >= PROGRESS_INTERVAL_MS) { + lastEmit = now; + onProgress(loaded); + } + }); + response.on('error', fail); + writeStream.on('error', fail); + writeStream.on('finish', () => { + onProgress(loaded); + resolve(); + }); + // Electron's IncomingMessage is a Readable at runtime but isn't typed as one. + (response as unknown as Readable).pipe(writeStream); + }); + request.end(); + }); +} diff --git a/src-electron/preload/expose-preload.ts b/src-electron/preload/expose-preload.ts index d8d11186f..494ce048b 100644 --- a/src-electron/preload/expose-preload.ts +++ b/src-electron/preload/expose-preload.ts @@ -2,6 +2,7 @@ import { contextBridge } from 'electron'; import * as path from './node-path'; import * as child_process from './node-child-process'; import * as fs from './node-fs'; +import * as net from './node-net'; import * as buffer from './node-buffer'; import * as os from './node-os'; import * as zip from './zip-preload'; @@ -12,6 +13,7 @@ contextBridge.exposeInMainWorld('node', { path: path, child_process: child_process, fs: fs, + net: net, buffer: buffer, os: os, }); diff --git a/src-electron/preload/node-net.ts b/src-electron/preload/node-net.ts new file mode 100644 index 000000000..f34dd41c8 --- /dev/null +++ b/src-electron/preload/node-net.ts @@ -0,0 +1,21 @@ +import { ipcRenderer } from 'electron'; + +let nextDownloadId = 0; + +export function download( + url: string, + destPath: string, + onProgress?: (downloadedBytes: number) => void +): Promise { + const downloadId = nextDownloadId++; + + const listener = (_event: unknown, payload: { downloadId: number; loaded: number }) => { + if (payload.downloadId === downloadId && onProgress) { + onProgress(payload.loaded); + } + }; + ipcRenderer.on('node:net:download-progress', listener); + + return ipcRenderer.invoke('node:net:download', url, destPath, downloadId) + .finally(() => ipcRenderer.removeListener('node:net:download-progress', listener)); +} diff --git a/src/env.d.ts b/src/env.d.ts index 67bd90d91..8501301fc 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -27,6 +27,9 @@ declare global { os: NodeOsProvider, child_process: NodeChildProcessProvider buffer: NodeBufferProvider + net: { + download: (url: string, destPath: string, onProgress?: (downloadedBytes: number) => void) => Promise; + } }, app: { checkForApplicationUpdates: () => Promise; diff --git a/src/providers/ror2/downloading/ThunderstoreDownloaderProvider.ts b/src/providers/ror2/downloading/ThunderstoreDownloaderProvider.ts index fcdd31073..a8a5fda74 100644 --- a/src/providers/ror2/downloading/ThunderstoreDownloaderProvider.ts +++ b/src/providers/ror2/downloading/ThunderstoreDownloaderProvider.ts @@ -23,13 +23,4 @@ export default abstract class ThunderstoreDownloaderProvider { totalProgressCallback: (downloadedSize: number, modName: string, status: DownloadStatusEnum, err: R2Error | null) => void ): Promise; - /** - * Save the download buffer to a zip file in the cache. - * - * @param response The download buffer. - * @param combo The mod being downloaded. - * @param callback Callback on if saving and extracting has been performed correctly. An error is provided if success is false. - */ - public abstract saveToFile(response: Buffer, combo: ThunderstoreCombo, callback: (success: boolean, error?: R2Error) => void): void; - } diff --git a/src/r2mm/downloading/BetterThunderstoreDownloader.ts b/src/r2mm/downloading/BetterThunderstoreDownloader.ts index 133460ce3..15e7c5f1d 100644 --- a/src/r2mm/downloading/BetterThunderstoreDownloader.ts +++ b/src/r2mm/downloading/BetterThunderstoreDownloader.ts @@ -1,16 +1,12 @@ -import axios, { AxiosResponse } from 'axios'; import ThunderstoreCombo from '../../model/ThunderstoreCombo'; import ZipExtract from '../installing/ZipExtract'; import R2Error from '../../model/errors/R2Error'; import PathResolver from '../../r2mm/manager/PathResolver'; import FsProvider from '../../providers/generic/file/FsProvider'; -import FileWriteError from '../../model/errors/FileWriteError'; import ThunderstoreDownloaderProvider from '../../providers/ror2/downloading/ThunderstoreDownloaderProvider'; -import ManagerInformation from '../../_managerinf/ManagerInformation'; import * as DownloadUtils from '../../utils/DownloadUtils'; import { DownloadStatusEnum } from '../../model/enums/DownloadStatusEnum'; import path from '../../providers/node/path/path'; -import Buffer from '../../providers/node/buffer/buffer'; export default class BetterThunderstoreDownloader extends ThunderstoreDownloaderProvider { @@ -58,8 +54,7 @@ export default class BetterThunderstoreDownloader extends ThunderstoreDownloader } try { - const response = await this._downloadCombo(comboInProgress, singleModProgressCallback); - await this._saveDownloadResponse(response, comboInProgress, singleModProgressCallback); + await this._downloadAndExtract(comboInProgress, singleModProgressCallback); await DownloadUtils.markAsDownloadedFromOnline(comboInProgress); } catch(e) { throw R2Error.fromThrownValue(e, `Failed to download mod ${comboInProgress.getVersion().getFullName()}`); @@ -67,55 +62,35 @@ export default class BetterThunderstoreDownloader extends ThunderstoreDownloader } } - private async _downloadCombo(combo: ThunderstoreCombo, callback: (downloadedBytes: number, status: DownloadStatusEnum, err: R2Error | null) => void): Promise { - return axios.get(combo.getVersion().getDownloadUrl(), { - onDownloadProgress: progress => callback(progress.loaded, DownloadStatusEnum.DOWNLOADING, null), - responseType: 'arraybuffer', - headers: { - 'Content-Type': 'application/zip', - 'Access-Control-Allow-Origin': '*' - } - }); - } + private async _downloadAndExtract(combo: ThunderstoreCombo, callback: (downloadedBytes: number, status: DownloadStatusEnum, err: R2Error | null) => void): Promise { + const fs = FsProvider.instance; + const modCacheDirectory = path.join(PathResolver.MOD_ROOT, 'cache', combo.getMod().getFullName()); + if (! await fs.exists(modCacheDirectory)) { + await fs.mkdirs(modCacheDirectory); + } + + const zipName = combo.getVersion().getVersionNumber().toString() + '.zip'; + + await window.node.net.download( + combo.getVersion().getDownloadUrl(), + path.join(modCacheDirectory, zipName), + (downloadedBytes) => callback(downloadedBytes, DownloadStatusEnum.DOWNLOADING, null) + ); - private async _saveDownloadResponse(response: AxiosResponse, combo: ThunderstoreCombo, callback: (downloadedBytes: number, status: DownloadStatusEnum, err: R2Error | null) => void): Promise { - const buf: Buffer = Buffer.from(response.data); const comboSize = combo.getVersion().getFileSize(); callback(comboSize, DownloadStatusEnum.EXTRACTING, null); - await this.saveToFile(buf, combo, (success: boolean, error?: R2Error) => { - if (success) { - callback(comboSize, DownloadStatusEnum.EXTRACTED, null); - } else { - callback(comboSize, DownloadStatusEnum.FAILED, error || null); + await ZipExtract.extractAndDelete( + modCacheDirectory, + zipName, + combo.getVersion().getVersionNumber().toString(), + (success: boolean, error?: R2Error) => { + if (success) { + callback(comboSize, DownloadStatusEnum.EXTRACTED, null); + } else { + callback(comboSize, DownloadStatusEnum.FAILED, error || null); + } } - }); - } - - public async saveToFile(response: Buffer, combo: ThunderstoreCombo, callback: (success: boolean, error?: R2Error) => void) { - const fs = FsProvider.instance; - const cacheDirectory = path.join(PathResolver.MOD_ROOT, 'cache'); - try { - if (! await fs.exists(path.join(cacheDirectory, combo.getMod().getFullName()))) { - await fs.mkdirs(path.join(cacheDirectory, combo.getMod().getFullName())); - } - await fs.writeFile(path.join( - cacheDirectory, - combo.getMod().getFullName(), - combo.getVersion().getVersionNumber().toString() + '.zip' - ), response); - await ZipExtract.extractAndDelete( - path.join(cacheDirectory, combo.getMod().getFullName()), - combo.getVersion().getVersionNumber().toString() + '.zip', - combo.getVersion().getVersionNumber().toString(), - callback - ); - } catch(e) { - callback(false, new FileWriteError( - 'File write error', - `Failed to write downloaded zip of ${combo.getMod().getFullName()} cache folder. \nReason: ${(e as Error).message}`, - `Try running ${ManagerInformation.APP_NAME} as an administrator` - )); - } + ); } }