diff --git a/Extensions/Spine/managers/pixi-spine-atlas-manager.ts b/Extensions/Spine/managers/pixi-spine-atlas-manager.ts index aa8038bf3477..849a91c42fe6 100644 --- a/Extensions/Spine/managers/pixi-spine-atlas-manager.ts +++ b/Extensions/Spine/managers/pixi-spine-atlas-manager.ts @@ -103,6 +103,11 @@ namespace gdjs { return imagesMap; }, {}); + // Note: "atlas" and "spine" resources are never put inside a resource + // pack at export (see `ResourcePackPlanner`), because `PIXI.Assets` + // decides which loader to use from the extension of the URL, and a + // `blob:` URL has none. Only the atlas pages, which are plain images + // given to the loader below as already-loaded textures, are packed. const url = this._resourceLoader.getFullUrl(resource.file); const alias = url; diff --git a/GDJS/GDJS/IDE/ExporterHelper.cpp b/GDJS/GDJS/IDE/ExporterHelper.cpp index 0bce9fd8838e..ce50adfa5f73 100644 --- a/GDJS/GDJS/IDE/ExporterHelper.cpp +++ b/GDJS/GDJS/IDE/ExporterHelper.cpp @@ -1165,6 +1165,7 @@ void ExporterHelper::AddLibsInclude(bool pixiRenderers, InsertUnique(includesFiles, "inputmanager.js"); InsertUnique(includesFiles, "jsonmanager.js"); InsertUnique(includesFiles, "Model3DManager.js"); + InsertUnique(includesFiles, "ResourcePackManager.js"); InsertUnique(includesFiles, "ResourceLoader.js"); InsertUnique(includesFiles, "ResourceCache.js"); InsertUnique(includesFiles, "timemanager.js"); diff --git a/GDJS/Runtime/ResourceLoader.ts b/GDJS/Runtime/ResourceLoader.ts index 0bb1ee9e4046..23da79946af2 100644 --- a/GDJS/Runtime/ResourceLoader.ts +++ b/GDJS/Runtime/ResourceLoader.ts @@ -148,6 +148,12 @@ namespace gdjs { private _spineManager: SpineManager | null = null; private _svgManager: InternalInGameEditorOnlySvgManager; + /** + * Gives access to the resources of a game exported with its resources + * packed into ".gdpak" archives. Does nothing otherwise. + */ + private _resourcePackManager = new gdjs.ResourcePackManager(); + private privateResourceManager = new PrivateResourceManager(this); private sceneResourceLoadingQueue = new ResourceLoadingQueue( 'scene', @@ -239,6 +245,12 @@ namespace gdjs { ): void { this._globalResources = globalResources; + // The exporter writes this at the end of `data.js` when it packed the + // game resources. It stays null for previews and for games exported + // without packing, in which case every resource is downloaded as its own + // file, as before. + this._resourcePackManager.setManifest(gdjs.resourcePacks); + // TODO We should probably instanciate new queues to avoid side effects from running tasks. this.sceneResourceLoadingQueue.clear(); for (const objectResourceLoadingQueue of this.objectResourceLoadingQueues.values()) { @@ -289,7 +301,7 @@ namespace gdjs { [...this.privateResourceManager._resources.values()], ResourceLoader.maxForegroundConcurrency, ResourceLoader.maxAttempt, - async (resource) => { + async resource => { await this.privateResourceManager._loadResource(resource); await this.privateResourceManager._processResource(resource); loadedCount++; @@ -312,9 +324,10 @@ namespace gdjs { resourceNames, ResourceLoader.maxForegroundConcurrency, ResourceLoader.maxAttempt, - async (resourceName) => { - const resource = - this.privateResourceManager._resources.get(resourceName); + async resourceName => { + const resource = this.privateResourceManager._resources.get( + resourceName + ); if (resource) { await this.privateResourceManager._loadResource(resource); await this.privateResourceManager._processResource(resource); @@ -332,8 +345,9 @@ namespace gdjs { firstSceneName: string, onProgress: (count: number, total: number) => void ): Promise { - const firstSceneResourceNames = - this.sceneResourceLoadingQueue.getResourceNamesFor(firstSceneName); + const firstSceneResourceNames = this.sceneResourceLoadingQueue.getResourceNamesFor( + firstSceneName + ); if (!firstSceneResourceNames) { logger.warn( 'Can\'t load resource for unknown scene: "' + firstSceneName + '".' @@ -346,24 +360,50 @@ namespace gdjs { ...this._globalResources, ...firstSceneResourceNames, ]; - await ResourceLoader.processAndRetryIfNeededWithPromisePool( - resourceNames, - ResourceLoader.maxForegroundConcurrency, - ResourceLoader.maxAttempt, - async (resourceName) => { - const resource = - this.privateResourceManager._resources.get(resourceName); - if (!resource) { - logger.warn('Unable to find resource "' + resourceName + '".'); - return; - } - await this.privateResourceManager._loadResource(resource); - await this.privateResourceManager._processResource(resource); - loadedCount++; - onProgress(loadedCount, resourceNames.length); + + // No resource can be loaded while its pack is downloading, so without + // this the loading bar would stay at 0% for the whole download. Report + // the download itself as a fraction of a resource. + let lastReportedPackProgress = 0; + this._resourcePackManager.setOnProgressCallback( + (loadedBytes, totalBytes) => { + const packProgress = loadedBytes / totalBytes; + if (Math.abs(packProgress - lastReportedPackProgress) < 0.01) return; + lastReportedPackProgress = packProgress; + onProgress(loadedCount + packProgress, resourceNames.length); } ); + try { + // Resources that are only reachable dynamically (a sound played by + // name from an expression) are in no loading task, so nothing else + // would ever download the pack holding them - and the engine asks for + // their URL synchronously, when it is too late to download anything. + const startupPacksPromise = this._resourcePackManager.ensureStartupPacksLoaded(); + if (startupPacksPromise) await startupPacksPromise; + + await ResourceLoader.processAndRetryIfNeededWithPromisePool( + resourceNames, + ResourceLoader.maxForegroundConcurrency, + ResourceLoader.maxAttempt, + async resourceName => { + const resource = this.privateResourceManager._resources.get( + resourceName + ); + if (!resource) { + logger.warn('Unable to find resource "' + resourceName + '".'); + return; + } + await this.privateResourceManager._loadResource(resource); + await this.privateResourceManager._processResource(resource); + loadedCount++; + onProgress(loadedCount, resourceNames.length); + } + ); + } finally { + this._resourcePackManager.setOnProgressCallback(null); + } + this.sceneResourceLoadingQueue.setResourcesAs(firstSceneName, 'ready'); } @@ -443,8 +483,9 @@ namespace gdjs { debugLogger.log( `Loading of resources for object ${objectName} was requested.` ); - const objectResourceLoadingQueue = - this.getObjectResourceLoadingQueue(sceneName); + const objectResourceLoadingQueue = this.getObjectResourceLoadingQueue( + sceneName + ); objectResourceLoadingQueue.registerResources(objectName, usedResources); const task = objectResourceLoadingQueue.enqueue(objectName); objectResourceLoadingQueue.loadAllTasksInBackground(); @@ -466,8 +507,9 @@ namespace gdjs { } private getObjectResourceLoadingQueue(sceneName: string) { - let objectResourceLoadingQueue = - this.objectResourceLoadingQueues.get(sceneName); + let objectResourceLoadingQueue = this.objectResourceLoadingQueues.get( + sceneName + ); if (!objectResourceLoadingQueue) { objectResourceLoadingQueue = new ResourceLoadingQueue( `Independent objects of ${sceneName}`, @@ -522,21 +564,57 @@ namespace gdjs { unloadedSceneName, newSceneName ); - const objectResourceLoadingQueue = - this.getObjectResourceLoadingQueue(unloadedSceneName); + const objectResourceLoadingQueue = this.getObjectResourceLoadingQueue( + unloadedSceneName + ); objectResourceLoadingQueue.clear(); + this._unloadUnusedResourcePacks(); + debugLogger.log( `Unloading of resources for scene ${unloadedSceneName} finished.` ); } + /** + * Give back the memory used by the archives of the scenes that are not + * loaded anymore. Does nothing for a game exported without packed + * resources. + */ + private _unloadUnusedResourcePacks(): void { + const stillLoadedFiles = new Set(); + const addFilesOf = (resourceNames: Array) => { + for (const resourceName of resourceNames) { + const resource = this.privateResourceManager._resources.get( + resourceName + ); + if (resource) stillLoadedFiles.add(resource.file); + } + }; + + // Global resources are never unloaded. + addFilesOf(this._globalResources); + for (const loadingState of this.sceneResourceLoadingQueue.loadingStates.values()) { + if (loadingState.status === 'not-loaded') continue; + addFilesOf(loadingState.resourceNames); + } + for (const objectResourceLoadingQueue of this.objectResourceLoadingQueues.values()) { + for (const loadingState of objectResourceLoadingQueue.loadingStates.values()) { + if (loadingState.status === 'not-loaded') continue; + addFilesOf(loadingState.resourceNames); + } + } + + this._resourcePackManager.unloadPacksWithNoFileIn(stillLoadedFiles); + } + /** * Unload an object assets in background. */ unloadObjectResources(sceneName: string, objectName: string): void { - const objectResourceLoadingQueue = - this.getObjectResourceLoadingQueue(sceneName); + const objectResourceLoadingQueue = this.getObjectResourceLoadingQueue( + sceneName + ); if (!objectResourceLoadingQueue.areAssetsReady(objectName)) { debugLogger.log( `Can't unload of resources for object ${objectName} as it is not loaded.` @@ -573,6 +651,9 @@ namespace gdjs { for (const objectResourceLoadingQueue of this.objectResourceLoadingQueues.values()) { objectResourceLoadingQueue.clear(); } + // Keep the manifest: the packs are downloaded again when the resources + // are loaded back. + this._resourcePackManager.unloadAllPacks(); debugLogger.log(`Unloading of all resources finished.`); } @@ -610,6 +691,26 @@ namespace gdjs { return this.privateResourceManager._resources.get(resourceName) || null; } + /** + * Download the resource pack holding this resource, if the game was + * exported with packed resources and the pack is not downloaded yet. + * + * @returns null when there is nothing to wait for. Callers must check it + * rather than awaiting unconditionally, so that loading a resource keeps + * starting synchronously when there is no pack involved. + */ + ensurePackLoadedFor(resource: ResourceData): Promise | null { + return this._resourcePackManager.ensureLoadedFor(resource.file); + } + + /** + * @returns true when this file is stored inside a resource pack, and so is + * read from memory rather than downloaded on its own. + */ + isFileInResourcePack(file: string): boolean { + return this._resourcePackManager.isPacked(file); + } + // Helper methods used when resources are loaded from an URL. /** @@ -617,6 +718,23 @@ namespace gdjs { * the resource (this can be for example a token needed to access the resource). */ getFullUrl(url: string) { + // When the game was exported with packed resources, the file lives inside + // an archive that was already downloaded (`_loadResource` waits for it), + // and is read from a `blob:` URL instead of being fetched on its own. + const packedUrl = this._resourcePackManager.getObjectUrl(url); + if (packedUrl) return packedUrl; + + if (this._resourcePackManager.isPacked(url)) { + // The file is in a pack that is not downloaded yet. The URL returned + // below points to a file that the export does not contain, so loading + // it will fail: warn rather than let it look like a missing file. + logger.warn( + 'The resource file "' + + url + + '" was requested before its resource pack was downloaded.' + ); + } + if (this._runtimeGame.isInGameEdition()) { // Avoid adding cache burst to URLs which are assumed to be immutable files, // to avoid costly useless requests each time the game is hot-reloaded. @@ -821,8 +939,9 @@ namespace gdjs { const resourceNamesToUnload = new Set( objectLoadingState.resourceNames ); - const currentSceneObjectResourceLoadingQueue = - this.getObjectResourceLoadingQueue(currentSceneName); + const currentSceneObjectResourceLoadingQueue = this.getObjectResourceLoadingQueue( + currentSceneName + ); // The resources used by the current scene are already excluded from the // object resources list at export. // Other manually loaded objects may use the same resources. @@ -882,8 +1001,8 @@ namespace gdjs { activePromises++; asyncFunction(item) - .then((result) => results.push(result)) - .catch((error) => errors.push({ item, error })) + .then(result => results.push(result)) + .catch(error => errors.push({ item, error })) .finally(() => { activePromises--; if (index === items.length && activePromises === 0) { @@ -977,6 +1096,15 @@ namespace gdjs { ); return; } + // Make sure the archive holding this file is downloaded before the + // manager asks for its URL. Concurrent calls share the same download. + // Nothing is awaited for a game exported without packed resources, so + // that the download of a resource still starts synchronously. + const packLoadingPromise = this.resourceLoader.ensurePackLoadedFor( + resource + ); + if (packLoadingPromise) await packLoadingPromise; + await resourceManager.loadResource(resource.name); } @@ -988,7 +1116,9 @@ namespace gdjs { ); if (resourceManager) { debugLogger.log( - `Unloading of resources of kind ${resourceData.kind} : ${resourceName}` + `Unloading of resources of kind ${ + resourceData.kind + } : ${resourceName}` ); resourceManager.unloadResource(resourceData); } @@ -1062,7 +1192,11 @@ namespace gdjs { debugLogger.log(`Loading all ${this.name} resources, in background.`); while (this.loadingTaskQueue.length > 0) { debugLogger.log( - `Still resources of ${this.loadingTaskQueue.length} ${this.name}(s) to load: ${this.loadingTaskQueue.map((task) => task.identifier).join(', ')}` + `Still resources of ${this.loadingTaskQueue.length} ${ + this.name + }(s) to load: ${this.loadingTaskQueue + .map(task => task.identifier) + .join(', ')}` ); const task = this.loadingTaskQueue[this.loadingTaskQueue.length - 1]; if (task === undefined) { @@ -1071,7 +1205,9 @@ namespace gdjs { this.currentLoadingTaskIdentifier = task.identifier; if (!this.areAssetsLoaded(task.identifier)) { debugLogger.log( - `Loading (but not processing) resources for ${this.name} ${task.identifier}.` + `Loading (but not processing) resources for ${this.name} ${ + task.identifier + }.` ); const loadingState = this.loadingStates.get(task.identifier); if (loadingState) { @@ -1080,18 +1216,22 @@ namespace gdjs { ); } else { logger.warn( - `Can\'t load resource for unknown ${this.name}: "${task.identifier}".` + `Can\'t load resource for unknown ${this.name}: "${ + task.identifier + }".` ); return; } debugLogger.log( - `Done loading (but not processing) resources for ${this.name} ${task.identifier}.` + `Done loading (but not processing) resources for ${this.name} ${ + task.identifier + }.` ); // A task may have been moved last while awaiting resources to be // downloaded (see _prioritize). this.loadingTaskQueue.splice( - this.loadingTaskQueue.findIndex((element) => element === task), + this.loadingTaskQueue.findIndex(element => element === task), 1 ); task.onFinish(); @@ -1114,7 +1254,7 @@ namespace gdjs { ? ResourceLoader.maxForegroundConcurrency : ResourceLoader.maxBackgroundConcurrency, ResourceLoader.maxAttempt, - async (resourceName) => { + async resourceName => { const resource = this.resourceLoader._resources.get(resourceName); if (!resource) { logger.warn('Unable to find resource "' + resourceName + '".'); @@ -1190,7 +1330,7 @@ namespace gdjs { // The scene is not loaded: either prioritize it or add it to the loading queue. const taskIndex = this.loadingTaskQueue.findIndex( - (task) => task.identifier === taskIdentifier + task => task.identifier === taskIdentifier ); let task: LoadingTask; if (taskIndex !== -1) { @@ -1227,7 +1367,7 @@ namespace gdjs { return; } this.loadingStates.set(taskIdentifier, { - resourceNames: usedResources.map((resource) => resource.name), + resourceNames: usedResources.map(resource => resource.name), status: 'not-loaded', }); } @@ -1249,7 +1389,9 @@ namespace gdjs { } if (objectLoadingState.status !== 'not-loaded') { debugLogger.log( - `Resources for ${this.name} ${taskIdentifier} are already loading or loaded.` + `Resources for ${ + this.name + } ${taskIdentifier} are already loading or loaded.` ); return null; } @@ -1270,7 +1412,9 @@ namespace gdjs { } if (!unloadedTaskIdentifier) return; debugLogger.log( - `Unloading of resources for ${this.name} ${unloadedTaskIdentifier} was requested.` + `Unloading of resources for ${ + this.name + } ${unloadedTaskIdentifier} was requested.` ); const unloadedTaskState = this.loadingStates.get(unloadedTaskIdentifier); @@ -1290,7 +1434,9 @@ namespace gdjs { } debugLogger.log( - `Unloading of resources for ${this.name} ${unloadedTaskIdentifier} finished.` + `Unloading of resources for ${ + this.name + } ${unloadedTaskIdentifier} finished.` ); unloadedTaskState.status = 'not-loaded'; @@ -1350,8 +1496,8 @@ namespace gdjs { return taskIdentifier === this.currentLoadingTaskIdentifier ? this.currentTaskProgress : this.areAssetsLoaded(taskIdentifier) - ? 1 - : 0; + ? 1 + : 0; } clear() { diff --git a/GDJS/Runtime/ResourcePackManager.ts b/GDJS/Runtime/ResourcePackManager.ts new file mode 100644 index 000000000000..f98dc1355e01 --- /dev/null +++ b/GDJS/Runtime/ResourcePackManager.ts @@ -0,0 +1,420 @@ +/* + * GDevelop JS Platform + * Copyright 2013-present Florian Rival (Florian.Rival@gmail.com). All rights reserved. + * This project is released under the MIT License. + */ +namespace gdjs { + const logger = new gdjs.Logger('ResourcePackManager'); + + const PACK_MAGIC = 'GDPK'; + const PACK_HEADER_SIZE = 12; + const SUPPORTED_PACK_VERSION = 1; + + /** + * An entry of the index stored at the beginning of a pack. + */ + type ResourcePackEntryData = { + path: string; + offset: integer; + size: integer; + type: string; + }; + + /** + * The list of packs of an exported game, and the pack each resource file + * lives in. + * + * This is written by the exporter at the end of `data.js`, and is left + * undefined when the game was exported without packing its resources (which + * is the case for previews and for in-game edition). + * @category Resources + */ + export type ResourcePacksManifest = { + version: integer; + /** The pack file names, relative to the game index.html. */ + packs: Array; + /** Resource file name -> index in `packs`. */ + files: Record; + /** + * Packs that must be downloaded before the first scene starts, even though + * no loading task refers to their files. + * + * A resource that is only reachable dynamically - a sound played by name + * from an expression, an animation picked by an expression - appears in no + * `usedResources` list, so nothing would ever trigger the download of its + * pack, and the engine asks for its URL synchronously when it is used. + * Those resources are gathered in a pack listed here. + */ + startupPacks?: Array; + }; + + /** + * Set by the exported `data.js` when the game resources were packed. + * @category Resources + */ + export let resourcePacks: ResourcePacksManifest | null = null; + + /** + * A single ".gdpak" archive, downloaded as one file and then sliced to give + * each resource its own `blob:` URL. + * + * See `newIDE/app/src/ExportAndShare/ResourcePacking/PackFormat.js` for the + * description of the format. + */ + class ResourcePack { + private readonly _url: string; + /** Resource file name -> a slice of the downloaded archive. */ + private _entries = new Map(); + /** Resource file name -> the object URL handed out for it. */ + private _objectUrls = new Map(); + + constructor(url: string) { + this._url = url; + } + + async load( + onProgress?: (loadedBytes: integer, totalBytes: integer) => void + ): Promise { + const response = await fetch(this._url); + if (!response.ok) { + throw new Error( + `Could not download the resource pack "${this._url}" (status ${ + response.status + }).` + ); + } + + const blob = await ResourcePack._readResponseBlob(response, onProgress); + + const headerBytes = await blob.slice(0, PACK_HEADER_SIZE).arrayBuffer(); + if (headerBytes.byteLength < PACK_HEADER_SIZE) { + throw new Error(`The resource pack "${this._url}" is truncated.`); + } + + const headerBytesArray = new Uint8Array(headerBytes); + const magic = String.fromCharCode( + headerBytesArray[0], + headerBytesArray[1], + headerBytesArray[2], + headerBytesArray[3] + ); + if (magic !== PACK_MAGIC) { + throw new Error( + `"${this._url}" is not a resource pack (unexpected magic "${magic}").` + ); + } + + const headerView = new DataView(headerBytes); + const version = headerView.getUint32(4, true); + if (version !== SUPPORTED_PACK_VERSION) { + throw new Error( + `The resource pack "${ + this._url + }" uses the unsupported version ${version}.` + ); + } + + const indexByteLength = headerView.getUint32(8, true); + const indexJson = await blob + .slice(PACK_HEADER_SIZE, PACK_HEADER_SIZE + indexByteLength) + .text(); + const entries: Array = JSON.parse(indexJson).files; + + // Slicing a Blob does not copy anything: the browser owns the downloaded + // bytes (and may keep them out of memory), and each entry is only a view + // on them. + for (const entry of entries) { + this._entries.set( + entry.path, + blob.slice(entry.offset, entry.offset + entry.size, entry.type) + ); + } + } + + /** + * Read the whole response, reporting progress as bytes come in when the + * server announced a content length. Falls back to a plain `blob()` when + * streaming is not available. + */ + private static async _readResponseBlob( + response: Response, + onProgress?: (loadedBytes: integer, totalBytes: integer) => void + ): Promise { + const contentLength = Number( + response.headers.get('Content-Length') || '0' + ); + if (!onProgress || !contentLength || !response.body) { + return await response.blob(); + } + + const reader = response.body.getReader(); + const chunks: Array = []; + let loadedBytes = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (value) { + chunks.push(value); + loadedBytes += value.length; + onProgress(loadedBytes, contentLength); + } + } + + return new Blob(chunks); + } + + /** + * @returns a `blob:` URL for this file, or null if the pack does not + * contain it. The same URL is returned for subsequent calls. + */ + getObjectUrl(filePath: string): string | null { + const existingUrl = this._objectUrls.get(filePath); + if (existingUrl !== undefined) return existingUrl; + + const blob = this._entries.get(filePath); + if (!blob) return null; + + const objectUrl = URL.createObjectURL(blob); + this._objectUrls.set(filePath, objectUrl); + return objectUrl; + } + + getFilePaths(): Array { + return Array.from(this._entries.keys()); + } + + /** + * Release the archive and every URL handed out for it. + */ + dispose(): void { + for (const objectUrl of this._objectUrls.values()) { + URL.revokeObjectURL(objectUrl); + } + this._objectUrls.clear(); + this._entries.clear(); + } + } + + /** + * Gives access to the resources of a game whose export packed them into + * ".gdpak" archives, so that the exported game stays below the file count + * limits of hosting services (itch.io refuses archives with more than 1000 + * files). + * + * When a game was exported without packing, every method is a no-op and the + * engine downloads each resource file as usual. + * @category Resources + */ + export class ResourcePackManager { + private _manifest: ResourcePacksManifest | null = null; + private _packs: Array = []; + /** In-flight downloads, so that a pack is only ever downloaded once. */ + private _loadingPromises: Array | null> = []; + private _onProgress: + | ((loadedBytes: integer, totalBytes: integer) => void) + | null = null; + /** + * The download progress of the packs currently being downloaded, so that + * progress can be reported for all of them at once rather than having each + * pack fight over the loading bar. + */ + private _pendingDownloads = new Map< + integer, + { loadedBytes: integer; totalBytes: integer } + >(); + /** + * Incremented every time the packs are released, so that a download + * started before does not resurrect a pack that was disposed in between. + */ + private _generation: integer = 0; + + /** + * Read the manifest written by the exporter. Called by the resource loader + * when the game data is set, so that hot-reloading picks up changes too. + */ + setManifest(manifest: ResourcePacksManifest | null): void { + if (manifest && manifest.version !== SUPPORTED_PACK_VERSION) { + logger.error( + `Unsupported resource pack manifest version ${ + manifest.version + }, resources will be loaded as individual files.` + ); + manifest = null; + } + + this.dispose(); + this._manifest = manifest; + this._packs = manifest ? manifest.packs.map(() => null) : []; + this._loadingPromises = manifest ? manifest.packs.map(() => null) : []; + } + + /** + * Register a callback notified while a pack is being downloaded, so that + * the loading screen can show something is happening. + */ + setOnProgressCallback( + onProgress: ((loadedBytes: integer, totalBytes: integer) => void) | null + ): void { + this._onProgress = onProgress; + } + + isPacked(filePath: string): boolean { + return !!this._manifest && this._manifest.files[filePath] !== undefined; + } + + /** + * Download the pack containing this file, if any and if not already done. + * + * @returns null when there is nothing to wait for: the game was exported + * without packing, the file was left as an individual file, or its pack is + * already downloaded. Callers must not await unconditionally, so that + * loading a resource keeps starting synchronously. + */ + ensureLoadedFor(filePath: string): Promise | null { + const manifest = this._manifest; + if (!manifest) return null; + + const packIndex = manifest.files[filePath]; + if (packIndex === undefined) return null; + + return this._ensurePackLoaded(packIndex); + } + + /** + * Download the packs holding the resources that no loading task refers to. + * To be awaited before the first scene is loaded. + * + * @returns null when there is nothing to wait for. + */ + ensureStartupPacksLoaded(): Promise | null { + const manifest = this._manifest; + if (!manifest || !manifest.startupPacks) return null; + + const loadingPromises: Array> = []; + for (const packIndex of manifest.startupPacks) { + const loadingPromise = this._ensurePackLoaded(packIndex); + if (loadingPromise) loadingPromises.push(loadingPromise); + } + if (!loadingPromises.length) return null; + + return Promise.all(loadingPromises).then(() => {}); + } + + private _ensurePackLoaded(packIndex: integer): Promise | null { + const manifest = this._manifest; + if (!manifest || !manifest.packs[packIndex]) return null; + + if (this._packs[packIndex]) return null; + + const existingPromise = this._loadingPromises[packIndex]; + if (existingPromise) return existingPromise; + + const generation = this._generation; + const pack = new ResourcePack(manifest.packs[packIndex]); + const loadingPromise = pack + .load((loadedBytes, totalBytes) => { + if (generation !== this._generation) return; + this._pendingDownloads.set(packIndex, { loadedBytes, totalBytes }); + this._reportProgress(); + }) + .then(() => { + if (generation !== this._generation) { + // The packs were released while this one was downloading. + pack.dispose(); + return; + } + this._packs[packIndex] = pack; + this._pendingDownloads.delete(packIndex); + }) + .catch(error => { + if (generation !== this._generation) throw error; + // Forget the failed download, so that the retries done by the + // resource loader actually try again. + this._loadingPromises[packIndex] = null; + this._pendingDownloads.delete(packIndex); + throw error; + }); + + this._loadingPromises[packIndex] = loadingPromise; + return loadingPromise; + } + + private _reportProgress(): void { + const onProgress = this._onProgress; + if (!onProgress) return; + + let loadedBytes = 0; + let totalBytes = 0; + for (const pendingDownload of this._pendingDownloads.values()) { + loadedBytes += pendingDownload.loadedBytes; + totalBytes += pendingDownload.totalBytes; + } + if (totalBytes) onProgress(loadedBytes, totalBytes); + } + + /** + * @returns the `blob:` URL to read this file from its pack, or null if the + * file is not packed or its pack is not downloaded yet. + */ + getObjectUrl(filePath: string): string | null { + const manifest = this._manifest; + if (!manifest) return null; + + const packIndex = manifest.files[filePath]; + if (packIndex === undefined) return null; + + const pack = this._packs[packIndex]; + if (!pack) return null; + + return pack.getObjectUrl(filePath); + } + + /** + * Release every pack that holds none of the given files. + * + * Called when scenes are unloaded: as each scene has its own pack, the + * memory used by the archive of a scene that is not needed anymore can be + * given back. + */ + unloadPacksWithNoFileIn(stillLoadedFilePaths: Set): void { + for (let packIndex = 0; packIndex < this._packs.length; packIndex++) { + const pack = this._packs[packIndex]; + if (!pack) continue; + // A pack being downloaded must not be disposed: the promise waiting for + // it would then resolve on a pack that gives out nothing. + if (this._pendingDownloads.has(packIndex)) continue; + + const isStillNeeded = pack + .getFilePaths() + .some(filePath => stillLoadedFilePaths.has(filePath)); + if (isStillNeeded) continue; + + pack.dispose(); + this._packs[packIndex] = null; + this._loadingPromises[packIndex] = null; + } + } + + /** + * Release every downloaded pack, but keep the manifest so that they are + * downloaded again when needed. Used when hot-reloading. + */ + unloadAllPacks(): void { + this._generation++; + for (let packIndex = 0; packIndex < this._packs.length; packIndex++) { + const pack = this._packs[packIndex]; + if (pack) pack.dispose(); + this._packs[packIndex] = null; + this._loadingPromises[packIndex] = null; + } + this._pendingDownloads.clear(); + } + + dispose(): void { + this.unloadAllPacks(); + this._manifest = null; + this._packs = []; + this._loadingPromises = []; + } + } +} diff --git a/GDJS/Runtime/fontfaceobserver-font-manager/fontfaceobserver-font-manager.ts b/GDJS/Runtime/fontfaceobserver-font-manager/fontfaceobserver-font-manager.ts index 0a55f1e1e806..f5c75da3b869 100644 --- a/GDJS/Runtime/fontfaceobserver-font-manager/fontfaceobserver-font-manager.ts +++ b/GDJS/Runtime/fontfaceobserver-font-manager/fontfaceobserver-font-manager.ts @@ -98,12 +98,13 @@ namespace gdjs { */ private _loadFont(fontFamily: string, src: string): Promise { const descriptors = {}; - const srcWithUrl = 'url(' + encodeURI(src) + ')'; + const fullUrl = this._resourceLoader.getFullUrl(src); + const srcWithUrl = 'url(' + encodeURI(fullUrl) + ')'; // @ts-ignore if (typeof FontFace !== 'undefined') { // Load the given font using CSS Font Loading API. - return fetch(this._resourceLoader.getFullUrl(src), { + return fetch(fullUrl, { credentials: this._resourceLoader.checkIfCredentialsRequired(src) ? // Any resource stored on the GDevelop Cloud buckets needs the "credentials" of the user, // i.e: its gdevelop.io cookie, to be passed. @@ -111,7 +112,7 @@ namespace gdjs { : // For other resources, use "same-origin" as done by default by fetch. 'same-origin', }) - .then((response) => { + .then(response => { if (!response.ok) { const errorMessage = 'Unable to fetch ' + @@ -125,7 +126,7 @@ namespace gdjs { return response.arrayBuffer(); }) - .then((arrayBuffer) => { + .then(arrayBuffer => { // @ts-ignore const fontFace = new FontFace(fontFamily, arrayBuffer, descriptors); diff --git a/GDJS/Runtime/howler-sound-manager/howler-sound-manager.ts b/GDJS/Runtime/howler-sound-manager/howler-sound-manager.ts index e62eeee5ab7d..f0121a0126a8 100644 --- a/GDJS/Runtime/howler-sound-manager/howler-sound-manager.ts +++ b/GDJS/Runtime/howler-sound-manager/howler-sound-manager.ts @@ -17,6 +17,49 @@ namespace gdjs { logger.error('Error while loading an audio file: ' + error), }; + /** + * The file extensions Howler knows how to check support for. + * See https://github.com/goldfire/howler.js#format-array- + */ + const supportedAudioFormats = [ + 'mp3', + 'mpeg', + 'opus', + 'ogg', + 'oga', + 'wav', + 'aac', + 'caf', + 'm4a', + 'm4b', + 'mp4', + 'weba', + 'webm', + 'dolby', + 'flac', + ]; + + /** + * Howler guesses the codec of a sound from the extension of its URL. This + * does not work when the game resources were packed at export: the sound is + * then read from a `blob:` URL, which has no extension. Tell Howler the + * format explicitly, using the name the resource file had. + */ + const getAudioFormats = (file: string): Array | undefined => { + const lastDotIndex = file.lastIndexOf('.'); + if (lastDotIndex === -1) return undefined; + + const extension = file + .slice(lastDotIndex + 1) + .toLowerCase() + // A resource file can keep a search parameter when it comes from a URL. + .replace(/[?#].*$/, ''); + + return supportedAudioFormats.indexOf(extension) === -1 + ? undefined + : [extension]; + }; + /** * Ensure the volume is between 0 and 1. */ @@ -171,12 +214,12 @@ namespace gdjs { // Before loading, howler won't register events as without an ID we cannot set a listener. // Once we have an ID, we can transfer control of the events to howler. // We also need to call them once as Howler doesn't for the first play event. - this._onPlay.forEach((func) => { + this._onPlay.forEach(func => { // Transfer the event to howler now that we have an ID this.on('play', func); func(newID); }); - this._oncePlay.forEach((func) => func(newID)); + this._oncePlay.forEach(func => func(newID)); this._onPlay = []; this._oncePlay = []; } else this._howl.once('load', () => this.play()); // Play only once the howl is fully loaded @@ -513,18 +556,18 @@ namespace gdjs { this._clearCachedSpatialPosition.bind(this) ); const that = this; - document.addEventListener('deviceready', function () { + document.addEventListener('deviceready', function() { // pause/resume sounds in Cordova when the app is being paused/resumed document.addEventListener( 'pause', - function () { + function() { that.pauseAllActiveSounds(); }, false ); document.addEventListener( 'resume', - function () { + function() { that.resumeAllActiveSounds(); }, false @@ -656,12 +699,14 @@ namespace gdjs { container[file] = new Howl( Object.assign({}, HowlParameters, { src: this._getSoundUrlsFromResource(resource), + format: getAudioFormats(resource.file), onload: resolve, onloaderror: (soundId: number, error?: string) => reject(error), html5: isMusic, xhr: { - withCredentials: - this._resourceLoader.checkIfCredentialsRequired(file), + withCredentials: this._resourceLoader.checkIfCredentialsRequired( + file + ), }, // Cache the sound with no volume. This avoids a bug where it plays at full volume // for a split second before setting its correct volume. @@ -720,12 +765,12 @@ namespace gdjs { Object.assign( { src: this._getSoundUrlsFromResource(resource), + format: getAudioFormats(resource.file), html5: isMusic, xhr: { - withCredentials: - this._resourceLoader.checkIfCredentialsRequired( - resource.file - ), + withCredentials: this._resourceLoader.checkIfCredentialsRequired( + resource.file + ), }, // Cache the sound with no volume. This avoids a bug where it plays at full volume // for a split second before setting its correct volume. @@ -759,12 +804,12 @@ namespace gdjs { Object.assign( { src: this._getSoundUrlsFromResource(resource), + format: getAudioFormats(resource.file), html5: isMusic, xhr: { - withCredentials: - this._resourceLoader.checkIfCredentialsRequired( - resource.file - ), + withCredentials: this._resourceLoader.checkIfCredentialsRequired( + resource.file + ), }, // Cache the sound with no volume. This avoids a bug where it plays at full volume // for a split second before setting its correct volume. @@ -1073,36 +1118,38 @@ namespace gdjs { throw error; } } else if ( - resource.preloadInCache || - // Force downloading of sounds. - // TODO Decide if sounds should be allowed to be downloaded after the scene starts. - // - they should be requested automatically at the end of the scene loading - // - they will be downloaded while the scene is playing - // - other scenes will be pre-loaded only when all the sounds for the current scene are in cache - !resource.preloadAsMusic + // A file read from a resource pack is already in memory: requesting it + // to put it in the browser cache would only copy it for nothing. + !this._resourceLoader.isFileInResourcePack(resource.file) && + (resource.preloadInCache || + // Force downloading of sounds. + // TODO Decide if sounds should be allowed to be downloaded after the scene starts. + // - they should be requested automatically at the end of the scene loading + // - they will be downloaded while the scene is playing + // - other scenes will be pre-loaded only when all the sounds for the current scene are in cache + !resource.preloadAsMusic) ) { // preloading as sound already does a XHR request, hence "else if" try { const file = resource.file; await new Promise((resolve, reject) => { const sound = new XMLHttpRequest(); - sound.withCredentials = - this._resourceLoader.checkIfCredentialsRequired(file); + sound.withCredentials = this._resourceLoader.checkIfCredentialsRequired( + file + ); sound.addEventListener('load', () => { if (sound.status >= 200 && sound.status < 300) { resolve(undefined); } else { reject( - `HTTP error while preloading audio file in cache. Status is ${sound.status}.` + `HTTP error while preloading audio file in cache. Status is ${ + sound.status + }.` ); } }); - sound.addEventListener('error', (_) => - reject('XHR error: ' + file) - ); - sound.addEventListener('abort', (_) => - reject('XHR abort: ' + file) - ); + sound.addEventListener('error', _ => reject('XHR error: ' + file)); + sound.addEventListener('abort', _ => reject('XHR abort: ' + file)); sound.open('GET', this._getDefaultSoundUrl(resource)); sound.send(); }); @@ -1118,12 +1165,12 @@ namespace gdjs { getNetworkSyncData(): SoundManagerSyncData { const freeMusicsNetworkSyncData: SoundSyncData[] = []; - this._freeMusics.forEach((freeMusic) => { + this._freeMusics.forEach(freeMusic => { const musicSyncData = freeMusic.getNetworkSyncData(); if (musicSyncData) freeMusicsNetworkSyncData.push(musicSyncData); }); const freeSoundsNetworkSyncData: SoundSyncData[] = []; - this._freeSounds.forEach((freeSound) => { + this._freeSounds.forEach(freeSound => { const soundSyncData = freeSound.getNetworkSyncData(); if (soundSyncData) freeSoundsNetworkSyncData.push(soundSyncData); }); diff --git a/GDJS/Runtime/pixi-renderers/pixi-image-manager.ts b/GDJS/Runtime/pixi-renderers/pixi-image-manager.ts index 7e28fb2379e8..232a71520a17 100644 --- a/GDJS/Runtime/pixi-renderers/pixi-image-manager.ts +++ b/GDJS/Runtime/pixi-renderers/pixi-image-manager.ts @@ -163,7 +163,7 @@ namespace gdjs { ? 'use-credentials' : 'anonymous', }, - }).on('error', (error) => { + }).on('error', error => { logFileLoadingError(file, error); }); if (!texture) { @@ -426,18 +426,21 @@ namespace gdjs { // to continue, otherwise if we try to play the video too soon (at the beginning of scene for instance), // it will fail. await new Promise((resolve, reject) => { - const texture = PIXI.Texture.from(resourceUrl, { - resourceOptions: { - crossorigin: this._resourceLoader.checkIfCredentialsRequired( - resource.file - ) - ? 'use-credentials' - : 'anonymous', - autoPlay: false, - }, - }).on('error', (error) => { - reject(error); + // The resource is explicitly built as a video one: `PIXI.Texture.from` + // picks the kind of resource from the file extension of the URL, + // and there is none when the game resources were packed at export + // (the video is then read from a `blob:` URL). + const videoResource = new PIXI.VideoResource(resourceUrl, { + crossorigin: this._resourceLoader.checkIfCredentialsRequired( + resource.file + ) + ? 'use-credentials' + : 'anonymous', + autoPlay: false, }); + const texture = new PIXI.Texture( + new PIXI.BaseTexture(videoResource) + ); const baseTexture = texture.baseTexture; baseTexture @@ -446,7 +449,7 @@ namespace gdjs { applyTextureSettings(texture, resource); resolve(); }) - .on('error', (error) => { + .on('error', error => { reject(error); }); }); @@ -625,10 +628,9 @@ namespace gdjs { this._loadedThreeMaterials.dispose(resourceName); - const cubeTextureKeys = - this._loadedThreeCubeTextureKeysByResourceName.getValuesFor( - resourceName - ); + const cubeTextureKeys = this._loadedThreeCubeTextureKeysByResourceName.getValuesFor( + resourceName + ); if (cubeTextureKeys) { for (const cubeTextureKey of cubeTextureKeys) { const cubeTexture = this._loadedThreeCubeTextures.get(cubeTextureKey); diff --git a/GDJS/tests/karma.conf.js b/GDJS/tests/karma.conf.js index ee2e189d549b..193ea0c3a5f6 100644 --- a/GDJS/tests/karma.conf.js +++ b/GDJS/tests/karma.conf.js @@ -69,6 +69,7 @@ module.exports = function (config) { './newIDE/app/resources/GDJS/Runtime/fontfaceobserver-font-manager/fontfaceobserver-font-manager.js', './newIDE/app/resources/GDJS/Runtime/Model3DManager.js', './newIDE/app/resources/GDJS/Runtime/jsonmanager.js', + './newIDE/app/resources/GDJS/Runtime/ResourcePackManager.js', './newIDE/app/resources/GDJS/Runtime/ResourceLoader.js', './newIDE/app/resources/GDJS/Runtime/ResourceCache.js', './newIDE/app/resources/GDJS/Runtime/timemanager.js', diff --git a/GDJS/tests/tests/ResourcePackManager.js b/GDJS/tests/tests/ResourcePackManager.js new file mode 100644 index 000000000000..357b3132d90f --- /dev/null +++ b/GDJS/tests/tests/ResourcePackManager.js @@ -0,0 +1,362 @@ +// @ts-check + +/** + * Tests for gdjs.ResourcePackManager, and its integration in gdjs.ResourceLoader. + * + * The packs read here are built exactly like the exporter builds them, see + * `newIDE/app/src/ExportAndShare/ResourcePacking/PackFormat.js`. + */ +describe('gdjs.ResourcePackManager', () => { + const PACK_HEADER_SIZE = 12; + const PACK_ALIGNMENT = 16; + + const alignUp = value => { + const remainder = value % PACK_ALIGNMENT; + return remainder === 0 ? value : value + (PACK_ALIGNMENT - remainder); + }; + + /** + * @param {string} file + * @returns {ResourceData} + */ + const makeResourceData = file => ({ + kind: 'fake-resource-kind-for-testing-only', + name: file, + metadata: '', + file, + userAdded: true, + }); + + /** + * A scene with no resource of its own. + * @param {string} name + * @returns {LayoutData} + */ + const makeEmptySceneData = name => ({ + r: 0, + v: 0, + b: 0, + mangledName: name, + name, + objects: [], + objectsGroups: [], + layers: [], + instances: [], + behaviorsSharedData: [], + stopSoundsOnStartup: false, + title: '', + variables: [], + usedResources: [], + uiSettings: { + grid: false, + gridType: 'rectangular', + gridWidth: 10, + gridHeight: 10, + gridDepth: 10, + gridOffsetX: 0, + gridOffsetY: 0, + gridOffsetZ: 0, + gridColor: 0, + gridAlpha: 1, + snap: false, + }, + }); + + /** + * Build a ".gdpak" archive and return an URL to download it from. + * @param {Array<{path: string, content: string, type: string}>} files + * @returns {string} + */ + const createPackUrl = files => { + const encoder = new TextEncoder(); + const contents = files.map(file => encoder.encode(file.content)); + + // The offsets depend on the length of the index, which depends on the + // offsets: grow the index until it settles. + let indexByteLength = 0; + let indexJson = ''; + const entries = files.map((file, index) => ({ + path: file.path, + offset: 0, + size: contents[index].length, + type: file.type, + })); + for (let attempt = 0; attempt < 8; attempt++) { + let offset = alignUp(PACK_HEADER_SIZE + indexByteLength); + for (const entry of entries) { + entry.offset = offset; + offset = alignUp(offset + entry.size); + } + indexJson = JSON.stringify({ files: entries }); + const newIndexByteLength = encoder.encode(indexJson).length; + if (newIndexByteLength <= indexByteLength) break; + indexByteLength = newIndexByteLength; + } + + const contentStart = alignUp(PACK_HEADER_SIZE + indexByteLength); + const packBytes = new Uint8Array( + entries.length + ? alignUp( + entries[entries.length - 1].offset + + entries[entries.length - 1].size + ) + : contentStart + ); + packBytes.set(encoder.encode('GDPK'), 0); + const view = new DataView(packBytes.buffer); + view.setUint32(4, 1, true); + view.setUint32(8, indexByteLength, true); + packBytes.set( + encoder.encode( + indexJson + + ' '.repeat(indexByteLength - encoder.encode(indexJson).length) + ), + PACK_HEADER_SIZE + ); + entries.forEach((entry, index) => { + packBytes.set(contents[index], entry.offset); + }); + + return URL.createObjectURL( + new Blob([packBytes], { type: 'application/octet-stream' }) + ); + }; + + /** @type {Array} */ + let createdUrls = []; + + const createPackedGame = (files, extraResourceFiles = []) => { + const packUrl = createPackUrl(files); + createdUrls.push(packUrl); + + gdjs.resourcePacks = { + version: 1, + packs: [packUrl], + files: files.reduce((filesMap, file) => { + filesMap[file.path] = 0; + return filesMap; + }, {}), + }; + + const allFiles = [...files.map(({ path }) => path), ...extraResourceFiles]; + return gdjs.getPixiRuntimeGame({ + resources: { + resources: allFiles.map(filePath => ({ + kind: 'fake-resource-kind-for-testing-only', + name: filePath, + metadata: '', + file: filePath, + userAdded: true, + })), + }, + }); + }; + + afterEach(() => { + gdjs.resourcePacks = null; + createdUrls.forEach(url => URL.revokeObjectURL(url)); + createdUrls = []; + }); + + it('reads a file back from a pack, keeping its content and its MIME type', async () => { + const runtimeGame = createPackedGame([ + { path: 'a.png', content: 'content of a', type: 'image/png' }, + { path: 'b.mp3', content: 'the content of b', type: 'audio/mpeg' }, + ]); + const resourceLoader = runtimeGame.getResourceLoader(); + + await resourceLoader.ensurePackLoadedFor(makeResourceData('a.png')); + + const aUrl = resourceLoader.getFullUrl('a.png'); + expect(aUrl.startsWith('blob:')).to.be(true); + const aResponse = await fetch(aUrl); + expect(await aResponse.text()).to.be('content of a'); + expect(aResponse.headers.get('Content-Type')).to.be('image/png'); + + // Every file of the pack is available once it is downloaded. + const bResponse = await fetch(resourceLoader.getFullUrl('b.mp3')); + expect(await bResponse.text()).to.be('the content of b'); + expect(bResponse.headers.get('Content-Type')).to.be('audio/mpeg'); + }); + + it('hands out the same URL for a file, so that caches stay valid', async () => { + const runtimeGame = createPackedGame([ + { path: 'a.png', content: 'content of a', type: 'image/png' }, + ]); + const resourceLoader = runtimeGame.getResourceLoader(); + await resourceLoader.ensurePackLoadedFor(makeResourceData('a.png')); + + expect(resourceLoader.getFullUrl('a.png')).to.be( + resourceLoader.getFullUrl('a.png') + ); + }); + + it('downloads a pack only once, even for concurrent requests', async () => { + const runtimeGame = createPackedGame([ + { path: 'a.png', content: 'content of a', type: 'image/png' }, + { path: 'b.png', content: 'content of b', type: 'image/png' }, + ]); + const resourceLoader = runtimeGame.getResourceLoader(); + + await Promise.all([ + resourceLoader.ensurePackLoadedFor(makeResourceData('a.png')), + resourceLoader.ensurePackLoadedFor(makeResourceData('b.png')), + resourceLoader.ensurePackLoadedFor(makeResourceData('a.png')), + ]); + + // Both files come from the same downloaded archive. + expect( + await (await fetch(resourceLoader.getFullUrl('a.png'))).text() + ).to.be('content of a'); + expect( + await (await fetch(resourceLoader.getFullUrl('b.png'))).text() + ).to.be('content of b'); + }); + + it('leaves the files that were not packed alone', async () => { + const runtimeGame = createPackedGame( + [{ path: 'a.png', content: 'content of a', type: 'image/png' }], + ['loading-screen.png'] + ); + const resourceLoader = runtimeGame.getResourceLoader(); + + expect(resourceLoader.isFileInResourcePack('a.png')).to.be(true); + expect(resourceLoader.isFileInResourcePack('loading-screen.png')).to.be( + false + ); + + // A file left out of the packs keeps being downloaded on its own. + await resourceLoader.ensurePackLoadedFor( + makeResourceData('loading-screen.png') + ); + expect(resourceLoader.getFullUrl('loading-screen.png')).to.be( + 'loading-screen.png' + ); + }); + + it('does nothing for a game exported without packed resources', async () => { + gdjs.resourcePacks = null; + const runtimeGame = gdjs.getPixiRuntimeGame({ + resources: { + resources: [ + { + kind: 'fake-resource-kind-for-testing-only', + name: 'a.png', + metadata: '', + file: 'a.png', + userAdded: true, + }, + ], + }, + }); + const resourceLoader = runtimeGame.getResourceLoader(); + + await resourceLoader.ensurePackLoadedFor(makeResourceData('a.png')); + expect(resourceLoader.isFileInResourcePack('a.png')).to.be(false); + expect(resourceLoader.getFullUrl('a.png')).to.be('a.png'); + }); + + it('releases the packs when all the resources are unloaded', async () => { + const runtimeGame = createPackedGame([ + { path: 'a.png', content: 'content of a', type: 'image/png' }, + ]); + const resourceLoader = runtimeGame.getResourceLoader(); + await resourceLoader.ensurePackLoadedFor(makeResourceData('a.png')); + + const urlBeforeUnload = resourceLoader.getFullUrl('a.png'); + expect(urlBeforeUnload.startsWith('blob:')).to.be(true); + + resourceLoader.unloadAllResources(); + + // The archive is not held in memory anymore... + expect(resourceLoader.getFullUrl('a.png')).to.be('a.png'); + // ...but the game knows it can download it again. + expect(resourceLoader.isFileInResourcePack('a.png')).to.be(true); + + await resourceLoader.ensurePackLoadedFor(makeResourceData('a.png')); + const response = await fetch(resourceLoader.getFullUrl('a.png')); + expect(await response.text()).to.be('content of a'); + }); + + it('downloads the startup packs before the first scene, for resources no scene refers to', async () => { + // A sound played by name from an expression is in no `usedResources` list, + // so nothing triggers the download of its pack - and the sound manager asks + // for its URL synchronously when it is played. Without the startup packs, + // the game would ask the server for a file that is not there anymore. + const packUrl = createPackUrl([ + { path: 'dynamic.wav', content: 'the sound', type: 'audio/wav' }, + ]); + createdUrls.push(packUrl); + gdjs.resourcePacks = { + version: 1, + packs: [packUrl], + files: { 'dynamic.wav': 0 }, + startupPacks: [0], + }; + + const runtimeGame = gdjs.getPixiRuntimeGame({ + layouts: [makeEmptySceneData('Scene1')], + resources: { + resources: [ + { + kind: 'audio', + name: 'dynamicSound', + metadata: '', + file: 'dynamic.wav', + userAdded: true, + }, + ], + }, + }); + const resourceLoader = runtimeGame.getResourceLoader(); + + // Before the game starts, the pack is not downloaded yet. + expect(resourceLoader.getFullUrl('dynamic.wav')).to.be('dynamic.wav'); + + await runtimeGame.loadFirstAssetsAndStartBackgroundLoading('Scene1'); + + const url = resourceLoader.getFullUrl('dynamic.wav'); + expect(url.startsWith('blob:')).to.be(true); + expect(await (await fetch(url)).text()).to.be('the sound'); + }); + + it('fails clearly when a pack cannot be downloaded, and allows retrying', async () => { + gdjs.resourcePacks = { + version: 1, + packs: ['this-pack-does-not-exist.pak'], + files: { 'a.png': 0 }, + }; + const runtimeGame = gdjs.getPixiRuntimeGame({ + resources: { + resources: [ + { + kind: 'fake-resource-kind-for-testing-only', + name: 'a.png', + metadata: '', + file: 'a.png', + userAdded: true, + }, + ], + }, + }); + const resourceLoader = runtimeGame.getResourceLoader(); + + let firstError = null; + try { + await resourceLoader.ensurePackLoadedFor(makeResourceData('a.png')); + } catch (error) { + firstError = error; + } + expect(firstError).not.to.be(null); + + // The failed download must not be remembered, otherwise the retries done by + // the resource loader would all resolve to the same failure. + let secondError = null; + try { + await resourceLoader.ensurePackLoadedFor(makeResourceData('a.png')); + } catch (error) { + secondError = error; + } + expect(secondError).not.to.be(null); + }); +}); diff --git a/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserHTML5Export.js b/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserHTML5Export.js index f6c1f7aa9c65..deb6094470d6 100644 --- a/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserHTML5Export.js +++ b/newIDE/app/src/ExportAndShare/BrowserExporters/BrowserHTML5Export.js @@ -25,11 +25,16 @@ import { ExplanationHeader, DoneFooter, ExportFlow, + PackResourcesField, } from '../GenericExporters/HTML5Export'; +import { packResourcesInBlobFiles } from '../ResourcePacking/BrowserResourcePacker'; +import { Column, Line } from '../../UI/Grid'; const gd: libGDevelop = global.gd; -type ExportState = null; +type ExportState = {| + packResources: boolean, +|}; type PreparedExporter = {| exporter: gdjsExporter, @@ -60,13 +65,28 @@ export const browserHTML5ExportPipeline: ExportPipeline< > = { name: exportPipelineName, - getInitialExportState: () => null, + getInitialExportState: () => ({ packResources: true }), canLaunchBuild: () => true, isNavigationDisabled: () => false, - renderHeader: () => , + renderHeader: ({ exportState, updateExportState, exportStep }) => + exportStep !== 'done' ? ( + + + + + + + updateExportState(() => ({ packResources })) + } + /> + + + ) : null, renderExportFlow: (props: ExportFlowProps) => ( @@ -133,14 +153,27 @@ export const browserHTML5ExportPipeline: ExportPipeline< })); }, - launchCompression: ( + launchCompression: async ( context: ExportPipelineContext, { textFiles, blobFiles }: ResourcesDownloadOutput ): Promise => { + const basePath = '/export/'; + + // Gather the resources into a few ".gdpak" archives before zipping, so + // that the zip stays below the file count limit of hosting services. + const filesToArchive = context.exportState.packResources + ? await packResourcesInBlobFiles({ + textFiles, + blobFiles, + basePath, + onProgress: context.updateStepProgress, + }) + : { textFiles, blobFiles }; + return archiveFiles({ - blobFiles, - textFiles, - basePath: '/export/', + blobFiles: filesToArchive.blobFiles, + textFiles: filesToArchive.textFiles, + basePath, onProgress: context.updateStepProgress, }); }, diff --git a/newIDE/app/src/ExportAndShare/GenericExporters/HTML5Export.js b/newIDE/app/src/ExportAndShare/GenericExporters/HTML5Export.js index c8d9d213e906..fd79e89d896d 100644 --- a/newIDE/app/src/ExportAndShare/GenericExporters/HTML5Export.js +++ b/newIDE/app/src/ExportAndShare/GenericExporters/HTML5Export.js @@ -17,6 +17,7 @@ import { ColumnStackLayout, LineStackLayout } from '../../UI/Layout'; import Check from '../../UI/CustomSvgIcons/Check'; import Help from '../../UI/CustomSvgIcons/Help'; import RaisedButton from '../../UI/RaisedButton'; +import Checkbox from '../../UI/Checkbox'; import { type ExportFlowProps } from '../ExportPipeline.flow'; const getIconStyle = ({ isMobile }: {| isMobile: boolean |}) => { @@ -52,6 +53,35 @@ export const ExplanationHeader = (): React.Node => { ); }; +type PackResourcesFieldProps = {| + packResources: boolean, + onChange: (packResources: boolean) => void, +|}; + +/** + * Most hosting services limit the number of files an uploaded archive may + * contain (itch.io refuses more than 1000). A game with a few detailed + * animations goes past that easily, so resources are packed by default. + */ +export const PackResourcesField = ({ + packResources, + onChange, +}: PackResourcesFieldProps): React.Node => ( + Pack the game resources into a few files} + tooltipOrHelperText={ + + Images, sounds and other resources are gathered into a handful of files + instead of one file each, so that the game can be uploaded to itch.io + and other services limiting the number of files. The game then has to be + served by a web server: opening index.html directly will not work. + + } + checked={packResources} + onCheck={(e, checked) => onChange(checked)} + /> +); + type HTML5ExportFlowProps = {| ...ExportFlowProps, exportPipelineName: string, diff --git a/newIDE/app/src/ExportAndShare/Headless/ExportLocalHtml5Headless.js b/newIDE/app/src/ExportAndShare/Headless/ExportLocalHtml5Headless.js index e279e012b60e..c2702871dee6 100644 --- a/newIDE/app/src/ExportAndShare/Headless/ExportLocalHtml5Headless.js +++ b/newIDE/app/src/ExportAndShare/Headless/ExportLocalHtml5Headless.js @@ -29,6 +29,12 @@ type Options = {| project: gdProject, i18n: I18nType, outputDir?: string, + /** + * Gather the game resources into a few ".gdpak" archives, so that the export + * stays below the file count limit of hosting services. On by default, as in + * the export dialog. + */ + packResources?: boolean, |}; type Result = {| outputDir: string |}; @@ -38,13 +44,14 @@ export const exportLocalHtml5Headless = async ({ project, i18n, outputDir, + packResources = true, }: Options): Promise => { const resolvedOutputDir = outputDir || resolveHtml5OutputDir(project); project.setLastCompilationDirectory(resolvedOutputDir); const context = { project, - exportState: { outputDir: resolvedOutputDir }, + exportState: { outputDir: resolvedOutputDir, packResources }, updateStepProgress: (count: number, total: number) => {}, i18n, }; diff --git a/newIDE/app/src/ExportAndShare/LocalExporters/LocalHTML5Export.js b/newIDE/app/src/ExportAndShare/LocalExporters/LocalHTML5Export.js index f1730ca25355..945b52922c2d 100644 --- a/newIDE/app/src/ExportAndShare/LocalExporters/LocalHTML5Export.js +++ b/newIDE/app/src/ExportAndShare/LocalExporters/LocalHTML5Export.js @@ -18,8 +18,10 @@ import { ExplanationHeader, DoneFooter, ExportFlow, + PackResourcesField, } from '../GenericExporters/HTML5Export'; import { downloadUrlsToLocalFiles } from '../../Utils/LocalFileDownloader'; +import { packResourcesInFolder } from '../ResourcePacking/LocalResourcePacker'; import DismissableTutorialMessage from '../../Hints/DismissableTutorialMessage'; // It's important to use remote and not electron for folder actions, @@ -32,6 +34,7 @@ const gd: libGDevelop = global.gd; type ExportState = { outputDir: string, + packResources: boolean, }; type PreparedExporter = {| @@ -60,6 +63,7 @@ export const localHTML5ExportPipeline: ExportPipeline< getInitialExportState: (project: gdProject) => ({ outputDir: project.getLastCompilationDirectory(), + packResources: true, }), canLaunchBuild: exportState => !!exportState.outputDir, @@ -82,12 +86,26 @@ export const localHTML5ExportPipeline: ExportPipeline< value={exportState.outputDir} defaultPath={project.getLastCompilationDirectory()} onChange={outputDir => { - updateExportState(() => ({ outputDir })); + updateExportState(prevExportState => ({ + ...prevExportState, + outputDir, + })); project.setLastCompilationDirectory(outputDir); }} fullWidth /> + + + updateExportState(prevExportState => ({ + ...prevExportState, + packResources, + })) + } + /> + ) : null, @@ -163,11 +181,21 @@ export const localHTML5ExportPipeline: ExportPipeline< return null; }, - launchCompression: ( + launchCompression: async ( context: ExportPipelineContext, exportOutput: ResourcesDownloadOutput ): Promise => { - return Promise.resolve(null); + // The export is a folder, so there is nothing to compress. This is where + // the resources are gathered into a few ".gdpak" archives instead, now + // that the ones stored as URLs have been downloaded. + if (context.exportState.packResources) { + await packResourcesInFolder({ + exportDir: context.exportState.outputDir, + onProgress: context.updateStepProgress, + }); + } + + return null; }, renderDoneFooter: ({ exportState }) => { diff --git a/newIDE/app/src/ExportAndShare/ResourcePacking/BrowserResourcePacker.js b/newIDE/app/src/ExportAndShare/ResourcePacking/BrowserResourcePacker.js new file mode 100644 index 000000000000..6e3380030226 --- /dev/null +++ b/newIDE/app/src/ExportAndShare/ResourcePacking/BrowserResourcePacker.js @@ -0,0 +1,124 @@ +// @flow +import path from 'path-browserify'; +import { buildPackLayout } from './PackFormat'; +import { + RESOURCE_KINDS_NEVER_PACKED, + appendResourcePacksManifestToDataJs, + buildResourcePacksManifest, + planResourcePacks, + readProjectDataFromDataJs, +} from './index'; +import { + type BlobFileDescriptor, + type TextFileDescriptor, +} from '../../Utils/BrowserArchiver'; + +// See BrowserFileSystem for why `path.posix` is not used directly. +const pathPosix = path.posix || path; + +type Args = {| + textFiles: Array, + blobFiles: Array, + basePath: string, + onProgress: (count: number, total: number) => void, +|}; + +type Output = {| + textFiles: Array, + blobFiles: Array, +|}; + +/** + * Replace the individual resource files of an exported game by a handful of + * ".gdpak" archives, so that the game can be uploaded to services limiting the + * number of files in an archive (itch.io allows 1000). + * + * Nothing is read back into memory: a pack is a `Blob` built from the blobs of + * the files it contains, which the browser keeps where they already are. + */ +export const packResourcesInBlobFiles = async ({ + textFiles, + blobFiles, + basePath, + onProgress, +}: Args): Promise => { + const dataJsFilePath = pathPosix.join(basePath, 'data.js'); + const dataJsFile = textFiles.find( + ({ filePath }) => filePath === dataJsFilePath + ); + if (!dataJsFile) { + throw new Error( + `Could not find "${dataJsFilePath}" in the exported game, so its resources can't be packed.` + ); + } + + const plan = planResourcePacks(readProjectDataFromDataJs(dataJsFile.text), { + excludedResourceKinds: RESOURCE_KINDS_NEVER_PACKED, + }); + if (!plan.packs.length) return { textFiles, blobFiles }; + + const blobByRelativePath: Map = new Map(); + blobFiles.forEach(({ filePath, blob }) => { + blobByRelativePath.set(pathPosix.relative(basePath, filePath), blob); + }); + + const packedFilePaths: Set = new Set(); + const packBlobFiles: Array = []; + let packedCount = 0; + + for (const pack of plan.packs) { + const contents: Array<{| filePath: string, blob: Blob |}> = []; + pack.filePaths.forEach(filePath => { + const blob = blobByRelativePath.get(filePath); + // A resource can be missing when the project references a file that was + // not exported. The engine already copes with a missing resource, so skip + // it rather than failing the whole export. + if (blob) contents.push({ filePath, blob }); + }); + + packedCount++; + onProgress(packedCount, plan.packs.length); + // Writing an empty archive would only waste a file. Nothing refers to it, + // as the manifest is built from the files that were really packed. + if (!contents.length) continue; + + const layout = buildPackLayout( + contents.map(({ filePath, blob }) => ({ filePath, size: blob.size })) + ); + + const parts: Array = [layout.headerBytes]; + layout.entries.forEach((entry, index) => { + parts.push(contents[index].blob); + if (layout.paddings[index] > 0) { + parts.push(new Uint8Array(layout.paddings[index])); + } + }); + + packBlobFiles.push({ + filePath: pathPosix.join(basePath, pack.name), + blob: new Blob(parts, { type: 'application/octet-stream' }), + }); + contents.forEach(({ filePath }) => packedFilePaths.add(filePath)); + } + + // The files that made it into a pack must not be exported on their own + // anymore - that is the whole point. + const remainingBlobFiles = blobFiles.filter( + ({ filePath }) => + !packedFilePaths.has(pathPosix.relative(basePath, filePath)) + ); + + const manifest = buildResourcePacksManifest(plan, packedFilePaths); + + return { + textFiles: textFiles.map(textFile => + textFile.filePath === dataJsFilePath + ? { + filePath: textFile.filePath, + text: appendResourcePacksManifestToDataJs(textFile.text, manifest), + } + : textFile + ), + blobFiles: [...remainingBlobFiles, ...packBlobFiles], + }; +}; diff --git a/newIDE/app/src/ExportAndShare/ResourcePacking/BrowserResourcePacker.spec.js b/newIDE/app/src/ExportAndShare/ResourcePacking/BrowserResourcePacker.spec.js new file mode 100644 index 000000000000..793d0c6dd1c8 --- /dev/null +++ b/newIDE/app/src/ExportAndShare/ResourcePacking/BrowserResourcePacker.spec.js @@ -0,0 +1,186 @@ +// @flow +import { packResourcesInBlobFiles } from './BrowserResourcePacker'; +import { parsePackIndex } from './PackFormat'; +import { readProjectDataFromDataJs } from './index'; + +const BASE_PATH = '/export/'; + +// The test environment has no `Blob`, while the browser this code runs in +// always has one. Node's implementation supports everything used here +// (`size`, `slice`, `text`, `arrayBuffer`, and Blobs as constructor parts). +beforeAll(() => { + if (typeof global.Blob === 'undefined') { + global.Blob = require('buffer').Blob; + } +}); + +const makeDataJs = (projectData: Object) => + 'gdjs.projectData = ' + + JSON.stringify(projectData) + + ';\ngdjs.runtimeGameOptions = {};\n'; + +const makeResource = (name: string, file: string, kind: string = 'image') => ({ + name, + file, + kind, + metadata: '', + userAdded: true, +}); + +const readFromPack = async (packBlob: Blob, filePath: string) => { + const packBytes = new Uint8Array(await packBlob.arrayBuffer()); + const entry = parsePackIndex(packBytes).entries.find( + entry => entry.path === filePath + ); + if (!entry) throw new Error(`"${filePath}" is not in this pack.`); + + return { + text: await packBlob.slice(entry.offset, entry.offset + entry.size).text(), + type: entry.type, + }; +}; + +describe('packResourcesInBlobFiles', () => { + const projectData = { + properties: { loadingScreen: { backgroundImageResourceName: 'splash' } }, + resources: { + resources: [ + makeResource('global', 'global.png'), + makeResource('music', 'music.mp3', 'audio'), + makeResource('menu', 'menu.png'), + makeResource('splash', 'splash.png'), + ], + }, + usedResources: [{ name: 'global' }, { name: 'music' }], + objects: [], + layouts: [{ name: 'Menu', usedResources: [{ name: 'menu' }], objects: [] }], + }; + + const makeInput = () => ({ + textFiles: [ + { filePath: '/export/data.js', text: makeDataJs(projectData) }, + { filePath: '/export/runtimegame.js', text: 'gdjs.RuntimeGame = ...' }, + ], + blobFiles: [ + { filePath: '/export/global.png', blob: new Blob(['the global image']) }, + { filePath: '/export/music.mp3', blob: new Blob(['the music']) }, + { filePath: '/export/menu.png', blob: new Blob(['the menu image']) }, + { filePath: '/export/splash.png', blob: new Blob(['the splash image']) }, + // A binary engine file, which must be left alone. + { + filePath: '/export/pixi-renderers/draco/gltf/draco_decoder.wasm', + blob: new Blob(['not a resource']), + }, + ], + basePath: BASE_PATH, + onProgress: (count: number, total: number) => {}, + }); + + it('replaces the resource blobs by packs, leaving the engine files alone', async () => { + const { textFiles, blobFiles } = await packResourcesInBlobFiles( + makeInput() + ); + + expect(blobFiles.map(({ filePath }) => filePath).sort()).toEqual([ + // The loading screen background is needed before the loading screen can + // be shown, so it stays an individual file. + '/export/pixi-renderers/draco/gltf/draco_decoder.wasm', + '/export/resources.pak', + '/export/scene-0.pak', + '/export/splash.png', + ]); + // Text files are untouched, apart from data.js. + expect( + textFiles.find(({ filePath }) => filePath === '/export/runtimegame.js') + ?.text + ).toBe('gdjs.RuntimeGame = ...'); + }); + + it('writes contents that can be read back, with their MIME type', async () => { + const { blobFiles } = await packResourcesInBlobFiles(makeInput()); + + const globalPack = blobFiles.find( + ({ filePath }) => filePath === '/export/resources.pak' + ); + if (!globalPack) throw new Error('The global pack was not written.'); + + expect(await readFromPack(globalPack.blob, 'global.png')).toEqual({ + text: 'the global image', + type: 'image/png', + }); + expect(await readFromPack(globalPack.blob, 'music.mp3')).toEqual({ + text: 'the music', + type: 'audio/mpeg', + }); + + const scenePack = blobFiles.find( + ({ filePath }) => filePath === '/export/scene-0.pak' + ); + if (!scenePack) throw new Error('The scene pack was not written.'); + expect(await readFromPack(scenePack.blob, 'menu.png')).toEqual({ + text: 'the menu image', + type: 'image/png', + }); + }); + + it('declares the packs in data.js without touching the project data', async () => { + const { textFiles } = await packResourcesInBlobFiles(makeInput()); + + const dataJs = textFiles.find( + ({ filePath }) => filePath === '/export/data.js' + ); + if (!dataJs) throw new Error('data.js is missing.'); + + expect(readProjectDataFromDataJs(dataJs.text)).toEqual(projectData); + + const manifest = JSON.parse( + dataJs.text + .slice( + dataJs.text.indexOf('gdjs.resourcePacks = ') + + 'gdjs.resourcePacks = '.length + ) + .trim() + .replace(/;$/, '') + ); + expect(manifest).toEqual({ + version: 1, + packs: ['resources.pak', 'scene-0.pak'], + files: { 'global.png': 0, 'music.mp3': 0, 'menu.png': 1 }, + // The global pack must be downloaded up front, as it holds the resources + // that no loading task refers to. + startupPacks: [0], + }); + }); + + it('leaves the export untouched when there is nothing to pack', async () => { + const emptyProjectData = { + properties: { loadingScreen: { backgroundImageResourceName: '' } }, + resources: { resources: [] }, + usedResources: [], + objects: [], + layouts: [], + }; + const { textFiles, blobFiles } = await packResourcesInBlobFiles({ + textFiles: [ + { filePath: '/export/data.js', text: makeDataJs(emptyProjectData) }, + ], + blobFiles: [], + basePath: BASE_PATH, + onProgress: (count: number, total: number) => {}, + }); + + expect(blobFiles).toEqual([]); + expect(textFiles[0].text).not.toContain('gdjs.resourcePacks'); + }); + + it('fails clearly when data.js is not in the export', async () => { + await expect( + packResourcesInBlobFiles({ + textFiles: [], + blobFiles: [], + basePath: BASE_PATH, + onProgress: (count: number, total: number) => {}, + }) + ).rejects.toThrow(/Could not find "\/export\/data.js"/); + }); +}); diff --git a/newIDE/app/src/ExportAndShare/ResourcePacking/FullExport.spec.js b/newIDE/app/src/ExportAndShare/ResourcePacking/FullExport.spec.js new file mode 100644 index 000000000000..a40ec5c4cad7 --- /dev/null +++ b/newIDE/app/src/ExportAndShare/ResourcePacking/FullExport.spec.js @@ -0,0 +1,217 @@ +// @flow +/** + * Runs a real HTML5 export (through libGD.js and the actual exporter) and packs + * its resources, so that the whole chain is checked: the exporter writes + * `data.js` and `index.html`, the packer reads them back, and the game engine + * would find `gdjs.ResourcePackManager` in the script list. + */ +import assignIn from 'lodash/assignIn'; +import { packResourcesInFolder } from './LocalResourcePacker'; +import { parsePackIndex } from './PackFormat'; +import optionalRequire from '../../Utils/OptionalRequire'; + +const fs = optionalRequire('fs-extra'); +const path = optionalRequire('path'); +const os = optionalRequire('os'); +const process = optionalRequire('process'); + +const gd: libGDevelop = global.gd; + +// The tests are run from `newIDE/app`, where the built game engine lives. +const GDJS_ROOT = path.resolve(process.cwd(), 'resources/GDJS'); + +const addImageResource = ( + project: gdProject, + name: string, + absoluteFilePath: string +) => { + const resource = new gd.ImageResource(); + resource.setName(name); + resource.setFile(absoluteFilePath); + project.getResourcesManager().addResource(resource); + resource.delete(); +}; + +/** + * Add a sprite object using the given image, so that the resource is really + * "used" by the scene and ends up in its `usedResources`. + */ +const addSpriteObject = ( + container: gdObjectsContainer, + objectName: string, + imageResourceName: string +) => { + const object = container.insertNewObject( + // $FlowFixMe[prop-missing] - the project is the platform holder here. + global.testProject, + 'Sprite', + objectName, + container.getObjectsCount() + ); + const configuration = gd.asSpriteConfiguration(object.getConfiguration()); + const animation = new gd.Animation(); + animation.setDirectionsCount(1); + const direction = animation.getDirection(0); + const sprite = new gd.Sprite(); + sprite.setImageName(imageResourceName); + direction.addSprite(sprite); + animation.setDirection(direction, 0); + configuration.getAnimations().addAnimation(animation); + animation.delete(); + sprite.delete(); +}; + +describe('Full HTML5 export with packed resources', () => { + let workingDir = ''; + let exportDir = ''; + let project: any = null; + + beforeAll(async () => { + workingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gdevelop-export-')); + exportDir = path.join(workingDir, 'export'); + await fs.ensureDir(exportDir); + + // Real files on disk, so that the exporter really copies them. + const assetsDir = path.join(workingDir, 'assets'); + await fs.ensureDir(assetsDir); + const imagePaths: { [string]: string } = {}; + for (const name of ['global', 'menu', 'level']) { + const filePath = path.join(assetsDir, `${name}.png`); + await fs.writeFile(filePath, `the ${name} image`, 'utf8'); + imagePaths[name] = filePath; + } + + project = gd.ProjectHelper.createNewGDJSProject(); + global.testProject = project; + project.setName('Packing test'); + + addImageResource(project, 'globalImage', imagePaths.global); + addImageResource(project, 'menuImage', imagePaths.menu); + addImageResource(project, 'levelImage', imagePaths.level); + + // A global object, so that its image lands in the project-wide resources. + addSpriteObject(project.getObjects(), 'GlobalSprite', 'globalImage'); + + const menuScene = project.insertNewLayout('Menu', 0); + addSpriteObject(menuScene.getObjects(), 'MenuSprite', 'menuImage'); + const levelScene = project.insertNewLayout('Level', 1); + addSpriteObject(levelScene.getObjects(), 'LevelSprite', 'levelImage'); + + // `LocalFileSystem` transitively imports a web worker module that expects + // `self` to exist, so it is required here rather than imported at the top. + if (typeof global.self === 'undefined') global.self = global; + const LocalFileSystem = require('../LocalExporters/LocalFileSystem') + .default; + + // Run the actual exporter, as the export pipeline does. + const localFileSystem = new LocalFileSystem({ + downloadUrlsToLocalFiles: true, + }); + const fileSystem = assignIn(new gd.AbstractFileSystemJS(), localFileSystem); + const exporter = new gd.Exporter(fileSystem, GDJS_ROOT); + const exportOptions = new gd.ExportOptions(project, exportDir); + const exportSucceeded = exporter.exportWholePixiProject(exportOptions); + exportOptions.delete(); + exporter.delete(); + + if (!exportSucceeded) throw new Error('The export failed.'); + }, 60000); + + afterAll(async () => { + if (project) project.delete(); + global.testProject = null; + if (workingDir) await fs.remove(workingDir); + }); + + it('exports a game whose index.html loads the resource pack manager', async () => { + const indexHtml = await fs.readFile( + path.join(exportDir, 'index.html'), + 'utf8' + ); + + // Without this script, `gdjs.ResourcePackManager` would be undefined and + // the game would not start. + expect(indexHtml).toContain('