From ffa1e9719179ccd32600d795f0723a9a76b2fd89 Mon Sep 17 00:00:00 2001 From: Ethan Green Date: Mon, 4 May 2026 06:05:46 +0300 Subject: [PATCH 01/13] Implement cached schema fetching Fetches the latest schema from the API on game selection. Uses the 304 response of the API to determine if the remote version has been updated since the last request. Not a huge difference in performance locally but more efficient on the server side. --- src/pages/GameSelectionScreen.vue | 4 + src/r2mm/ecosystem/EcosystemSchema.ts | 181 ++++++++++++++---- src/store/modules/EcosystemUpdateModule.ts | 6 +- .../EcosystemSchema/EcosystemSchema.spec.ts | 163 +++++++++++++++- 4 files changed, 308 insertions(+), 46 deletions(-) diff --git a/src/pages/GameSelectionScreen.vue b/src/pages/GameSelectionScreen.vue index 28ecfc381..71b5fc06a 100644 --- a/src/pages/GameSelectionScreen.vue +++ b/src/pages/GameSelectionScreen.vue @@ -98,7 +98,10 @@ import Game from '../model/game/Game'; import { capitalize } from '../utils/StringUtils'; import { StorePlatform as platformLabels } from '../model/platform/StorePlatform'; import EcosystemUpdateIndicator from '../components/navigation/EcosystemUpdateIndicator.vue'; +import { getStore } from '../providers/generic/store/StoreProvider'; +import { State } from '../store'; +const store = getStore(); const gameSelection = useGameSelectionComposable(); provide(gameSelectionKey, gameSelection); @@ -159,6 +162,7 @@ function selectPlatform() { onMounted(async () => { window.app.checkForApplicationUpdates(); await initialize(); + void store.dispatch('ecosystemUpdate/updateEcosystemSchema'); }); diff --git a/src/r2mm/ecosystem/EcosystemSchema.ts b/src/r2mm/ecosystem/EcosystemSchema.ts index 7764f6d87..04986a8f1 100644 --- a/src/r2mm/ecosystem/EcosystemSchema.ts +++ b/src/r2mm/ecosystem/EcosystemSchema.ts @@ -12,37 +12,26 @@ import ManagerInformation from "../../_managerinf/ManagerInformation"; import {EcosystemModloaderPackages, EcosystemSupportedGames} from "../../model/schema/ThunderstoreSchema"; import {updateModLoaderExports} from "../installing/profile_installers/ModLoaderVariantRecord"; import LoggerProvider, {LogSeverity} from "../../providers/ror2/logging/LoggerProvider"; +import {getAxiosWithTimeouts} from "../../utils/HttpUtils"; +import {retry} from "../../utils/Common"; -export type VersionedThunderstoreEcosystem = ThunderstoreEcosystem & {version: string}; +export type VersionedThunderstoreEcosystem = ThunderstoreEcosystem & { + version: string; + lastModified?: string; +}; -async function getMergedEcosystemPath(): Promise { - return path.join(PathResolver.ROOT, "latest-ecosystem-schema.json"); -} - -export async function updateLatestEcosystemSchema(): Promise { - const latestSchema = await fetchLatestSchema(); - await writeLatestEcosystemSchema(latestSchema); - await internalUpdateEcosystemReactives(latestSchema); -} +type LatestSchemaFetchResult = + | {kind: "not-modified"} + | {kind: "fetched", schema: ThunderstoreEcosystem, lastModified?: string} + | {kind: "failed"}; -async function writeLatestEcosystemSchema(schema: ThunderstoreEcosystem): Promise { - const asMergedSchema: VersionedThunderstoreEcosystem = { - ...schema, - version: ManagerInformation.VERSION.toString(), - }; - const writable = JSON.stringify(asMergedSchema); - return FsProvider.instance.writeFile(await getMergedEcosystemPath(), writable); -} +const ECOSYSTEM_DATA_URL = "https://thunderstore.io/api/experimental/schema/dev/latest/"; -async function getLastSavedEcosystemSchema(): Promise { - const contentBuffer = await FsProvider.instance.readFile(await getMergedEcosystemPath()); - const content = contentBuffer.toString("utf8"); - const parsedContent = JSON.parse(content); - await validateSchema(parsedContent); - return parsedContent; +async function getMergedEcosystemPath(): Promise { + return path.join(PathResolver.ROOT, "latest-ecosystem-schema.json"); } -async function validateSchema(schema: any): Promise { +function validateSchema(schema: unknown): ThunderstoreEcosystem { const ajv = new Ajv(); addFormats(ajv); @@ -52,15 +41,15 @@ async function validateSchema(schema: any): Promise { if (!isOk) { throw new R2Error("Schema validation error", ajv.errorsText(validate.errors)); } + + return schema as ThunderstoreEcosystem; } -async function loadBundledSchema(): Promise { - await validateSchema(bundledEcosystem); - return bundledEcosystem as ThunderstoreEcosystem; +function loadBundledSchema(): ThunderstoreEcosystem { + return validateSchema(bundledEcosystem); } -async function fetchLatestSchema(): Promise { - // TODO - Implement fetching of latest resource +function createEmptySchema(): ThunderstoreEcosystem { return { schemaVersion: "", communities: {}, @@ -70,16 +59,128 @@ async function fetchLatestSchema(): Promise { }; } -async function resolveCachedEcosystemSchema(): Promise { +function mergeSchemas( + bundledSchema: ThunderstoreEcosystem, + latestSchema: ThunderstoreEcosystem +): ThunderstoreEcosystem { + const modloaderMap = new Map( + [...bundledSchema.modloaderPackages, ...latestSchema.modloaderPackages] + .map(pkg => [pkg.packageId, pkg]) + ); + + return { + schemaVersion: latestSchema.schemaVersion, + communities: { + ...bundledSchema.communities, + ...latestSchema.communities, + }, + games: { + ...bundledSchema.games, + ...latestSchema.games, + }, + modloaderPackages: [...modloaderMap.values()], + packageInstallers: { + ...bundledSchema.packageInstallers, + ...latestSchema.packageInstallers, + }, + }; +} + +async function fetchLatestSchema( + currentSchema: VersionedThunderstoreEcosystem | null +): Promise { + const timeout = 5000; + const requestConfig = { + validateStatus: (status: number) => { + if (status === 304) { + return true; + } + return status >= 200 && status < 300; + }, + ...(currentSchema?.lastModified ? {headers: {"If-Modified-Since": currentSchema.lastModified}} : {}), + }; + + try { + const axios = getAxiosWithTimeouts(timeout, timeout * 2); + const response = await retry( + () => axios.get(ECOSYSTEM_DATA_URL, requestConfig), + {attempts: 3, interval: 1000, throwLastErrorAsIs: true} + ); + const lastModified = typeof response.headers["last-modified"] === "string" + ? response.headers["last-modified"] + : undefined; + + if (response.status === 304) { + return {kind: "not-modified"}; + } + + return { + kind: "fetched", + schema: validateSchema(response.data), + ...(lastModified ? {lastModified} : {}), + }; + } catch (e) { + console.error(e); + return {kind: "failed"}; + } +} + +export async function updateLatestEcosystemSchema(): Promise { + const bundledSchema = loadBundledSchema(); + const currentSchema = await loadSavedEcosystemSchema(); + const result = await fetchLatestSchema(currentSchema); + + if (result.kind === "not-modified") { + return; + } + + if (result.kind === "failed") { + if (currentSchema == null) { + await writeLatestEcosystemSchema(bundledSchema); + await internalUpdateEcosystemReactives(bundledSchema); + } + throw new Error("Failed to update game list"); + } + + const mergedSchema = mergeSchemas(bundledSchema, result.schema); + await writeLatestEcosystemSchema(mergedSchema, result.lastModified); + await internalUpdateEcosystemReactives(mergedSchema); +} + +async function writeLatestEcosystemSchema( + schema: ThunderstoreEcosystem, + lastModified?: string +): Promise { + const asMergedSchema: VersionedThunderstoreEcosystem = { + ...schema, + version: ManagerInformation.VERSION.toString(), + ...(lastModified != null ? {lastModified} : {}), + }; + const writable = JSON.stringify(asMergedSchema); + return FsProvider.instance.writeFile(await getMergedEcosystemPath(), writable); +} + +async function readSavedEcosystemSchema(): Promise { + const contentBuffer = await FsProvider.instance.readFile(await getMergedEcosystemPath()); + const content = contentBuffer.toString("utf8"); + const parsedContent = JSON.parse(content); + const {version, lastModified, ...schemaContent} = parsedContent as VersionedThunderstoreEcosystem; + void version; + void lastModified; + validateSchema(schemaContent); + return parsedContent as VersionedThunderstoreEcosystem; +} + +async function loadSavedEcosystemSchema(): Promise { const mergeFilePath = await getMergedEcosystemPath(); - const bundledSchema = async () => ({...(await loadBundledSchema()), version: ManagerInformation.VERSION.toString()}); if (!(await FsProvider.instance.exists(mergeFilePath))) { - return bundledSchema(); + return null; } + try { - let content = await getLastSavedEcosystemSchema(); + const content = await readSavedEcosystemSchema(); if (!new VersionNumber(content.version).isEqualTo(ManagerInformation.VERSION)) { - return bundledSchema(); + return null; } return content; } catch (e) { @@ -88,8 +189,16 @@ async function resolveCachedEcosystemSchema(): Promise { + const content = await loadSavedEcosystemSchema(); + if (content != null) { + return content; } + return loadBundledSchema(); } async function internalUpdateEcosystemReactives(schema: ThunderstoreEcosystem): Promise { diff --git a/src/store/modules/EcosystemUpdateModule.ts b/src/store/modules/EcosystemUpdateModule.ts index 95e995d69..525027800 100644 --- a/src/store/modules/EcosystemUpdateModule.ts +++ b/src/store/modules/EcosystemUpdateModule.ts @@ -1,7 +1,7 @@ import { ActionTree, GetterTree, MutationTree } from 'vuex'; import { State as RootState } from '../../store'; -import { EcosystemSupportedGames } from '../../model/schema/ThunderstoreSchema'; +import { updateLatestEcosystemSchema } from '../../r2mm/ecosystem/EcosystemSchema'; export interface EcosystemUpdateState { isInProgress: boolean; @@ -47,9 +47,7 @@ export const EcosystemUpdateModule = { commit('startUpdate'); try { - // Re-evaluate the ecosystem schema. Future remote refresh - // logic should be plugged in here. - void EcosystemSupportedGames.value; + await updateLatestEcosystemSchema(); } catch (e) { commit('setError', e instanceof Error ? e : new Error(String(e))); } finally { diff --git a/test/vitest/tests/unit/EcosystemSchema/EcosystemSchema.spec.ts b/test/vitest/tests/unit/EcosystemSchema/EcosystemSchema.spec.ts index 6ffab811f..351d94056 100644 --- a/test/vitest/tests/unit/EcosystemSchema/EcosystemSchema.spec.ts +++ b/test/vitest/tests/unit/EcosystemSchema/EcosystemSchema.spec.ts @@ -6,6 +6,7 @@ vi.mock('../../../../../src/assets/data/ecosystemJsonSchema.json', () => ({ get default() { return mockJsonSchema; } })); +import type {ThunderstoreEcosystem} from '../../../../../src/assets/data/ecosystemTypes'; import {VersionedThunderstoreEcosystem, updateEcosystemReactives, updateLatestEcosystemSchema} from '../../../../../src/r2mm/ecosystem/EcosystemSchema'; import {EcosystemModloaderPackages, EcosystemSupportedGames} from '../../../../../src/model/schema/ThunderstoreSchema'; import {MODLOADER_PACKAGES, MOD_LOADER_VARIANTS, updateModLoaderExports} from '../../../../../src/r2mm/installing/profile_installers/ModLoaderVariantRecord'; @@ -18,16 +19,32 @@ import ManagerInformation from '../../../../../src/_managerinf/ManagerInformatio import LoggerProvider from '../../../../../src/providers/ror2/logging/LoggerProvider'; import StubLoggerProvider from '../../../stubs/providers/stub.LoggerProvider'; +const mockAxiosGet = vi.fn(); + +vi.mock('../../../../../src/utils/HttpUtils', () => ({ + getAxiosWithTimeouts: () => ({get: mockAxiosGet}), +})); + +vi.mock('../../../../../src/utils/Common', () => ({ + retry: (fn: () => Promise) => fn(), +})); + const TEST_ROOT = 'TEST_ROOT'; const latestSchemaFilePath = path.join(TEST_ROOT, 'latest-ecosystem-schema.json'); -async function writeCacheFile(schema: Partial) { - const full: VersionedThunderstoreEcosystem = { +function createMinimalSchema(): ThunderstoreEcosystem { + return { schemaVersion: '0.0.0', communities: {}, games: {}, modloaderPackages: [], packageInstallers: {}, + }; +} + +async function writeCacheFile(schema: Partial) { + const full: VersionedThunderstoreEcosystem = { + ...createMinimalSchema(), version: ManagerInformation.VERSION.toString(), ...schema, }; @@ -50,6 +67,7 @@ describe('EcosystemSchema', () => { LoggerProvider.provide(() => loggerImpl); spyLogger = vi.spyOn(LoggerProvider.instance, 'Log').mockImplementation(() => {}); updateModLoaderExports(); + mockAxiosGet.mockReset(); await FsProvider.instance.mkdirs(TEST_ROOT); }); @@ -115,16 +133,24 @@ describe('EcosystemSchema', () => { describe('updateLatestEcosystemSchema', () => { - test('writes file to disk', async () => { - expect(await FsProvider.instance.exists(latestSchemaFilePath)).toBe(false); + test('writes file to disk when fetch succeeds', async () => { + mockAxiosGet.mockResolvedValue({ + status: 200, + data: createMinimalSchema(), + headers: {}, + }); + expect(await FsProvider.instance.exists(latestSchemaFilePath)).toBe(false); await updateLatestEcosystemSchema(); - expect(await FsProvider.instance.exists(latestSchemaFilePath)).toBe(true); }); test('written file contains current version', async () => { - expect(await FsProvider.instance.exists(latestSchemaFilePath)).toBe(false); + mockAxiosGet.mockResolvedValue({ + status: 200, + data: createMinimalSchema(), + headers: {}, + }); await updateLatestEcosystemSchema(); @@ -133,6 +159,131 @@ describe('EcosystemSchema', () => { expect(parsed.version).toBe(ManagerInformation.VERSION.toString()); }); + test('stores lastModified from response header', async () => { + const lastModified = 'Tue, 15 Apr 2026 12:00:00 GMT'; + mockAxiosGet.mockResolvedValue({ + status: 200, + data: createMinimalSchema(), + headers: {'last-modified': lastModified}, + }); + + await updateLatestEcosystemSchema(); + + const content = (await FsProvider.instance.readFile(latestSchemaFilePath)).toString('utf8'); + const parsed: VersionedThunderstoreEcosystem = JSON.parse(content); + expect(parsed.lastModified).toBe(lastModified); + }); + + test('sends If-Modified-Since header when cache has lastModified', async () => { + const cachedLastModified = 'Mon, 14 Apr 2026 12:00:00 GMT'; + await writeCacheFile({lastModified: cachedLastModified}); + + mockAxiosGet.mockResolvedValue({ + status: 304, + data: null, + headers: {}, + }); + + await updateLatestEcosystemSchema(); + + expect(mockAxiosGet).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ + headers: {'If-Modified-Since': cachedLastModified}, + }) + ); + }); + + test('does nothing on 304 Not Modified', async () => { + await writeCacheFile({}); + const originalContent = (await FsProvider.instance.readFile(latestSchemaFilePath)).toString('utf8'); + + mockAxiosGet.mockResolvedValue({ + status: 304, + data: null, + headers: {}, + }); + + await updateLatestEcosystemSchema(); + + const afterContent = (await FsProvider.instance.readFile(latestSchemaFilePath)).toString('utf8'); + expect(afterContent).toBe(originalContent); + }); + + test('writes bundled schema and throws when fetch fails with no cache', async () => { + mockAxiosGet.mockRejectedValue(new Error('Network error')); + + expect(await FsProvider.instance.exists(latestSchemaFilePath)).toBe(false); + await expect(updateLatestEcosystemSchema()).rejects.toThrow(); + expect(await FsProvider.instance.exists(latestSchemaFilePath)).toBe(true); + + const content = (await FsProvider.instance.readFile(latestSchemaFilePath)).toString('utf8'); + const parsed: VersionedThunderstoreEcosystem = JSON.parse(content); + expect(parsed.schemaVersion).not.toBe(''); + expect(EcosystemSupportedGames.value.length).toBeGreaterThan(0); + }); + + test('keeps existing cache and throws when fetch fails with existing cache', async () => { + await writeCacheFile({}); + const originalContent = (await FsProvider.instance.readFile(latestSchemaFilePath)).toString('utf8'); + + mockAxiosGet.mockRejectedValue(new Error('Network error')); + + await expect(updateLatestEcosystemSchema()).rejects.toThrow(); + + const afterContent = (await FsProvider.instance.readFile(latestSchemaFilePath)).toString('utf8'); + expect(afterContent).toBe(originalContent); + }); + + test('merges fetched data with bundled schema', async () => { + mockAxiosGet.mockResolvedValue({ + status: 200, + data: { + ...createMinimalSchema(), + communities: { + 'test-added': { + displayName: 'Added Community', + categories: {}, + sections: {'default': {name: 'Default'}}, + }, + }, + }, + headers: {}, + }); + + await updateLatestEcosystemSchema(); + + const content = (await FsProvider.instance.readFile(latestSchemaFilePath)).toString('utf8'); + const parsed: VersionedThunderstoreEcosystem = JSON.parse(content); + expect(parsed.communities['test-added']).toBeDefined(); + // Bundled games are preserved through the merge + expect(EcosystemSupportedGames.value.length).toBeGreaterThan(0); + }); + + test('deduplicates modloader packages during merge', async () => { + mockAxiosGet.mockResolvedValue({ + status: 200, + data: { + ...createMinimalSchema(), + modloaderPackages: [ + {packageId: 'test-unique-loader', rootFolder: 'test', loader: 'bepinex'}, + {packageId: 'test-unique-loader', rootFolder: 'test-updated', loader: 'bepinex'}, + ], + }, + headers: {}, + }); + + await updateLatestEcosystemSchema(); + + const content = (await FsProvider.instance.readFile(latestSchemaFilePath)).toString('utf8'); + const parsed: VersionedThunderstoreEcosystem = JSON.parse(content); + const testLoaders = parsed.modloaderPackages.filter( + (p: any) => p.packageId === 'test-unique-loader' + ); + expect(testLoaders).toHaveLength(1); + expect(testLoaders[0]!.rootFolder).toBe('test-updated'); + }); + }); }); From 1aacf1cf4189be42b30dd5981beaa888de2f31d1 Mon Sep 17 00:00:00 2001 From: Ethan Green Date: Sat, 2 May 2026 10:05:57 +0300 Subject: [PATCH 02/13] Implement CDN-based image retrieval This is basically a four-step approach. The provider: 1. Checks to see if the images are bundled. 2. Checks to see if an instance of the schema is being served on localhost:1337 3. Checks to see if any images have been cached locally. 4. Retrieves the image from the CDN. Notably this also features the fire-once-and-fail-quick CDN check where it checks the connection once during startup and, if it fails, attempts steps 1-3 and then uses a default image. --- .../game-selection/GameSelectionCard.vue | 13 +- .../navigation/ManagerActivityBar.vue | 12 +- src/model/game/Game.ts | 10 +- src/model/game/GameManager.ts | 2 +- .../generic/connection/CdnProvider.ts | 9 + .../generic/image/GameImageProvider.ts | 158 ++++++++++++++++++ 6 files changed, 190 insertions(+), 14 deletions(-) create mode 100644 src/providers/generic/image/GameImageProvider.ts diff --git a/src/components/game-selection/GameSelectionCard.vue b/src/components/game-selection/GameSelectionCard.vue index b867f5d25..d74b4dfbb 100644 --- a/src/components/game-selection/GameSelectionCard.vue +++ b/src/components/game-selection/GameSelectionCard.vue @@ -3,7 +3,7 @@