diff --git a/apps/archiver/src/constants.ts b/apps/archiver/src/constants.ts index 4e2e518..718d95c 100644 --- a/apps/archiver/src/constants.ts +++ b/apps/archiver/src/constants.ts @@ -3,10 +3,13 @@ export const SIGNATURE_HEADER = 'Encoded-Data-Signature' export const STEMS_ARCHIVE_QUEUE_NAME = 'stems-archive' export const CLEANUP_ORPHANED_FILES_QUEUE_NAME = 'cleanup-orphaned-files' -// Archive content node — guaranteed to host every file uploaded to the network. -// Used as the final fallback when none of the mirrors listed on the upload are -// reachable, so a single mirror outage doesn't kill an archive job. -export const ARCHIVE_FALLBACK_HOST = 'https://creatornode2.audius.co' +// Archive content node — used as the final fallback when neither the +// canonical redirect host nor any mirror can serve the file, so a single node +// outage doesn't kill an archive job. creatornode2.audius.co was +// decommissioned (bare nginx 404s everything, 2026-07-16 stems incident); +// creatornode.audius.co is the node that ingests/transcodes uploads and +// peer-redirects for blobs it doesn't hold locally. +export const ARCHIVE_FALLBACK_HOST = 'https://creatornode.audius.co' // Per-mirror download timeout. A hung node otherwise blocks the whole job — // node-fetch has no default timeout, so without this we'd wait indefinitely. diff --git a/apps/archiver/src/workers/createStemsArchive/createStemsArchive.test.ts b/apps/archiver/src/workers/createStemsArchive/createStemsArchive.test.ts index c34cec5..15a8dbf 100644 --- a/apps/archiver/src/workers/createStemsArchive/createStemsArchive.test.ts +++ b/apps/archiver/src/workers/createStemsArchive/createStemsArchive.test.ts @@ -25,6 +25,8 @@ type FetchBehavior = { brokenTrackIds?: string[] /** Track ids whose /download endpoint should fail to resolve entirely. */ unresolvableTrackIds?: string[] + /** Origins whose cidstream endpoint should 404 for every track. */ + brokenHosts?: string[] } // Stand-in for node-fetch: /v1/tracks/{id}/download resolves to a 302 whose @@ -59,7 +61,10 @@ const createMockFetch = (behavior: FetchBehavior = {}) => { const cidstreamMatch = url.match(/\/tracks\/cidstream\/([^?/]+)/) if (cidstreamMatch) { const trackId = cidstreamMatch[1] - if (behavior.brokenTrackIds?.includes(trackId)) { + if ( + behavior.brokenTrackIds?.includes(trackId) || + behavior.brokenHosts?.includes(new URL(url).origin) + ) { return { ok: false, status: 404, @@ -77,21 +82,31 @@ const createMockFetch = (behavior: FetchBehavior = {}) => { throw new Error(`Unexpected fetch in test: ${url}`) } - return mockFetch as unknown as WorkerServices['fetch'] + return vi.fn(mockFetch) as unknown as WorkerServices['fetch'] } +// Origins of all cidstream requests made through the (vi.fn-wrapped) mock fetch. +const cidstreamOrigins = (fetch: WorkerServices['fetch']): string[] => + (fetch as unknown as ReturnType).mock.calls + .map((call) => String(call[0])) + .filter((url) => url.includes('/tracks/cidstream/')) + .map((url) => new URL(url).origin) + const createMockSdk = ({ parentInspectFails = false }: { parentInspectFails?: boolean } = {}) => { const sdk = { tracks: { + // No download/stream fields: the pinned @audius/sdk (10.0.0) drops them + // during deserialization, so at runtime the archiver never sees mirrors. + // The mock must mirror that or it hides the empty-mirror-list path + // (2026-07-16 stems incident). getTrack: vi.fn(async () => ({ data: { id: PARENT_ID, title: 'Parent Track', isDownloadable: true, - origFilename: 'parent.wav', - download: { mirrors: ['https://mirror-a.test'] } + origFilename: 'parent.wav' } })), getTrackStems: vi.fn(async () => ({ @@ -236,6 +251,34 @@ describe('createStemsArchiveWorker parent-track handling', () => { } }) + it('downloads via the canonical redirect host when the SDK provides no mirrors', async () => { + const services = createServices() + const { processJob } = createStemsArchiveWorker(services) + + const result = await processJob(createJob()) + + expect(fsSync.existsSync(result.outputFile)).toBe(true) + // Every file must come from the canonical host on the first try — no + // fallback traffic when the redirect target is healthy. + const origins = cidstreamOrigins(services.fetch) + expect(origins.length).toBeGreaterThan(0) + expect(new Set(origins)).toEqual(new Set(['https://mirror-a.test'])) + }) + + it('falls back to the archive node when the canonical host cannot serve the file', async () => { + const services = createServices({ + fetchBehavior: { brokenHosts: ['https://mirror-a.test'] } + }) + const { processJob } = createStemsArchiveWorker(services) + + const result = await processJob(createJob()) + + expect(fsSync.existsSync(result.outputFile)).toBe(true) + expect(cidstreamOrigins(services.fetch)).toContain( + 'https://creatornode.audius.co' + ) + }) + it('still fails the job when a stem download fails', async () => { const { processJob } = createStemsArchiveWorker( createServices({ fetchBehavior: { brokenTrackIds: [STEM_IDS[0]] } }) diff --git a/apps/archiver/src/workers/createStemsArchive/createStemsArchive.ts b/apps/archiver/src/workers/createStemsArchive/createStemsArchive.ts index 7d19de8..692f6ad 100644 --- a/apps/archiver/src/workers/createStemsArchive/createStemsArchive.ts +++ b/apps/archiver/src/workers/createStemsArchive/createStemsArchive.ts @@ -166,15 +166,14 @@ export const createStemsArchiveWorker = (services: WorkerServices) => { // Mirror list comes from the upload metadata. `track.download` is null // on tracks that aren't user-downloadable (stem-only uploads), so fall // back to the stream mirrors — same content node set, same placement. - // downloadFile shuffles this list, tries each with a per-mirror timeout, - // and falls back to the archive node when every mirror fails, so an - // empty/missing list is still safe. // - // Cast: the pinned @audius/sdk (10.0.0) Track type predates the - // download/stream UrlWithMirrors fields that the API has been returning - // for a while. The fields exist at runtime on api.audius.co — see the - // Track schema in the current OpenAPI spec — so we read them through a - // local shape rather than upgrading the SDK in this PR. + // CAVEAT: the pinned @audius/sdk (10.0.0) predates these fields, and + // its generated deserializer (TrackFromJSONTyped) drops unknown keys — + // so even though api.audius.co returns download/stream at runtime, + // this cast reads undefined and the list is effectively EMPTY until + // the SDK is upgraded (2026-07-16 stems incident). That's safe only + // because downloadFile tries the canonical redirect host first and the + // archive node last; the mirrors are an optional middle tier. const trackWithMirrors = track as typeof track & { download?: { mirrors?: string[] } | null stream?: { mirrors?: string[] } | null diff --git a/apps/archiver/src/workers/createStemsArchive/utils.ts b/apps/archiver/src/workers/createStemsArchive/utils.ts index ce1edf1..78527b0 100644 --- a/apps/archiver/src/workers/createStemsArchive/utils.ts +++ b/apps/archiver/src/workers/createStemsArchive/utils.ts @@ -61,10 +61,16 @@ const shuffle = (items: readonly T[]): T[] => { } // Build the ordered list of hosts to try, preserving first-occurrence order -// after dedupe. Mirrors are shuffled; the archive fallback is appended last -// so it only gets hit when every mirror fails. -const buildCandidateHosts = (mirrors: readonly string[]): string[] => { - const ordered = [...shuffle(mirrors), ARCHIVE_FALLBACK_HOST] +// after dedupe. The canonical host goes first: the API's 302 already selected +// it by rendezvous for this exact cid, so it's the host most likely to hold +// the file — and mediorum peer-redirects on a miss anyway. Mirrors are +// shuffled after it; the archive fallback is appended last so it only gets +// hit when everything else fails. +const buildCandidateHosts = ( + canonicalUrl: string, + mirrors: readonly string[] +): string[] => { + const ordered = [canonicalUrl, ...shuffle(mirrors), ARCHIVE_FALLBACK_HOST] const seen = new Set() const result: string[] = [] for (const m of ordered) { @@ -408,7 +414,7 @@ export const createUtils = (services: WorkerServices) => { signal }) - const candidateHosts = buildCandidateHosts(mirrors ?? []) + const candidateHosts = buildCandidateHosts(canonicalUrl, mirrors ?? []) const attempted: { host: string; error?: string }[] = [] for (const host of candidateHosts) {