From b78b0569f236b43200dcb7f66cf74b5ceb51cd87 Mon Sep 17 00:00:00 2001 From: Dylan Jeffers Date: Thu, 16 Jul 2026 13:59:57 -0700 Subject: [PATCH] fix(archiver): try the canonical redirect host first, fix dead archive fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stems downloads were still failing after #75 (Endevie job 62416-1416946406, 2026-07-16 20:47 UTC): every stem failed with 'All mirrors failed' after attempting exactly one host — creatornode2.audius.co. Two compounding bugs: 1. The mirror list is always empty. The pinned @audius/sdk 10.0.0 generated deserializer (TrackFromJSONTyped) drops the download/stream fields the API returns, so the cast in createStemsArchive reads undefined. The tests passed because the mock SDK returned mirrors the real SDK never delivers. 2. With no mirrors, the only candidate was ARCHIVE_FALLBACK_HOST — creatornode2.audius.co — which is decommissioned (bare nginx, 404s everything including /health_check). Fix: buildCandidateHosts now puts the canonical host from the API's 302 first — the API already selected it by rendezvous for the exact cid, and mediorum peer-redirects on a miss — so downloads succeed even with no mirrors. The fallback host moves to creatornode.audius.co (verified serving signed cidstream requests, 206). The mock SDK no longer fabricates mirrors, and new tests cover canonical-first and dead-canonical→fallback paths. Co-Authored-By: Claude Fable 5 --- apps/archiver/src/constants.ts | 11 ++-- .../createStemsArchive.test.ts | 51 +++++++++++++++++-- .../createStemsArchive/createStemsArchive.ts | 15 +++--- .../src/workers/createStemsArchive/utils.ts | 16 ++++-- 4 files changed, 72 insertions(+), 21 deletions(-) diff --git a/apps/archiver/src/constants.ts b/apps/archiver/src/constants.ts index 4e2e518..718d95c 100644 --- a/apps/archiver/src/constants.ts +++ b/apps/archiver/src/constants.ts @@ -3,10 +3,13 @@ export const SIGNATURE_HEADER = 'Encoded-Data-Signature' export const STEMS_ARCHIVE_QUEUE_NAME = 'stems-archive' export const CLEANUP_ORPHANED_FILES_QUEUE_NAME = 'cleanup-orphaned-files' -// Archive content node — guaranteed to host every file uploaded to the network. -// Used as the final fallback when none of the mirrors listed on the upload are -// reachable, so a single mirror outage doesn't kill an archive job. -export const ARCHIVE_FALLBACK_HOST = 'https://creatornode2.audius.co' +// Archive content node — used as the final fallback when neither the +// canonical redirect host nor any mirror can serve the file, so a single node +// outage doesn't kill an archive job. creatornode2.audius.co was +// decommissioned (bare nginx 404s everything, 2026-07-16 stems incident); +// creatornode.audius.co is the node that ingests/transcodes uploads and +// peer-redirects for blobs it doesn't hold locally. +export const ARCHIVE_FALLBACK_HOST = 'https://creatornode.audius.co' // Per-mirror download timeout. A hung node otherwise blocks the whole job — // node-fetch has no default timeout, so without this we'd wait indefinitely. diff --git a/apps/archiver/src/workers/createStemsArchive/createStemsArchive.test.ts b/apps/archiver/src/workers/createStemsArchive/createStemsArchive.test.ts index c34cec5..15a8dbf 100644 --- a/apps/archiver/src/workers/createStemsArchive/createStemsArchive.test.ts +++ b/apps/archiver/src/workers/createStemsArchive/createStemsArchive.test.ts @@ -25,6 +25,8 @@ type FetchBehavior = { brokenTrackIds?: string[] /** Track ids whose /download endpoint should fail to resolve entirely. */ unresolvableTrackIds?: string[] + /** Origins whose cidstream endpoint should 404 for every track. */ + brokenHosts?: string[] } // Stand-in for node-fetch: /v1/tracks/{id}/download resolves to a 302 whose @@ -59,7 +61,10 @@ const createMockFetch = (behavior: FetchBehavior = {}) => { const cidstreamMatch = url.match(/\/tracks\/cidstream\/([^?/]+)/) if (cidstreamMatch) { const trackId = cidstreamMatch[1] - if (behavior.brokenTrackIds?.includes(trackId)) { + if ( + behavior.brokenTrackIds?.includes(trackId) || + behavior.brokenHosts?.includes(new URL(url).origin) + ) { return { ok: false, status: 404, @@ -77,21 +82,31 @@ const createMockFetch = (behavior: FetchBehavior = {}) => { throw new Error(`Unexpected fetch in test: ${url}`) } - return mockFetch as unknown as WorkerServices['fetch'] + return vi.fn(mockFetch) as unknown as WorkerServices['fetch'] } +// Origins of all cidstream requests made through the (vi.fn-wrapped) mock fetch. +const cidstreamOrigins = (fetch: WorkerServices['fetch']): string[] => + (fetch as unknown as ReturnType).mock.calls + .map((call) => String(call[0])) + .filter((url) => url.includes('/tracks/cidstream/')) + .map((url) => new URL(url).origin) + const createMockSdk = ({ parentInspectFails = false }: { parentInspectFails?: boolean } = {}) => { const sdk = { tracks: { + // No download/stream fields: the pinned @audius/sdk (10.0.0) drops them + // during deserialization, so at runtime the archiver never sees mirrors. + // The mock must mirror that or it hides the empty-mirror-list path + // (2026-07-16 stems incident). getTrack: vi.fn(async () => ({ data: { id: PARENT_ID, title: 'Parent Track', isDownloadable: true, - origFilename: 'parent.wav', - download: { mirrors: ['https://mirror-a.test'] } + origFilename: 'parent.wav' } })), getTrackStems: vi.fn(async () => ({ @@ -236,6 +251,34 @@ describe('createStemsArchiveWorker parent-track handling', () => { } }) + it('downloads via the canonical redirect host when the SDK provides no mirrors', async () => { + const services = createServices() + const { processJob } = createStemsArchiveWorker(services) + + const result = await processJob(createJob()) + + expect(fsSync.existsSync(result.outputFile)).toBe(true) + // Every file must come from the canonical host on the first try — no + // fallback traffic when the redirect target is healthy. + const origins = cidstreamOrigins(services.fetch) + expect(origins.length).toBeGreaterThan(0) + expect(new Set(origins)).toEqual(new Set(['https://mirror-a.test'])) + }) + + it('falls back to the archive node when the canonical host cannot serve the file', async () => { + const services = createServices({ + fetchBehavior: { brokenHosts: ['https://mirror-a.test'] } + }) + const { processJob } = createStemsArchiveWorker(services) + + const result = await processJob(createJob()) + + expect(fsSync.existsSync(result.outputFile)).toBe(true) + expect(cidstreamOrigins(services.fetch)).toContain( + 'https://creatornode.audius.co' + ) + }) + it('still fails the job when a stem download fails', async () => { const { processJob } = createStemsArchiveWorker( createServices({ fetchBehavior: { brokenTrackIds: [STEM_IDS[0]] } }) diff --git a/apps/archiver/src/workers/createStemsArchive/createStemsArchive.ts b/apps/archiver/src/workers/createStemsArchive/createStemsArchive.ts index 7d19de8..692f6ad 100644 --- a/apps/archiver/src/workers/createStemsArchive/createStemsArchive.ts +++ b/apps/archiver/src/workers/createStemsArchive/createStemsArchive.ts @@ -166,15 +166,14 @@ export const createStemsArchiveWorker = (services: WorkerServices) => { // Mirror list comes from the upload metadata. `track.download` is null // on tracks that aren't user-downloadable (stem-only uploads), so fall // back to the stream mirrors — same content node set, same placement. - // downloadFile shuffles this list, tries each with a per-mirror timeout, - // and falls back to the archive node when every mirror fails, so an - // empty/missing list is still safe. // - // Cast: the pinned @audius/sdk (10.0.0) Track type predates the - // download/stream UrlWithMirrors fields that the API has been returning - // for a while. The fields exist at runtime on api.audius.co — see the - // Track schema in the current OpenAPI spec — so we read them through a - // local shape rather than upgrading the SDK in this PR. + // CAVEAT: the pinned @audius/sdk (10.0.0) predates these fields, and + // its generated deserializer (TrackFromJSONTyped) drops unknown keys — + // so even though api.audius.co returns download/stream at runtime, + // this cast reads undefined and the list is effectively EMPTY until + // the SDK is upgraded (2026-07-16 stems incident). That's safe only + // because downloadFile tries the canonical redirect host first and the + // archive node last; the mirrors are an optional middle tier. const trackWithMirrors = track as typeof track & { download?: { mirrors?: string[] } | null stream?: { mirrors?: string[] } | null diff --git a/apps/archiver/src/workers/createStemsArchive/utils.ts b/apps/archiver/src/workers/createStemsArchive/utils.ts index ce1edf1..78527b0 100644 --- a/apps/archiver/src/workers/createStemsArchive/utils.ts +++ b/apps/archiver/src/workers/createStemsArchive/utils.ts @@ -61,10 +61,16 @@ const shuffle = (items: readonly T[]): T[] => { } // Build the ordered list of hosts to try, preserving first-occurrence order -// after dedupe. Mirrors are shuffled; the archive fallback is appended last -// so it only gets hit when every mirror fails. -const buildCandidateHosts = (mirrors: readonly string[]): string[] => { - const ordered = [...shuffle(mirrors), ARCHIVE_FALLBACK_HOST] +// after dedupe. The canonical host goes first: the API's 302 already selected +// it by rendezvous for this exact cid, so it's the host most likely to hold +// the file — and mediorum peer-redirects on a miss anyway. Mirrors are +// shuffled after it; the archive fallback is appended last so it only gets +// hit when everything else fails. +const buildCandidateHosts = ( + canonicalUrl: string, + mirrors: readonly string[] +): string[] => { + const ordered = [canonicalUrl, ...shuffle(mirrors), ARCHIVE_FALLBACK_HOST] const seen = new Set() const result: string[] = [] for (const m of ordered) { @@ -408,7 +414,7 @@ export const createUtils = (services: WorkerServices) => { signal }) - const candidateHosts = buildCandidateHosts(mirrors ?? []) + const candidateHosts = buildCandidateHosts(canonicalUrl, mirrors ?? []) const attempted: { host: string; error?: string }[] = [] for (const host of candidateHosts) {