fix(archiver): try canonical redirect host first, fix dead archive fallback

apps/archiver/src/constants.ts

@@ -3,10 +3,13 @@ export const SIGNATURE_HEADER = 'Encoded-Data-Signature'
 export const STEMS_ARCHIVE_QUEUE_NAME = 'stems-archive'
 export const CLEANUP_ORPHANED_FILES_QUEUE_NAME = 'cleanup-orphaned-files'
 
-// Archive content node — guaranteed to host every file uploaded to the network.
-// Used as the final fallback when none of the mirrors listed on the upload are
-// reachable, so a single mirror outage doesn't kill an archive job.
-export const ARCHIVE_FALLBACK_HOST = 'https://creatornode2.audius.co'
+// Archive content node — used as the final fallback when neither the
+// canonical redirect host nor any mirror can serve the file, so a single node
+// outage doesn't kill an archive job. creatornode2.audius.co was
+// decommissioned (bare nginx 404s everything, 2026-07-16 stems incident);
+// creatornode.audius.co is the node that ingests/transcodes uploads and
+// peer-redirects for blobs it doesn't hold locally.
+export const ARCHIVE_FALLBACK_HOST = 'https://creatornode.audius.co'
 
 // Per-mirror download timeout. A hung node otherwise blocks the whole job —
 // node-fetch has no default timeout, so without this we'd wait indefinitely.

apps/archiver/src/workers/createStemsArchive/createStemsArchive.test.ts

@@ -25,6 +25,8 @@ type FetchBehavior = {
   brokenTrackIds?: string[]
   /** Track ids whose /download endpoint should fail to resolve entirely. */
   unresolvableTrackIds?: string[]
+  /** Origins whose cidstream endpoint should 404 for every track. */
+  brokenHosts?: string[]
 }
 
 // Stand-in for node-fetch: /v1/tracks/{id}/download resolves to a 302 whose
@@ -59,7 +61,10 @@ const createMockFetch = (behavior: FetchBehavior = {}) => {
     const cidstreamMatch = url.match(/\/tracks\/cidstream\/([^?/]+)/)
     if (cidstreamMatch) {
       const trackId = cidstreamMatch[1]
-      if (behavior.brokenTrackIds?.includes(trackId)) {
+      if (
+        behavior.brokenTrackIds?.includes(trackId) ||
+        behavior.brokenHosts?.includes(new URL(url).origin)
+      ) {
         return {
           ok: false,
           status: 404,
@@ -77,21 +82,31 @@ const createMockFetch = (behavior: FetchBehavior = {}) => {
 
     throw new Error(`Unexpected fetch in test: ${url}`)
   }
-  return mockFetch as unknown as WorkerServices['fetch']
+  return vi.fn(mockFetch) as unknown as WorkerServices['fetch']
 }
 
+// Origins of all cidstream requests made through the (vi.fn-wrapped) mock fetch.
+const cidstreamOrigins = (fetch: WorkerServices['fetch']): string[] =>
+  (fetch as unknown as ReturnType<typeof vi.fn>).mock.calls
+    .map((call) => String(call[0]))
+    .filter((url) => url.includes('/tracks/cidstream/'))
+    .map((url) => new URL(url).origin)
+
 const createMockSdk = ({
   parentInspectFails = false
 }: { parentInspectFails?: boolean } = {}) => {
   const sdk = {
     tracks: {
+      // No download/stream fields: the pinned @audius/sdk (10.0.0) drops them
+      // during deserialization, so at runtime the archiver never sees mirrors.
+      // The mock must mirror that or it hides the empty-mirror-list path
+      // (2026-07-16 stems incident).
       getTrack: vi.fn(async () => ({
         data: {
           id: PARENT_ID,
           title: 'Parent Track',
           isDownloadable: true,
-          origFilename: 'parent.wav',
-          download: { mirrors: ['https://mirror-a.test'] }
+          origFilename: 'parent.wav'
         }
       })),
       getTrackStems: vi.fn(async () => ({
@@ -236,6 +251,34 @@ describe('createStemsArchiveWorker parent-track handling', () => {
     }
   })
 
+  it('downloads via the canonical redirect host when the SDK provides no mirrors', async () => {
+    const services = createServices()
+    const { processJob } = createStemsArchiveWorker(services)
+
+    const result = await processJob(createJob())
+
+    expect(fsSync.existsSync(result.outputFile)).toBe(true)
+    // Every file must come from the canonical host on the first try — no
+    // fallback traffic when the redirect target is healthy.
+    const origins = cidstreamOrigins(services.fetch)
+    expect(origins.length).toBeGreaterThan(0)
+    expect(new Set(origins)).toEqual(new Set(['https://mirror-a.test']))
+  })
+
+  it('falls back to the archive node when the canonical host cannot serve the file', async () => {
+    const services = createServices({
+      fetchBehavior: { brokenHosts: ['https://mirror-a.test'] }
+    })
+    const { processJob } = createStemsArchiveWorker(services)
+
+    const result = await processJob(createJob())
+
+    expect(fsSync.existsSync(result.outputFile)).toBe(true)
+    expect(cidstreamOrigins(services.fetch)).toContain(
+      'https://creatornode.audius.co'
+    )
+  })
+
   it('still fails the job when a stem download fails', async () => {
     const { processJob } = createStemsArchiveWorker(
       createServices({ fetchBehavior: { brokenTrackIds: [STEM_IDS[0]] } })

apps/archiver/src/workers/createStemsArchive/createStemsArchive.ts

@@ -166,15 +166,14 @@ export const createStemsArchiveWorker = (services: WorkerServices) => {
       // Mirror list comes from the upload metadata. `track.download` is null
       // on tracks that aren't user-downloadable (stem-only uploads), so fall
       // back to the stream mirrors — same content node set, same placement.
-      // downloadFile shuffles this list, tries each with a per-mirror timeout,
-      // and falls back to the archive node when every mirror fails, so an
-      // empty/missing list is still safe.
       //
-      // Cast: the pinned @audius/sdk (10.0.0) Track type predates the
-      // download/stream UrlWithMirrors fields that the API has been returning
-      // for a while. The fields exist at runtime on api.audius.co — see the
-      // Track schema in the current OpenAPI spec — so we read them through a
-      // local shape rather than upgrading the SDK in this PR.
+      // CAVEAT: the pinned @audius/sdk (10.0.0) predates these fields, and
+      // its generated deserializer (TrackFromJSONTyped) drops unknown keys —
+      // so even though api.audius.co returns download/stream at runtime,
+      // this cast reads undefined and the list is effectively EMPTY until
+      // the SDK is upgraded (2026-07-16 stems incident). That's safe only
+      // because downloadFile tries the canonical redirect host first and the
+      // archive node last; the mirrors are an optional middle tier.
       const trackWithMirrors = track as typeof track & {
         download?: { mirrors?: string[] } | null
         stream?: { mirrors?: string[] } | null

apps/archiver/src/workers/createStemsArchive/utils.ts

@@ -61,10 +61,16 @@ const shuffle = <T>(items: readonly T[]): T[] => {
 }
 
 // Build the ordered list of hosts to try, preserving first-occurrence order
-// after dedupe. Mirrors are shuffled; the archive fallback is appended last
-// so it only gets hit when every mirror fails.
-const buildCandidateHosts = (mirrors: readonly string[]): string[] => {
-  const ordered = [...shuffle(mirrors), ARCHIVE_FALLBACK_HOST]
+// after dedupe. The canonical host goes first: the API's 302 already selected
+// it by rendezvous for this exact cid, so it's the host most likely to hold
+// the file — and mediorum peer-redirects on a miss anyway. Mirrors are
+// shuffled after it; the archive fallback is appended last so it only gets
+// hit when everything else fails.
+const buildCandidateHosts = (
+  canonicalUrl: string,
+  mirrors: readonly string[]
+): string[] => {
+  const ordered = [canonicalUrl, ...shuffle(mirrors), ARCHIVE_FALLBACK_HOST]
   const seen = new Set<string>()
   const result: string[] = []
   for (const m of ordered) {
@@ -408,7 +414,7 @@ export const createUtils = (services: WorkerServices) => {
       signal
     })
 
-    const candidateHosts = buildCandidateHosts(mirrors ?? [])
+    const candidateHosts = buildCandidateHosts(canonicalUrl, mirrors ?? [])
     const attempted: { host: string; error?: string }[] = []
 
     for (const host of candidateHosts) {