Treat LONG_TERM_STORAGE_SLOT_SKIPPED as a skipped slot by default

.gitignore

@@ -87,6 +87,12 @@ web_modules/
 # TypeScript cache
 *.tsbuildinfo
 
+# Stray tsc output if it ever lands next to source instead of dist/ (e.g. from a stale
+# incremental build info predating outDir, or a non-project-mode tsc invocation)
+packages/*/src/**/*.js
+packages/*/src/**/*.js.map
+packages/*/src/**/*.d.ts
+
 # Optional npm cache directory
 .npm
 

packages/node/src/configure/NodeConfig.ts

@@ -3,8 +3,9 @@
 
 import { IConfig, NodeConfig } from '@subql/node-core';
 
-// NOTE to extend IConfig, change the type to interface
-export type ISolanaConfig = IConfig;
+export interface ISolanaConfig extends IConfig {
+  treatLongTermStorageSkipAsSkipped: boolean;
+}
 
 export class SolanaNodeConfig extends NodeConfig<ISolanaConfig> {
   /**
@@ -22,4 +23,8 @@ export class SolanaNodeConfig extends NodeConfig<ISolanaConfig> {
     // Rebuild with internal config
     super((config as any)._config, (config as any)._isTest);
   }
+
+  get treatLongTermStorageSkipAsSkipped(): boolean {
+    return this._config.treatLongTermStorageSkipAsSkipped ?? true;
+  }
 }

packages/node/src/solana/api.connection.ts

@@ -47,8 +47,15 @@ export class SolanaApiConnection
     eventEmitter: EventEmitter2,
     decoder: SolanaDecoder,
     config?: ISolanaEndpointConfig,
+    treatLongTermStorageSkipAsSkipped?: boolean,
   ): Promise<SolanaApiConnection> {
-    const api = await SolanaApi.create(endpoint, eventEmitter, decoder, config);
+    const api = await SolanaApi.create(
+      endpoint,
+      eventEmitter,
+      decoder,
+      config,
+      treatLongTermStorageSkipAsSkipped,
+    );
 
     return new SolanaApiConnection(api, fetchBlocksBatches);
   }

packages/node/src/solana/api.service.solana.ts

@@ -19,6 +19,7 @@ import {
   ISolanaEndpointConfig,
   SolanaBlock,
 } from '@subql/types-solana';
+import { SolanaNodeConfig } from '../configure/NodeConfig';
 import { SubqueryProject } from '../configure/SubqueryProject';
 import { SolanaApiConnection, FetchFunc, GetFetchFunc } from './api.connection';
 import { SolanaApi, SolanaSafeApi } from './api.solana';
@@ -78,13 +79,17 @@ export class SolanaApiService extends ApiService<
 
     apiService.updateBlockFetching();
 
+    const treatLongTermStorageSkipAsSkipped = new SolanaNodeConfig(nodeConfig)
+      .treatLongTermStorageSkipAsSkipped;
+
     await apiService.createConnections(network, (endpoint, config) =>
       SolanaApiConnection.create(
         endpoint,
         apiService.fetchBlocksBatches,
         eventEmitter,
         decoder,
         config,
+        treatLongTermStorageSkipAsSkipped,
       ),
     );
 

packages/node/src/solana/api.solana.skip-slot.spec.ts

@@ -0,0 +1,89 @@
+// Copyright 2020-2025 SubQuery Pte Ltd authors & contributors
+// SPDX-License-Identifier: GPL-3.0
+
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import {
+  SOLANA_ERROR__JSON_RPC__SERVER_ERROR_LONG_TERM_STORAGE_SLOT_SKIPPED,
+  SOLANA_ERROR__JSON_RPC__SERVER_ERROR_SLOT_SKIPPED,
+  SolanaError,
+} from '@solana/errors';
+import { BlockUnavailableError } from '@subql/node-core';
+import { SolanaApi } from './api.solana';
+import { SolanaDecoder } from './decoder';
+
+jest.mock('@solana/rpc', () => ({
+  createSolanaRpc: jest.fn(),
+}));
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const { createSolanaRpc } = require('@solana/rpc');
+
+function mockRpcClient(getBlockError: unknown) {
+  return {
+    getGenesisHash: () => ({ send: () => Promise.resolve('genesis-hash') }),
+    getBlock: () => ({ send: () => Promise.reject(getBlockError) }),
+  };
+}
+
+describe('SolanaApi skipped slot handling', () => {
+  const eventEmitter = new EventEmitter2();
+  const decoder = new SolanaDecoder();
+
+  it('treats SOLANA_ERROR__JSON_RPC__SERVER_ERROR_SLOT_SKIPPED as a confirmed skip', async () => {
+    createSolanaRpc.mockReturnValue(
+      mockRpcClient(
+        new SolanaError(SOLANA_ERROR__JSON_RPC__SERVER_ERROR_SLOT_SKIPPED, {
+          __serverMessage: 'Slot 1 was skipped',
+        }),
+      ),
+    );
+    const api = await SolanaApi.create(
+      'http://localhost',
+      eventEmitter,
+      decoder,
+    );
+    await expect(api.fetchBlock(1)).rejects.toBeInstanceOf(
+      BlockUnavailableError,
+    );
+  });
+
+  it('treats LONG_TERM_STORAGE_SLOT_SKIPPED as a skip by default', async () => {
+    createSolanaRpc.mockReturnValue(
+      mockRpcClient(
+        new SolanaError(
+          SOLANA_ERROR__JSON_RPC__SERVER_ERROR_LONG_TERM_STORAGE_SLOT_SKIPPED,
+          {
+            __serverMessage:
+              'Slot 1 was skipped, or missing in long-term storage',
+          },
+        ),
+      ),
+    );
+    const api = await SolanaApi.create(
+      'http://localhost',
+      eventEmitter,
+      decoder,
+    );
+    await expect(api.fetchBlock(1)).rejects.toBeInstanceOf(
+      BlockUnavailableError,
+    );
+  });
+
+  it('does not treat LONG_TERM_STORAGE_SLOT_SKIPPED as a skip when disabled at startup', async () => {
+    const rpcError = new SolanaError(
+      SOLANA_ERROR__JSON_RPC__SERVER_ERROR_LONG_TERM_STORAGE_SLOT_SKIPPED,
+      {
+        __serverMessage: 'Slot 1 was skipped, or missing in long-term storage',
+      },
+    );
+    createSolanaRpc.mockReturnValue(mockRpcClient(rpcError));
+    const api = await SolanaApi.create(
+      'http://localhost',
+      eventEmitter,
+      decoder,
+      undefined,
+      false,
+    );
+    await expect(api.fetchBlock(1)).rejects.toBe(rpcError);
+  });
+});

packages/node/src/solana/api.solana.ts

@@ -5,6 +5,7 @@ import { EventEmitter2 } from '@nestjs/event-emitter';
 import { assertIsAddress } from '@solana/addresses';
 import {
   isSolanaError,
+  SOLANA_ERROR__JSON_RPC__SERVER_ERROR_LONG_TERM_STORAGE_SLOT_SKIPPED,
   SOLANA_ERROR__JSON_RPC__SERVER_ERROR_SLOT_SKIPPED,
 } from '@solana/errors';
 import { createSolanaRpc, Rpc } from '@solana/rpc';
@@ -35,10 +36,27 @@ export type SolanaSafeApi = undefined;
 const REQUEST_TIMEOUT = 30_000;
 
 // Solana doesn't produce a block for every slot. For a skipped slot the RPC throws rather than returning null.
-// Other "not available" RPC errors (e.g. block not yet rooted on this node, or missing from long-term storage)
-// don't reliably mean the slot was skipped, so they're intentionally not treated as a skip here.
-function isSkippedSlotError(e: unknown): boolean {
-  return isSolanaError(e, SOLANA_ERROR__JSON_RPC__SERVER_ERROR_SLOT_SKIPPED);
+// BLOCK_NOT_AVAILABLE (block not yet rooted on this node) is intentionally not treated as a skip here, since
+// it's often just this endpoint lagging behind and not a confirmed skip.
+// LONG_TERM_STORAGE_SLOT_SKIPPED fires once a slot falls outside an RPC's local blockstore retention window and
+// its archival (e.g. bigtable) lookup returns not-found. This is ambiguous in general (could be a genuine
+// archival gap), but in practice recently-skipped slots commonly surface this way, so it's treated as a skip
+// unless disabled via the network-level config. This isn't endpoint-specific (it's about how to interpret an
+// RPC response, not the endpoint itself), so it applies uniformly across every endpoint in the pool.
+function isSkippedSlotError(
+  e: unknown,
+  treatLongTermStorageSkipAsSkipped: boolean,
+): boolean {
+  if (isSolanaError(e, SOLANA_ERROR__JSON_RPC__SERVER_ERROR_SLOT_SKIPPED)) {
+    return true;
+  }
+  return (
+    treatLongTermStorageSkipAsSkipped &&
+    isSolanaError(
+      e,
+      SOLANA_ERROR__JSON_RPC__SERVER_ERROR_LONG_TERM_STORAGE_SLOT_SKIPPED,
+    )
+  );
 }
 
 export class SolanaApi {
@@ -47,6 +65,7 @@ export class SolanaApi {
   // This is used within the sandbox when HTTP is used
   #genesisBlockHash: string;
   #requestTimeout: number;
+  #treatLongTermStorageSkipAsSkipped: boolean;
 
   /**
    * @param {string} endpoint - The endpoint of the RPC provider
@@ -60,17 +79,20 @@ export class SolanaApi {
     private eventEmitter: EventEmitter2,
     readonly decoder: SolanaDecoder,
     requestTimeout: number = REQUEST_TIMEOUT,
+    treatLongTermStorageSkipAsSkipped = true,
   ) {
     this.#client = client;
     this.#genesisBlockHash = genesisHash;
     this.#requestTimeout = requestTimeout;
+    this.#treatLongTermStorageSkipAsSkipped = treatLongTermStorageSkipAsSkipped;
   }
 
   static async create(
     endpoint: string,
     eventEmitter: EventEmitter2,
     decoder: SolanaDecoder,
     config?: ISolanaEndpointConfig,
+    treatLongTermStorageSkipAsSkipped = true,
   ): Promise<SolanaApi> {
     try {
       // Keep-Alive is enabled by default, for more details see:
@@ -100,6 +122,7 @@ export class SolanaApi {
         eventEmitter,
         decoder,
         config?.requestTimeout,
+        treatLongTermStorageSkipAsSkipped,
       );
     } catch (e) {
       console.error('CrateSoalana API', e);
@@ -176,7 +199,7 @@ export class SolanaApi {
         })
         .send({ abortSignal: AbortSignal.timeout(this.#requestTimeout) });
     } catch (e) {
-      if (isSkippedSlotError(e)) {
+      if (isSkippedSlotError(e, this.#treatLongTermStorageSkipAsSkipped)) {
         throw new BlockUnavailableError();
       }
       throw e;
@@ -211,7 +234,7 @@ export class SolanaApi {
         blockNumber,
       );
     } catch (e: any) {
-      if (isSkippedSlotError(e)) {
+      if (isSkippedSlotError(e, this.#treatLongTermStorageSkipAsSkipped)) {
         throw new BlockUnavailableError();
       }
       console.log('Failed to fetch block', blockNumber, e.message);

packages/node/src/yargs.ts

@@ -29,5 +29,12 @@ export const yargsOptions = yargsBuilder({
       type: 'number',
       required: false,
     },
+    treatLongTermStorageSkipAsSkipped: {
+      description:
+        'Treat a SOLANA_ERROR__JSON_RPC__SERVER_ERROR_LONG_TERM_STORAGE_SLOT_SKIPPED (-32009) RPC error the same as a confirmed skipped slot, instead of crashing the node',
+      default: true,
+      type: 'boolean',
+      required: false,
+    },
   },
 });