diff --git a/.gitignore b/.gitignore index 9a1780cfa..1c3ed3d0e 100644 --- a/.gitignore +++ b/.gitignore @@ -87,6 +87,12 @@ web_modules/ # TypeScript cache *.tsbuildinfo +# Stray tsc output if it ever lands next to source instead of dist/ (e.g. from a stale +# incremental build info predating outDir, or a non-project-mode tsc invocation) +packages/*/src/**/*.js +packages/*/src/**/*.js.map +packages/*/src/**/*.d.ts + # Optional npm cache directory .npm diff --git a/packages/node/src/configure/NodeConfig.ts b/packages/node/src/configure/NodeConfig.ts index bee97691b..51d412f8d 100644 --- a/packages/node/src/configure/NodeConfig.ts +++ b/packages/node/src/configure/NodeConfig.ts @@ -3,8 +3,9 @@ import { IConfig, NodeConfig } from '@subql/node-core'; -// NOTE to extend IConfig, change the type to interface -export type ISolanaConfig = IConfig; +export interface ISolanaConfig extends IConfig { + treatLongTermStorageSkipAsSkipped: boolean; +} export class SolanaNodeConfig extends NodeConfig { /** @@ -22,4 +23,8 @@ export class SolanaNodeConfig extends NodeConfig { // Rebuild with internal config super((config as any)._config, (config as any)._isTest); } + + get treatLongTermStorageSkipAsSkipped(): boolean { + return this._config.treatLongTermStorageSkipAsSkipped ?? true; + } } diff --git a/packages/node/src/solana/api.connection.ts b/packages/node/src/solana/api.connection.ts index 8a11e0619..858a12d09 100644 --- a/packages/node/src/solana/api.connection.ts +++ b/packages/node/src/solana/api.connection.ts @@ -47,8 +47,15 @@ export class SolanaApiConnection eventEmitter: EventEmitter2, decoder: SolanaDecoder, config?: ISolanaEndpointConfig, + treatLongTermStorageSkipAsSkipped?: boolean, ): Promise { - const api = await SolanaApi.create(endpoint, eventEmitter, decoder, config); + const api = await SolanaApi.create( + endpoint, + eventEmitter, + decoder, + config, + treatLongTermStorageSkipAsSkipped, + ); return new SolanaApiConnection(api, fetchBlocksBatches); } diff --git a/packages/node/src/solana/api.service.solana.ts b/packages/node/src/solana/api.service.solana.ts index 560633bb3..7a49850a0 100644 --- a/packages/node/src/solana/api.service.solana.ts +++ b/packages/node/src/solana/api.service.solana.ts @@ -19,6 +19,7 @@ import { ISolanaEndpointConfig, SolanaBlock, } from '@subql/types-solana'; +import { SolanaNodeConfig } from '../configure/NodeConfig'; import { SubqueryProject } from '../configure/SubqueryProject'; import { SolanaApiConnection, FetchFunc, GetFetchFunc } from './api.connection'; import { SolanaApi, SolanaSafeApi } from './api.solana'; @@ -78,6 +79,9 @@ export class SolanaApiService extends ApiService< apiService.updateBlockFetching(); + const treatLongTermStorageSkipAsSkipped = new SolanaNodeConfig(nodeConfig) + .treatLongTermStorageSkipAsSkipped; + await apiService.createConnections(network, (endpoint, config) => SolanaApiConnection.create( endpoint, @@ -85,6 +89,7 @@ export class SolanaApiService extends ApiService< eventEmitter, decoder, config, + treatLongTermStorageSkipAsSkipped, ), ); diff --git a/packages/node/src/solana/api.solana.skip-slot.spec.ts b/packages/node/src/solana/api.solana.skip-slot.spec.ts new file mode 100644 index 000000000..f0e1dfb66 --- /dev/null +++ b/packages/node/src/solana/api.solana.skip-slot.spec.ts @@ -0,0 +1,89 @@ +// Copyright 2020-2025 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { + SOLANA_ERROR__JSON_RPC__SERVER_ERROR_LONG_TERM_STORAGE_SLOT_SKIPPED, + SOLANA_ERROR__JSON_RPC__SERVER_ERROR_SLOT_SKIPPED, + SolanaError, +} from '@solana/errors'; +import { BlockUnavailableError } from '@subql/node-core'; +import { SolanaApi } from './api.solana'; +import { SolanaDecoder } from './decoder'; + +jest.mock('@solana/rpc', () => ({ + createSolanaRpc: jest.fn(), +})); + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { createSolanaRpc } = require('@solana/rpc'); + +function mockRpcClient(getBlockError: unknown) { + return { + getGenesisHash: () => ({ send: () => Promise.resolve('genesis-hash') }), + getBlock: () => ({ send: () => Promise.reject(getBlockError) }), + }; +} + +describe('SolanaApi skipped slot handling', () => { + const eventEmitter = new EventEmitter2(); + const decoder = new SolanaDecoder(); + + it('treats SOLANA_ERROR__JSON_RPC__SERVER_ERROR_SLOT_SKIPPED as a confirmed skip', async () => { + createSolanaRpc.mockReturnValue( + mockRpcClient( + new SolanaError(SOLANA_ERROR__JSON_RPC__SERVER_ERROR_SLOT_SKIPPED, { + __serverMessage: 'Slot 1 was skipped', + }), + ), + ); + const api = await SolanaApi.create( + 'http://localhost', + eventEmitter, + decoder, + ); + await expect(api.fetchBlock(1)).rejects.toBeInstanceOf( + BlockUnavailableError, + ); + }); + + it('treats LONG_TERM_STORAGE_SLOT_SKIPPED as a skip by default', async () => { + createSolanaRpc.mockReturnValue( + mockRpcClient( + new SolanaError( + SOLANA_ERROR__JSON_RPC__SERVER_ERROR_LONG_TERM_STORAGE_SLOT_SKIPPED, + { + __serverMessage: + 'Slot 1 was skipped, or missing in long-term storage', + }, + ), + ), + ); + const api = await SolanaApi.create( + 'http://localhost', + eventEmitter, + decoder, + ); + await expect(api.fetchBlock(1)).rejects.toBeInstanceOf( + BlockUnavailableError, + ); + }); + + it('does not treat LONG_TERM_STORAGE_SLOT_SKIPPED as a skip when disabled at startup', async () => { + const rpcError = new SolanaError( + SOLANA_ERROR__JSON_RPC__SERVER_ERROR_LONG_TERM_STORAGE_SLOT_SKIPPED, + { + __serverMessage: 'Slot 1 was skipped, or missing in long-term storage', + }, + ); + createSolanaRpc.mockReturnValue(mockRpcClient(rpcError)); + const api = await SolanaApi.create( + 'http://localhost', + eventEmitter, + decoder, + undefined, + false, + ); + await expect(api.fetchBlock(1)).rejects.toBe(rpcError); + }); +}); diff --git a/packages/node/src/solana/api.solana.ts b/packages/node/src/solana/api.solana.ts index 0ce2c3d7b..8065f2d41 100644 --- a/packages/node/src/solana/api.solana.ts +++ b/packages/node/src/solana/api.solana.ts @@ -5,6 +5,7 @@ import { EventEmitter2 } from '@nestjs/event-emitter'; import { assertIsAddress } from '@solana/addresses'; import { isSolanaError, + SOLANA_ERROR__JSON_RPC__SERVER_ERROR_LONG_TERM_STORAGE_SLOT_SKIPPED, SOLANA_ERROR__JSON_RPC__SERVER_ERROR_SLOT_SKIPPED, } from '@solana/errors'; import { createSolanaRpc, Rpc } from '@solana/rpc'; @@ -35,10 +36,27 @@ export type SolanaSafeApi = undefined; const REQUEST_TIMEOUT = 30_000; // Solana doesn't produce a block for every slot. For a skipped slot the RPC throws rather than returning null. -// Other "not available" RPC errors (e.g. block not yet rooted on this node, or missing from long-term storage) -// don't reliably mean the slot was skipped, so they're intentionally not treated as a skip here. -function isSkippedSlotError(e: unknown): boolean { - return isSolanaError(e, SOLANA_ERROR__JSON_RPC__SERVER_ERROR_SLOT_SKIPPED); +// BLOCK_NOT_AVAILABLE (block not yet rooted on this node) is intentionally not treated as a skip here, since +// it's often just this endpoint lagging behind and not a confirmed skip. +// LONG_TERM_STORAGE_SLOT_SKIPPED fires once a slot falls outside an RPC's local blockstore retention window and +// its archival (e.g. bigtable) lookup returns not-found. This is ambiguous in general (could be a genuine +// archival gap), but in practice recently-skipped slots commonly surface this way, so it's treated as a skip +// unless disabled via the network-level config. This isn't endpoint-specific (it's about how to interpret an +// RPC response, not the endpoint itself), so it applies uniformly across every endpoint in the pool. +function isSkippedSlotError( + e: unknown, + treatLongTermStorageSkipAsSkipped: boolean, +): boolean { + if (isSolanaError(e, SOLANA_ERROR__JSON_RPC__SERVER_ERROR_SLOT_SKIPPED)) { + return true; + } + return ( + treatLongTermStorageSkipAsSkipped && + isSolanaError( + e, + SOLANA_ERROR__JSON_RPC__SERVER_ERROR_LONG_TERM_STORAGE_SLOT_SKIPPED, + ) + ); } export class SolanaApi { @@ -47,6 +65,7 @@ export class SolanaApi { // This is used within the sandbox when HTTP is used #genesisBlockHash: string; #requestTimeout: number; + #treatLongTermStorageSkipAsSkipped: boolean; /** * @param {string} endpoint - The endpoint of the RPC provider @@ -60,10 +79,12 @@ export class SolanaApi { private eventEmitter: EventEmitter2, readonly decoder: SolanaDecoder, requestTimeout: number = REQUEST_TIMEOUT, + treatLongTermStorageSkipAsSkipped = true, ) { this.#client = client; this.#genesisBlockHash = genesisHash; this.#requestTimeout = requestTimeout; + this.#treatLongTermStorageSkipAsSkipped = treatLongTermStorageSkipAsSkipped; } static async create( @@ -71,6 +92,7 @@ export class SolanaApi { eventEmitter: EventEmitter2, decoder: SolanaDecoder, config?: ISolanaEndpointConfig, + treatLongTermStorageSkipAsSkipped = true, ): Promise { try { // Keep-Alive is enabled by default, for more details see: @@ -100,6 +122,7 @@ export class SolanaApi { eventEmitter, decoder, config?.requestTimeout, + treatLongTermStorageSkipAsSkipped, ); } catch (e) { console.error('CrateSoalana API', e); @@ -176,7 +199,7 @@ export class SolanaApi { }) .send({ abortSignal: AbortSignal.timeout(this.#requestTimeout) }); } catch (e) { - if (isSkippedSlotError(e)) { + if (isSkippedSlotError(e, this.#treatLongTermStorageSkipAsSkipped)) { throw new BlockUnavailableError(); } throw e; @@ -211,7 +234,7 @@ export class SolanaApi { blockNumber, ); } catch (e: any) { - if (isSkippedSlotError(e)) { + if (isSkippedSlotError(e, this.#treatLongTermStorageSkipAsSkipped)) { throw new BlockUnavailableError(); } console.log('Failed to fetch block', blockNumber, e.message); diff --git a/packages/node/src/yargs.ts b/packages/node/src/yargs.ts index 99610e59d..cb19ef515 100644 --- a/packages/node/src/yargs.ts +++ b/packages/node/src/yargs.ts @@ -29,5 +29,12 @@ export const yargsOptions = yargsBuilder({ type: 'number', required: false, }, + treatLongTermStorageSkipAsSkipped: { + description: + 'Treat a SOLANA_ERROR__JSON_RPC__SERVER_ERROR_LONG_TERM_STORAGE_SLOT_SKIPPED (-32009) RPC error the same as a confirmed skipped slot, instead of crashing the node', + default: true, + type: 'boolean', + required: false, + }, }, });