From c0c0782bceda70d40e14060e58fe8dfbe1ef5b9b Mon Sep 17 00:00:00 2001 From: brech1 <113475764+brech1@users.noreply.github.com> Date: Wed, 15 Apr 2026 20:39:08 -0300 Subject: [PATCH 1/4] feat: nightly tests --- packages/beacon-node/package.json | 1 + .../beacon-node/test/spec/downloadTests.ts | 30 +++++-- packages/spec-test-util/package.json | 5 ++ .../src/downloadNightlyTests.ts | 85 +++++++++++++++++++ packages/spec-test-util/src/downloadTests.ts | 18 +++- 5 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 packages/spec-test-util/src/downloadNightlyTests.ts diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 7e0bcd6c1c15..9b5f067125ce 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -173,6 +173,7 @@ "@types/js-yaml": "^4.0.5", "@types/qs": "^6.9.7", "@types/tmp": "^0.2.3", + "dotenv": "^16.4.5", "js-yaml": "^4.1.0", "rewiremock": "^3.14.5", "rimraf": "^4.4.1", diff --git a/packages/beacon-node/test/spec/downloadTests.ts b/packages/beacon-node/test/spec/downloadTests.ts index 8dc53a0c0f49..f76e0bb8c5bc 100644 --- a/packages/beacon-node/test/spec/downloadTests.ts +++ b/packages/beacon-node/test/spec/downloadTests.ts @@ -1,9 +1,29 @@ +import path from "node:path"; +import {config} from "dotenv"; +import {downloadNightlyTests} from "@lodestar/spec-test-util/downloadNightlyTests"; import {downloadTests} from "@lodestar/spec-test-util/downloadTests"; import {blsSpecTests, ethereumConsensusSpecsTests} from "./specTestVersioning.js"; -for (const downloadTestOpts of [ethereumConsensusSpecsTests, blsSpecTests]) { - downloadTests(downloadTestOpts, console.log).catch((e: Error) => { - console.error(e); - process.exit(1); - }); +const onError = (e: Error): void => { + console.error(e); + process.exit(1); +}; + +const [nightlyArg, repo, branch] = process.argv.slice(2); + +if (nightlyArg) { + config({path: path.join(import.meta.dirname, "../../../../.env")}); + + const opts = { + ...ethereumConsensusSpecsTests, + ...(repo && {specTestsRepoUrl: `https://github.com/${repo}`}), + ...(branch && {branch}), + }; + + downloadTests(blsSpecTests, console.log).catch(onError); + downloadNightlyTests(opts, console.log, nightlyArg).catch(onError); +} else { + for (const opts of [ethereumConsensusSpecsTests, blsSpecTests]) { + downloadTests(opts, console.log).catch(onError); + } } diff --git a/packages/spec-test-util/package.json b/packages/spec-test-util/package.json index c4b8264e2a22..29ffc6b1777f 100644 --- a/packages/spec-test-util/package.json +++ b/packages/spec-test-util/package.json @@ -19,6 +19,11 @@ "bun": "./src/downloadTests.ts", "types": "./lib/downloadTests.d.ts", "import": "./lib/downloadTests.js" + }, + "./downloadNightlyTests": { + "bun": "./src/downloadNightlyTests.ts", + "types": "./lib/downloadNightlyTests.d.ts", + "import": "./lib/downloadNightlyTests.js" } }, "files": [ diff --git a/packages/spec-test-util/src/downloadNightlyTests.ts b/packages/spec-test-util/src/downloadNightlyTests.ts new file mode 100644 index 000000000000..0a347f220dab --- /dev/null +++ b/packages/spec-test-util/src/downloadNightlyTests.ts @@ -0,0 +1,85 @@ +import {fetch} from "@lodestar/utils"; +import {downloadGenericSpecTests} from "./downloadTests.js"; + +type WorkflowRunsResponse = {workflow_runs: {id: number}[]}; +type ArtifactsListResponse = {artifacts: {archive_download_url: string; expired: boolean; name: string}[]}; + +async function ghApiFetch(endpoint: string, token: string): Promise { + const res = await fetch(`https://api.github.com${endpoint}`, { + headers: {Authorization: `token ${token}`, Accept: "application/vnd.github+json"}, + signal: AbortSignal.timeout(30_000), + }); + + if (!res.ok) { + throw new Error( + res.status === 401 ? "GITHUB_TOKEN is invalid or expired" : `GitHub API ${res.status} (${endpoint})` + ); + } + + return res.json() as Promise; +} + +async function resolveNightlyRunId(repo: string, token: string, date?: string, branch?: string): Promise { + const query = "status=success&per_page=1" + (branch ? `&branch=${branch}` : "") + (date ? `&created=${date}` : ""); + const {workflow_runs} = await ghApiFetch( + `/repos/${repo}/actions/workflows/tests.yml/runs?${query}`, + token + ); + + const runId = workflow_runs[0]?.id; + if (!runId) { + throw new Error(`No successful run found${date ? ` on ${date}` : ""} for ${repo}${branch ? ` (${branch})` : ""}`); + } + return runId; +} + +export async function downloadNightlyTests( + opts: {specTestsRepoUrl: string; outputDir: string; testsToDownload: string[]; branch?: string}, + log: (msg: string) => void, + date?: string +): Promise { + const token = process.env.GITHUB_TOKEN; + if (!token) throw new Error("GITHUB_TOKEN is required for nightly downloads"); + + const resolvedDate = date === "latest" || !date ? undefined : date; + if (resolvedDate && !/^\d{4}-\d{2}-\d{2}$/.test(resolvedDate)) { + throw new Error(`Invalid date: "${date}". Expected "latest" or YYYY-MM-DD`); + } + + const repo = new URL(opts.specTestsRepoUrl).pathname.slice(1).replace(/\/$/, ""); + const runId = await resolveNightlyRunId(repo, token, resolvedDate, opts.branch); + log(`Resolved nightly${resolvedDate ? ` ${resolvedDate}` : ""} to run ${runId}`); + + const {artifacts} = await ghApiFetch( + `/repos/${repo}/actions/runs/${runId}/artifacts`, + token + ); + + const urlByTest: Record = {}; + const available: string[] = []; + for (const test of opts.testsToDownload) { + const artifact = artifacts.find((a) => a.name === `${test}.tar.gz` && !a.expired); + if (artifact) { + urlByTest[test] = artifact.archive_download_url; + available.push(test); + } else { + log(`Skipping ${test} (not found in run ${runId})`); + } + } + + if (available.length === 0) throw new Error(`No matching artifacts found in run ${runId}`); + + const authInit: RequestInit = {headers: {Authorization: `token ${token}`, Accept: "application/vnd.github+json"}}; + + await downloadGenericSpecTests( + { + specVersion: `nightly-${runId}`, + specTestsRepoUrl: opts.specTestsRepoUrl, + outputDir: opts.outputDir, + testsToDownload: available, + testUrls: urlByTest, + fetchInit: authInit, + }, + log + ); +} diff --git a/packages/spec-test-util/src/downloadTests.ts b/packages/spec-test-util/src/downloadTests.ts index 933d3f6e0448..8e35213f654c 100644 --- a/packages/spec-test-util/src/downloadTests.ts +++ b/packages/spec-test-util/src/downloadTests.ts @@ -25,6 +25,8 @@ export interface DownloadGenericTestsOptions { outputDir: string; specTestsRepoUrl: string; testsToDownload: TestNames[]; + testUrls?: Record; + fetchInit?: RequestInit; } /** @@ -39,7 +41,14 @@ export async function downloadTests(opts: DownloadTestsOptions, log: (msg: strin * Used by spec tests and SlashingProtectionInterchangeTest */ export async function downloadGenericSpecTests( - {specVersion, specTestsRepoUrl, outputDir, testsToDownload}: DownloadGenericTestsOptions, + { + specVersion, + specTestsRepoUrl, + outputDir, + testsToDownload, + testUrls, + fetchInit, + }: DownloadGenericTestsOptions, log: (msg: string) => void = logEmpty ): Promise { log(`outputDir = ${outputDir}`); @@ -62,12 +71,13 @@ export async function downloadGenericSpecTests( await Promise.all( testsToDownload.map(async (test) => { - const url = `${specTestsRepoUrl ?? defaultSpecTestsRepoUrl}/releases/download/${specVersion}/${test}.tar.gz`; + const defaultUrl = `${specTestsRepoUrl ?? defaultSpecTestsRepoUrl}/releases/download/${specVersion}/${test}.tar.gz`; const tarball = path.join(outputDir, `${test}.tar.gz`); await retry( async () => { - const res = await fetch(url, {signal: AbortSignal.timeout(30 * 60 * 1000)}); + const url = testUrls?.[test] ?? defaultUrl; + const res = await fetch(url, {signal: AbortSignal.timeout(30 * 60 * 1000), ...fetchInit}); if (!res.ok) { throw new Error(`Failed to download file from ${url}: ${res.status} ${res.statusText}`); @@ -95,7 +105,7 @@ export async function downloadGenericSpecTests( { retries: 3, onRetry: (e, attempt) => { - log(`Download attempt ${attempt} for ${url} failed: ${e.message}`); + log(`Download attempt ${attempt} for ${test} failed: ${e.message}`); }, } ); From c2f06a1b2c9a750f49722cd7d2eb7841d9ff5b89 Mon Sep 17 00:00:00 2001 From: brech1 <113475764+brech1@users.noreply.github.com> Date: Thu, 16 Apr 2026 10:11:14 -0300 Subject: [PATCH 2/4] fix: review requests --- .../beacon-node/test/spec/downloadTests.ts | 22 +++++++++---------- .../src/downloadNightlyTests.ts | 14 ++++++------ 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/packages/beacon-node/test/spec/downloadTests.ts b/packages/beacon-node/test/spec/downloadTests.ts index f76e0bb8c5bc..a3302fbc606e 100644 --- a/packages/beacon-node/test/spec/downloadTests.ts +++ b/packages/beacon-node/test/spec/downloadTests.ts @@ -4,14 +4,10 @@ import {downloadNightlyTests} from "@lodestar/spec-test-util/downloadNightlyTest import {downloadTests} from "@lodestar/spec-test-util/downloadTests"; import {blsSpecTests, ethereumConsensusSpecsTests} from "./specTestVersioning.js"; -const onError = (e: Error): void => { - console.error(e); - process.exit(1); -}; - -const [nightlyArg, repo, branch] = process.argv.slice(2); +const [date, repo, branch] = process.argv.slice(2); +const downloads = [downloadTests(blsSpecTests, console.log)]; -if (nightlyArg) { +if (date) { config({path: path.join(import.meta.dirname, "../../../../.env")}); const opts = { @@ -20,10 +16,12 @@ if (nightlyArg) { ...(branch && {branch}), }; - downloadTests(blsSpecTests, console.log).catch(onError); - downloadNightlyTests(opts, console.log, nightlyArg).catch(onError); + downloads.push(downloadNightlyTests(opts, console.log, date)); } else { - for (const opts of [ethereumConsensusSpecsTests, blsSpecTests]) { - downloadTests(opts, console.log).catch(onError); - } + downloads.push(downloadTests(ethereumConsensusSpecsTests, console.log)); } + +await Promise.all(downloads).catch((e: Error) => { + console.error(e); + process.exit(1); +}); diff --git a/packages/spec-test-util/src/downloadNightlyTests.ts b/packages/spec-test-util/src/downloadNightlyTests.ts index 0a347f220dab..8cf24cdf247d 100644 --- a/packages/spec-test-util/src/downloadNightlyTests.ts +++ b/packages/spec-test-util/src/downloadNightlyTests.ts @@ -20,9 +20,12 @@ async function ghApiFetch(endpoint: string, token: string): Promise { } async function resolveNightlyRunId(repo: string, token: string, date?: string, branch?: string): Promise { - const query = "status=success&per_page=1" + (branch ? `&branch=${branch}` : "") + (date ? `&created=${date}` : ""); - const {workflow_runs} = await ghApiFetch( - `/repos/${repo}/actions/workflows/tests.yml/runs?${query}`, + const params = new URLSearchParams({status: "success", per_page: "1"}); + if (branch) params.append("branch", branch); + if (date) params.append("created", date); + + const { workflow_runs } = await ghApiFetch( + `/repos/${repo}/actions/workflows/tests.yml/runs?${params}`, token ); @@ -50,10 +53,7 @@ export async function downloadNightlyTests( const runId = await resolveNightlyRunId(repo, token, resolvedDate, opts.branch); log(`Resolved nightly${resolvedDate ? ` ${resolvedDate}` : ""} to run ${runId}`); - const {artifacts} = await ghApiFetch( - `/repos/${repo}/actions/runs/${runId}/artifacts`, - token - ); + const {artifacts} = await ghApiFetch(`/repos/${repo}/actions/runs/${runId}/artifacts`, token); const urlByTest: Record = {}; const available: string[] = []; From b4c68811cb3d73851a335f72ac7b3e4ac872e4eb Mon Sep 17 00:00:00 2001 From: lodekeeper Date: Thu, 23 Apr 2026 21:50:56 +0000 Subject: [PATCH 3/4] chore: update pnpm-lock.yaml for dotenv devDependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dotenv@^16.4.5 was added to packages/beacon-node/package.json in the nightly-spec-tests support work but pnpm-lock.yaml wasn't regenerated, causing `pnpm install --frozen-lockfile` to fail in CI with ERR_PNPM_OUTDATED_LOCKFILE. 🤖 Generated with AI assistance --- pnpm-lock.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1d2dd20db720..c6afe08246cb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -406,6 +406,9 @@ importers: '@types/tmp': specifier: ^0.2.3 version: 0.2.3 + dotenv: + specifier: ^16.4.5 + version: 16.4.5 js-yaml: specifier: ^4.1.0 version: 4.1.1 From f84920bd23ef262cebb32c6a7a64ffe48dda5d91 Mon Sep 17 00:00:00 2001 From: lodekeeper Date: Thu, 23 Apr 2026 22:15:45 +0000 Subject: [PATCH 4/4] test: add epoch_boundary sanity runner case for nightly spec tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream consensus-specs #5127 adds epoch_boundary sanity tests for Gloas (and post-Gloas `heze`). The case format is identical to blocks (pre / post / blocks_N / meta.blocks_count), so route it through the existing sanityBlocks runner. Surfaced by pnpm test:spec:minimal against nightly artifacts downloaded via the new nightly workflow in ChainSafe/lodestar#9221. 🤖 Generated with AI assistance --- packages/beacon-node/test/spec/presets/sanity.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/beacon-node/test/spec/presets/sanity.test.ts b/packages/beacon-node/test/spec/presets/sanity.test.ts index 1d35ab48ecf8..c7562a16411d 100644 --- a/packages/beacon-node/test/spec/presets/sanity.test.ts +++ b/packages/beacon-node/test/spec/presets/sanity.test.ts @@ -23,6 +23,7 @@ const sanity: TestRunnerFn = (fork, testName, testSuit case "slots": return sanitySlots(fork, testName, testSuite); case "blocks": + case "epoch_boundary": return sanityBlocks(fork, testName, testSuite); default: throw Error(`Unknown sanity test ${testName}`);