From 8ff8bbfde7bb8830dc7f83e8b630bfedc5a601a9 Mon Sep 17 00:00:00 2001 From: Arran Schlosberg Date: Fri, 2 Jan 2026 11:24:43 +0000 Subject: [PATCH 1/6] feat: flag to fail instead of skip if execution spec tests unavailable --- .github/workflows/go.yml | 2 +- tests/block_test.go | 3 +-- tests/spec.libevm_test.go | 55 +++++++++++++++++++++++++++++++++++++++ tests/state_test.go | 3 +-- 4 files changed, 58 insertions(+), 5 deletions(-) create mode 100644 tests/spec.libevm_test.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index b39674fb371f..30d625d785da 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -36,7 +36,7 @@ jobs: go list ./... | grep -P "${FLAKY_REGEX}" | xargs -n 1 go test -short; - name: Run non-flaky tests concurrently run: | - go test -short $(go list ./... | grep -Pv "${FLAKY_REGEX}"); + go test -short -libevm.fail_without_execution_spec_tests $(go list ./... | grep -Pv "${FLAKY_REGEX}"); go_test_tooling: runs-on: ubuntu-latest diff --git a/tests/block_test.go b/tests/block_test.go index 2f31007c193e..6ddd80cc692a 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -21,7 +21,6 @@ import ( "runtime" "testing" - "github.com/ava-labs/libevm/common" "github.com/ava-labs/libevm/core/rawdb" ) @@ -63,7 +62,7 @@ func TestBlockchain(t *testing.T) { // TestExecutionSpecBlocktests runs the test fixtures from execution-spec-tests. func TestExecutionSpecBlocktests(t *testing.T) { - if !common.FileExist(executionSpecBlockchainTestDir) { + if !executionSpecTestDirExists(t, executionSpecBlockchainTestDir) { t.Skipf("directory %s does not exist", executionSpecBlockchainTestDir) } bt := new(testMatcher) diff --git a/tests/spec.libevm_test.go b/tests/spec.libevm_test.go new file mode 100644 index 000000000000..beb634b9a20b --- /dev/null +++ b/tests/spec.libevm_test.go @@ -0,0 +1,55 @@ +// Copyright 2026 the libevm authors. +// +// The libevm additions to go-ethereum are free software: you can redistribute +// them and/or modify them under the terms of the GNU Lesser General Public License +// as published by the Free Software Foundation, either version 3 of the License, +// or (at your option) any later version. +// +// The libevm additions are distributed in the hope that they will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser +// General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see +// . + +package tests + +import ( + "flag" + "testing" + + "github.com/ava-labs/libevm/common" +) + +func TestMain(m *testing.M) { + flag.BoolVar( + &fatalIfNoSpecTestDir, + "libevm.fail_without_execution_spec_tests", + false, + "If true and no execution spec tests are found then respective tests will fail", + ) + flag.Parse() + + m.Run() +} + +var fatalIfNoSpecTestDir bool + +// executionSpecTestDirExists is equivalent to [common.FileExist] except that if +// it were to return false and [fatalIfNoSpecTestDir] is true then it results in +// a fatal error. See the flag in [TestMain]. +// +// Without this, the block and state execution spec tests fail silently when not +// present. This resulted in them not being run on libevm for over a year. +func executionSpecTestDirExists(t *testing.T, dirPath string) bool { + t.Helper() + if common.FileExist(dirPath) { + return true + } + if fatalIfNoSpecTestDir { + t.Fatalf("directory %q does not exist", dirPath) + } + return false +} diff --git a/tests/state_test.go b/tests/state_test.go index 888717980b47..e86d3659d439 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -30,7 +30,6 @@ import ( "testing" "time" - "github.com/ava-labs/libevm/common" "github.com/ava-labs/libevm/core" "github.com/ava-labs/libevm/core/rawdb" "github.com/ava-labs/libevm/core/types" @@ -88,7 +87,7 @@ func TestLegacyState(t *testing.T) { // TestExecutionSpecState runs the test fixtures from execution-spec-tests. func TestExecutionSpecState(t *testing.T) { - if !common.FileExist(executionSpecStateTestDir) { + if !executionSpecTestDirExists(t, executionSpecStateTestDir) { t.Skipf("directory %s does not exist", executionSpecStateTestDir) } st := new(testMatcher) From 4b59fe7e386cf7468088aca80b2a89aa7929bb23 Mon Sep 17 00:00:00 2001 From: Arran Schlosberg Date: Fri, 2 Jan 2026 11:40:04 +0000 Subject: [PATCH 2/6] fix: run spec tests in a separate step --- .github/workflows/go.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 30d625d785da..d059c62a1052 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -36,7 +36,10 @@ jobs: go list ./... | grep -P "${FLAKY_REGEX}" | xargs -n 1 go test -short; - name: Run non-flaky tests concurrently run: | - go test -short -libevm.fail_without_execution_spec_tests $(go list ./... | grep -Pv "${FLAKY_REGEX}"); + go test -short $(go list ./... | grep -Pv "${FLAKY_REGEX}" | grep -Pv "ava-labs/libevm/tests$"); + - name: Run execution spec tests + run: | # Without the flag these would silently skip if test cases aren't found + go test -short ./tests -libevm.fail_without_execution_spec_tests; go_test_tooling: runs-on: ubuntu-latest From e19d5dfc46e1ef8726b7047317da3004b2c9e513 Mon Sep 17 00:00:00 2001 From: Arran Schlosberg Date: Fri, 2 Jan 2026 11:46:41 +0000 Subject: [PATCH 3/6] refactor: run `./tests` directory first + document --- .github/workflows/go.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index d059c62a1052..68386cb19f0d 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -34,12 +34,12 @@ jobs: run: | # Upstream flakes are race conditions exacerbated by concurrent tests go list ./... | grep -P "${FLAKY_REGEX}" | xargs -n 1 go test -short; - - name: Run non-flaky tests concurrently - run: | - go test -short $(go list ./... | grep -Pv "${FLAKY_REGEX}" | grep -Pv "ava-labs/libevm/tests$"); - - name: Run execution spec tests + - name: Run ./tests directory tests run: | # Without the flag these would silently skip if test cases aren't found go test -short ./tests -libevm.fail_without_execution_spec_tests; + - name: Run non-flaky tests concurrently + run: | # The ./tests directory is run above to allow use of a specific flag + go test -short $(go list ./... | grep -Pv "${FLAKY_REGEX}" | grep -Pv "ava-labs/libevm/tests$"); go_test_tooling: runs-on: ubuntu-latest From 87e5d9d34bfb22a1faf6cd544fcb358e81ac65e2 Mon Sep 17 00:00:00 2001 From: Arran Schlosberg Date: Fri, 2 Jan 2026 11:49:39 +0000 Subject: [PATCH 4/6] chore: rename file --- tests/{spec.libevm_test.go => noskip.libevm_test.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{spec.libevm_test.go => noskip.libevm_test.go} (100%) diff --git a/tests/spec.libevm_test.go b/tests/noskip.libevm_test.go similarity index 100% rename from tests/spec.libevm_test.go rename to tests/noskip.libevm_test.go From 5f5793893d512853615c396f1b966763f3694beb Mon Sep 17 00:00:00 2001 From: Arran Schlosberg Date: Fri, 2 Jan 2026 12:08:54 +0000 Subject: [PATCH 5/6] feat: also fail when submodule missing --- .github/workflows/go.yml | 4 ++-- tests/block_test.go | 5 +++-- tests/init_test.go | 2 +- tests/noskip.libevm_test.go | 33 +++++++++++++-------------------- tests/state_test.go | 7 ++++--- 5 files changed, 23 insertions(+), 28 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 68386cb19f0d..97994f83b515 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -36,9 +36,9 @@ jobs: go list ./... | grep -P "${FLAKY_REGEX}" | xargs -n 1 go test -short; - name: Run ./tests directory tests run: | # Without the flag these would silently skip if test cases aren't found - go test -short ./tests -libevm.fail_without_execution_spec_tests; + go test -short ./tests -libevm.fail_instead_of_skip; - name: Run non-flaky tests concurrently - run: | # The ./tests directory is run above to allow use of a specific flag + run: | # The ./tests directory is run separetely to allow use of a specific flag go test -short $(go list ./... | grep -Pv "${FLAKY_REGEX}" | grep -Pv "ava-labs/libevm/tests$"); go_test_tooling: diff --git a/tests/block_test.go b/tests/block_test.go index 6ddd80cc692a..3d4ae1516bf0 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -21,6 +21,7 @@ import ( "runtime" "testing" + "github.com/ava-labs/libevm/common" "github.com/ava-labs/libevm/core/rawdb" ) @@ -62,8 +63,8 @@ func TestBlockchain(t *testing.T) { // TestExecutionSpecBlocktests runs the test fixtures from execution-spec-tests. func TestExecutionSpecBlocktests(t *testing.T) { - if !executionSpecTestDirExists(t, executionSpecBlockchainTestDir) { - t.Skipf("directory %s does not exist", executionSpecBlockchainTestDir) + if !common.FileExist(executionSpecBlockchainTestDir) { + failOrSkip(t, "directory %s does not exist", executionSpecBlockchainTestDir) } bt := new(testMatcher) diff --git a/tests/init_test.go b/tests/init_test.go index 3028bd14a337..6faca4717b18 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -197,7 +197,7 @@ func (tm *testMatcher) walk(t *testing.T, dir string, runTest interface{}) { dirinfo, err := os.Stat(dir) if os.IsNotExist(err) || !dirinfo.IsDir() { fmt.Fprintf(os.Stderr, "can't find test files in %s, did you clone the tests submodule?\n", dir) - t.Skip("missing test files") + failOrSkip(t, "missing test files") } err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { name := filepath.ToSlash(strings.TrimPrefix(path, dir+string(filepath.Separator))) diff --git a/tests/noskip.libevm_test.go b/tests/noskip.libevm_test.go index beb634b9a20b..4ce7ac23bb05 100644 --- a/tests/noskip.libevm_test.go +++ b/tests/noskip.libevm_test.go @@ -19,37 +19,30 @@ package tests import ( "flag" "testing" - - "github.com/ava-labs/libevm/common" ) func TestMain(m *testing.M) { flag.BoolVar( - &fatalIfNoSpecTestDir, - "libevm.fail_without_execution_spec_tests", + &failInsteadOfSkip, + "libevm.fail_instead_of_skip", false, - "If true and no execution spec tests are found then respective tests will fail", + "If true and test cases are missing then respective tests fail instead of skipping", ) flag.Parse() m.Run() } -var fatalIfNoSpecTestDir bool +var failInsteadOfSkip bool -// executionSpecTestDirExists is equivalent to [common.FileExist] except that if -// it were to return false and [fatalIfNoSpecTestDir] is true then it results in -// a fatal error. See the flag in [TestMain]. -// -// Without this, the block and state execution spec tests fail silently when not -// present. This resulted in them not being run on libevm for over a year. -func executionSpecTestDirExists(t *testing.T, dirPath string) bool { - t.Helper() - if common.FileExist(dirPath) { - return true - } - if fatalIfNoSpecTestDir { - t.Fatalf("directory %q does not exist", dirPath) +// failOrSkip propagates its arguments to Skipf or Fatalf, depending on the +// value of [failInsteadOfSkip], defaulting to skipping for backwards +// compatibility. See [TestMain] for the respective flag. +func failOrSkip(tb testing.TB, format string, args ...any) { + tb.Helper() + fn := tb.Skipf + if failInsteadOfSkip { + fn = tb.Fatalf } - return false + fn(format, args...) } diff --git a/tests/state_test.go b/tests/state_test.go index e86d3659d439..a0b47a9249f3 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -30,6 +30,7 @@ import ( "testing" "time" + "github.com/ava-labs/libevm/common" "github.com/ava-labs/libevm/core" "github.com/ava-labs/libevm/core/rawdb" "github.com/ava-labs/libevm/core/types" @@ -87,8 +88,8 @@ func TestLegacyState(t *testing.T) { // TestExecutionSpecState runs the test fixtures from execution-spec-tests. func TestExecutionSpecState(t *testing.T) { - if !executionSpecTestDirExists(t, executionSpecStateTestDir) { - t.Skipf("directory %s does not exist", executionSpecStateTestDir) + if !common.FileExist(executionSpecStateTestDir) { + failOrSkip(t, "directory %s does not exist", executionSpecStateTestDir) } st := new(testMatcher) @@ -197,7 +198,7 @@ func BenchmarkEVM(b *testing.B) { dirinfo, err := os.Stat(dir) if os.IsNotExist(err) || !dirinfo.IsDir() { fmt.Fprintf(os.Stderr, "can't find test files in %s, did you clone the evm-benchmarks submodule?\n", dir) - b.Skip("missing test files") + failOrSkip(b, "missing test files") } err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if info.IsDir() { From bce868dcc12f92251e3f73c0449a3fa1a7ac2ecc Mon Sep 17 00:00:00 2001 From: Arran Schlosberg Date: Fri, 2 Jan 2026 12:11:14 +0000 Subject: [PATCH 6/6] doc: remove redundant comment --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 97994f83b515..aa92b633b747 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -35,7 +35,7 @@ jobs: | # Upstream flakes are race conditions exacerbated by concurrent tests go list ./... | grep -P "${FLAKY_REGEX}" | xargs -n 1 go test -short; - name: Run ./tests directory tests - run: | # Without the flag these would silently skip if test cases aren't found + run: | go test -short ./tests -libevm.fail_instead_of_skip; - name: Run non-flaky tests concurrently run: | # The ./tests directory is run separetely to allow use of a specific flag