From 6378940ae65150571f3b997169c8fba1701392e6 Mon Sep 17 00:00:00 2001 From: aIbrahiim Date: Wed, 15 Jul 2026 10:34:16 +0300 Subject: [PATCH] Add Spark JVM --add-opens for (Nexmark, TPC-DS, PortableJar) --- .../beam_PostCommit_Java_Nexmark_Spark.json | 4 ++++ .../beam_PostCommit_Java_Tpcds_Spark.json | 4 ++++ .../beam_PostCommit_PortableJar_Spark.json | 4 ++++ runners/portability/test_pipeline_jar.sh | 11 ++++++++++- sdks/java/testing/nexmark/build.gradle | 14 ++++++++++++++ sdks/java/testing/tpcds/build.gradle | 14 ++++++++++++++ 6 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 .github/trigger_files/beam_PostCommit_Java_Nexmark_Spark.json create mode 100644 .github/trigger_files/beam_PostCommit_Java_Tpcds_Spark.json create mode 100644 .github/trigger_files/beam_PostCommit_PortableJar_Spark.json diff --git a/.github/trigger_files/beam_PostCommit_Java_Nexmark_Spark.json b/.github/trigger_files/beam_PostCommit_Java_Nexmark_Spark.json new file mode 100644 index 000000000000..e3d6056a5de9 --- /dev/null +++ b/.github/trigger_files/beam_PostCommit_Java_Nexmark_Spark.json @@ -0,0 +1,4 @@ +{ + "comment": "Modify this file in a trivial way to cause this test suite to run", + "modification": 1 +} diff --git a/.github/trigger_files/beam_PostCommit_Java_Tpcds_Spark.json b/.github/trigger_files/beam_PostCommit_Java_Tpcds_Spark.json new file mode 100644 index 000000000000..e3d6056a5de9 --- /dev/null +++ b/.github/trigger_files/beam_PostCommit_Java_Tpcds_Spark.json @@ -0,0 +1,4 @@ +{ + "comment": "Modify this file in a trivial way to cause this test suite to run", + "modification": 1 +} diff --git a/.github/trigger_files/beam_PostCommit_PortableJar_Spark.json b/.github/trigger_files/beam_PostCommit_PortableJar_Spark.json new file mode 100644 index 000000000000..e3d6056a5de9 --- /dev/null +++ b/.github/trigger_files/beam_PostCommit_PortableJar_Spark.json @@ -0,0 +1,4 @@ +{ + "comment": "Modify this file in a trivial way to cause this test suite to run", + "modification": 1 +} diff --git a/runners/portability/test_pipeline_jar.sh b/runners/portability/test_pipeline_jar.sh index f01d67b6580c..871a604e319f 100755 --- a/runners/portability/test_pipeline_jar.sh +++ b/runners/portability/test_pipeline_jar.sh @@ -123,7 +123,16 @@ OUTPUT_JAR="test-pipeline-${RUNNER}-$(date +%Y%m%d-%H%M%S).jar" if [[ "$TEST_EXIT_CODE" -eq 0 ]]; then # Execute the jar - java -jar $OUTPUT_JAR || TEST_EXIT_CODE=$? + JAVA_ARGS=() + if [[ "$RUNNER" = "SparkRunner" ]]; then + JAVA_ARGS+=( + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED" + "--add-opens=java.base/java.nio=ALL-UNNAMED" + "--add-opens=java.base/java.util=ALL-UNNAMED" + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED" + ) + fi + java "${JAVA_ARGS[@]}" -jar $OUTPUT_JAR || TEST_EXIT_CODE=$? fi rm -rf $ENV_DIR diff --git a/sdks/java/testing/nexmark/build.gradle b/sdks/java/testing/nexmark/build.gradle index bd917a3935ad..b554e9d9297f 100644 --- a/sdks/java/testing/nexmark/build.gradle +++ b/sdks/java/testing/nexmark/build.gradle @@ -114,6 +114,19 @@ if (isSparkRunner) { } } +def sparkJvmArgs() { + def testJavaVer = project.findProperty('testJavaVersion') ? (project.property('testJavaVersion') as int) : JavaVersion.current().majorVersion.toInteger() + if (testJavaVer >= 17) { + return [ + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED" + ] + } + return [] +} + def getNexmarkArgs = { def nexmarkArgsStr = project.findProperty(nexmarkArgsProperty) ?: "" def nexmarkArgsList = new ArrayList() @@ -179,6 +192,7 @@ task run(type: JavaExec) { systemProperty "spark.ui.showConsoleProgress", "false" // Dataset runner only systemProperty "spark.sql.shuffle.partitions", "4" + jvmArgs += sparkJvmArgs() } mainClass = "org.apache.beam.sdk.nexmark.Main" diff --git a/sdks/java/testing/tpcds/build.gradle b/sdks/java/testing/tpcds/build.gradle index 6a05df96ab20..15fdd480f076 100644 --- a/sdks/java/testing/tpcds/build.gradle +++ b/sdks/java/testing/tpcds/build.gradle @@ -104,6 +104,19 @@ if (isSpark) { } } +def sparkJvmArgs() { + def testJavaVer = project.findProperty('testJavaVersion') ? (project.property('testJavaVersion') as int) : JavaVersion.current().majorVersion.toInteger() + if (testJavaVer >= 17) { + return [ + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED" + ] + } + return [] +} + // Execute the TPC-DS queries or suites via Gradle. // // Parameters: @@ -141,6 +154,7 @@ task run(type: JavaExec) { // Dataset runner only systemProperty "spark.sql.shuffle.partitions", "4" systemProperty "spark.sql.adaptive.enabled", "false" // high overhead for complex queries + jvmArgs += sparkJvmArgs() } mainClass = "org.apache.beam.sdk.tpcds.BeamTpcds"