diff --git a/.github/actions/setup-environment-action/action.yml b/.github/actions/setup-environment-action/action.yml index 06633c6c7279..c82ad087fe46 100644 --- a/.github/actions/setup-environment-action/action.yml +++ b/.github/actions/setup-environment-action/action.yml @@ -71,7 +71,7 @@ runs: - name: Install Java if: ${{ inputs.java-version != '' }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: ${{ inputs.java-version == 'default' && '11' || inputs.java-version }} diff --git a/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml b/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml index 7ed3fce19a0e..564f1be187d7 100644 --- a/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml +++ b/.github/workflows/beam_PerformanceTests_xlang_KafkaIO_Python.yml @@ -73,6 +73,9 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: + java-version: | + 17 + 11 python-version: default - name: Set k8s access uses: ./.github/actions/setup-k8s-access @@ -179,6 +182,8 @@ jobs: uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:java:io:expansion-service:shadowJar + arguments: | + -Pjava17Home=$JAVA_HOME_17_X64 # The env variable is created and populated in the test-arguments-action as "_test_arguments_" - name: run integrationTest uses: ./.github/actions/gradle-command-self-hosted-action diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml index 01997b22515d..2c442af725e6 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Dataflow.yml @@ -74,6 +74,9 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: + java-version: | + 17 + 11 python-version: | 3.10 3.14 @@ -84,6 +87,7 @@ jobs: arguments: | -PpythonVersion=3.14 \ -PuseWheelDistribution \ + -Pjava17Home=$JAVA_HOME_17_X64 \ - name: Archive Python Test Results uses: actions/upload-artifact@v7 if: failure() diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml index e4a71ee8edd3..5dc1ed0dbaf2 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_Gcp_Direct.yml @@ -74,6 +74,9 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: + java-version: | + 17 + 11 python-version: | 3.10 3.14 @@ -85,6 +88,8 @@ jobs: uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:python:test-suites:direct:gcpCrossLanguagePostCommit + arguments: | + -Pjava17Home=$JAVA_HOME_17_X64 - name: Archive Python Test Results uses: actions/upload-artifact@v7 if: failure() diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml index 832db23a73fc..7bfe29e4b509 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Dataflow.yml @@ -73,6 +73,9 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: + java-version: | + 17 + 11 python-version: | 3.10 3.14 @@ -84,6 +87,7 @@ jobs: -PpythonVersion=3.14 \ -PuseWheelDistribution \ -PkafkaBootstrapServer=10.128.0.40:9094,10.128.0.28:9094,10.128.0.165:9094 \ + -Pjava17Home=$JAVA_HOME_17_X64 \ - name: Archive Python Test Results uses: actions/upload-artifact@v7 if: failure() diff --git a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml index bb3198f83d2c..ebb891dae457 100644 --- a/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml +++ b/.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml @@ -73,6 +73,9 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: + java-version: | + 17 + 11 python-version: | 3.10 3.14 @@ -80,7 +83,9 @@ jobs: uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:python:test-suites:direct:ioCrossLanguagePostCommit - arguments: -PuseWheelDistribution + arguments: | + -PuseWheelDistribution \ + -Pjava17Home=$JAVA_HOME_17_X64 - name: Archive Python Test Results uses: actions/upload-artifact@v7 if: failure() diff --git a/.github/workflows/beam_PostCommit_TransformService_Direct.yml b/.github/workflows/beam_PostCommit_TransformService_Direct.yml index a127cb5b6046..2111afa48015 100644 --- a/.github/workflows/beam_PostCommit_TransformService_Direct.yml +++ b/.github/workflows/beam_PostCommit_TransformService_Direct.yml @@ -74,7 +74,9 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: 11 + java-version: | + 17 + 11 python-version: | 3.10 ${{ matrix.python_version }} @@ -83,7 +85,8 @@ jobs: with: gradle-command: :sdks:python:test-suites:direct:xlang:transformServicePythonUsingJava arguments: | - -PtestJavaVersion=11 \ + -PtestJavaVersion=17 \ + -Pjava17Home=$JAVA_HOME_17_X64 \ -Pjava11Home=$JAVA_HOME_11_X64 \ -PpythonVersion=${{ matrix.python_version }} \ - name: Archive Python Test Results diff --git a/.github/workflows/beam_PostCommit_XVR_Direct.yml b/.github/workflows/beam_PostCommit_XVR_Direct.yml index f2174aa3ec98..d95fe3bec3d0 100644 --- a/.github/workflows/beam_PostCommit_XVR_Direct.yml +++ b/.github/workflows/beam_PostCommit_XVR_Direct.yml @@ -74,6 +74,9 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: + java-version: | + 17 + 11 python-version: ${{ matrix.python_version }} - name: run PostCommit XVR Direct script env: @@ -83,6 +86,7 @@ jobs: gradle-command: :sdks:python:test-suites:direct:xlang:validatesCrossLanguageRunner arguments: | -PpythonVersion=${{ matrix.python_version }} \ + -Pjava17Home=$JAVA_HOME_17_X64 \ -PskipNonPythonTask=${{ (matrix.python_version == '3.10' && true) || false }} \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v7 diff --git a/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml b/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml index 9598b19ebbe0..e1887ce7864e 100644 --- a/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml +++ b/.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml @@ -74,13 +74,17 @@ jobs: uses: ./.github/actions/setup-environment-action with: python-version: default - java-version: '11' + java-version: | + 17 + 11 - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db - name: run PostCommit Yaml Xlang Direct script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:python:postCommitYamlIntegrationTests -PyamlTestSet=${{ matrix.test_set }} -PbeamPythonExtra=p310_ml_test,yaml + arguments: | + -Pjava17Home=$JAVA_HOME_17_X64 - name: Archive Python Test Results uses: actions/upload-artifact@v7 if: failure() diff --git a/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml b/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml index 959f36234d70..2f9741c01a01 100644 --- a/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml +++ b/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml @@ -102,7 +102,9 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: default + java-version: | + 17 + 11 python-version: ${{ matrix.python_version }} - name: Set PY_VER_CLEAN id: set_py_ver_clean @@ -114,3 +116,5 @@ jobs: uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:python:test-suites:direct:crossLanguageWrapperValidationPreCommit --info + arguments: | + -Pjava17Home=$JAVA_HOME_17_X64 diff --git a/.github/workflows/beam_PreCommit_Yaml_Xlang_Direct.yml b/.github/workflows/beam_PreCommit_Yaml_Xlang_Direct.yml index 21281b8cd3e5..10693af1bf4e 100644 --- a/.github/workflows/beam_PreCommit_Yaml_Xlang_Direct.yml +++ b/.github/workflows/beam_PreCommit_Yaml_Xlang_Direct.yml @@ -87,11 +87,16 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: + java-version: | + 17 + 11 python-version: default - name: run PreCommit Yaml Xlang Direct script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :sdks:python:yamlIntegrationTests -PbeamPythonExtra=p310_ml_test,yaml + arguments: | + -Pjava17Home=$JAVA_HOME_17_X64 - name: Archive Python Test Results uses: actions/upload-artifact@v7 if: failure() diff --git a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml index 526abfd4e30b..ebf2ec703f61 100644 --- a/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml +++ b/.github/workflows/beam_Publish_Beam_SDK_Snapshots.yml @@ -109,7 +109,9 @@ jobs: if: ${{ startsWith(matrix.container_task, 'java') }} uses: ./.github/actions/setup-environment-action with: - java-version: 11 + java-version: | + 17 + 11 - name: Setup Python environment if: ${{ startsWith(matrix.container_task, 'python') }} uses: ./.github/actions/setup-environment-action @@ -120,6 +122,7 @@ jobs: with: gradle-command: :sdks:${{ matrix.container_task }} arguments: | + -Pjava17Home=$JAVA_HOME_17_X64 \ -Pdocker-repository-root=gcr.io/apache-beam-testing/beam-sdk \ -Pdocker-tag-list=${{ github.sha }},${BEAM_VERSION}${LATEST_TAG} \ -Pcontainer-architecture-list=arm64,amd64 \ diff --git a/.github/workflows/build_release_candidate.yml b/.github/workflows/build_release_candidate.yml index c53c0712a600..9e2a0c820178 100644 --- a/.github/workflows/build_release_candidate.yml +++ b/.github/workflows/build_release_candidate.yml @@ -276,11 +276,13 @@ jobs: repository: apache/beam - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@v1.3.1 - - name: Install Java 11 + - name: Install Java uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: '11' + java-version: | + 17 + 11 - name: Install Python 3.10 uses: actions/setup-python@v5 with: @@ -570,11 +572,11 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.10' - - name: Install Java 11 + - name: Install Java 17 uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: '11' + java-version: '17' - name: Remove default github maven configuration # This step is a workaround to avoid a decryption issue of Beam's # net.linguica.gradle.maven.settings plugin and github's provided maven diff --git a/.github/workflows/typescript_tests.yml b/.github/workflows/typescript_tests.yml index 9bc352379913..17947290d4c3 100644 --- a/.github/workflows/typescript_tests.yml +++ b/.github/workflows/typescript_tests.yml @@ -111,6 +111,11 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.10' + - name: Install Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' - name: Setup Beam Python working-directory: ./sdks/python run: | @@ -174,6 +179,11 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.10' + - name: Install Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' - name: Setup Beam Python working-directory: ./sdks/python run: | diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index abeede24709a..847bfa63382e 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -1645,6 +1645,12 @@ class BeamModulePlugin implements Plugin { useJUnit() executable = "${testJavaHome}/bin/java" } + // redirect java exec tasks (expansion service, run, shadowJar execs) to specified JDK + if (testJavaHome) { + project.tasks.withType(JavaExec).configureEach { + executable = "${testJavaHome}/bin/java" + } + } } if (configuration.shadowClosure) { diff --git a/sdks/java/expansion-service/container/Dockerfile b/sdks/java/expansion-service/container/Dockerfile index 968f5cd2ac25..513dd6b75b88 100644 --- a/sdks/java/expansion-service/container/Dockerfile +++ b/sdks/java/expansion-service/container/Dockerfile @@ -16,7 +16,7 @@ # limitations under the License. ############################################################################### -FROM eclipse-temurin:11 +FROM eclipse-temurin:17 LABEL Author "Apache Beam " ARG TARGETOS ARG TARGETARCH diff --git a/sdks/java/io/expansion-service/build.gradle b/sdks/java/io/expansion-service/build.gradle index 70a3fce538b6..f7b241a75944 100644 --- a/sdks/java/io/expansion-service/build.gradle +++ b/sdks/java/io/expansion-service/build.gradle @@ -25,8 +25,8 @@ applyJavaNature( exportJavadoc: false, validateShadowJar: false, shadowClosure: {}, - // iceberg requires Java11+ - requireJavaVersion: JavaVersion.VERSION_11 + // iceberg requires Java11+ and delta lake requires Java17+ + requireJavaVersion: JavaVersion.VERSION_17 ) // We don't want to use the latest version for the entire beam sdk since beam Java users can override it themselves.