From 88f33aefb9b48f4a3e97545fa26d152b773951a3 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Thu, 21 May 2026 12:46:48 -0400 Subject: [PATCH 1/5] Implement Maven Central mirroring in CI environments --- buildSrc/build.gradle.kts | 19 ++++++++ buildSrc/settings.gradle.kts | 43 +++++++++++++++++++ .../apache/beam/gradle/Repositories.groovy | 22 ++++++++-- gradle.properties | 3 ++ settings.gradle.kts | 14 ++++++ 5 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 buildSrc/settings.gradle.kts diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 9ad1a6a5bf3b..96b1cb12dd4c 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -15,6 +15,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import java.util.Properties + +val parentProperties = Properties().apply { + val file = file("../gradle.properties") + if (file.exists()) { + file.inputStream().use { load(it) } + } +} + +val mavenCentralMirrorUrl = parentProperties.getProperty("mavenCentralMirrorUrl") +val isCi = System.getenv("GITHUB_ACTIONS") != null || System.getenv("JENKINS_HOME") != null +val useMirror = isCi && !mavenCentralMirrorUrl.isNullOrBlank() // Plugins for configuring _this build_ of the module plugins { @@ -25,6 +37,13 @@ plugins { // Define the set of repositories required to fetch and enable plugins. repositories { + if (useMirror) { + logger.lifecycle("Running in CI. Mirroring Maven Central repositories via Google Maven Mirror for buildSrc.") + } + + if (useMirror) { + maven { url = uri(mavenCentralMirrorUrl!!) } + } maven { url = uri("https://plugins.gradle.org/m2/") } maven { url = uri("https://repo.spring.io/plugins-release/") diff --git a/buildSrc/settings.gradle.kts b/buildSrc/settings.gradle.kts new file mode 100644 index 000000000000..1fae63f5cc1e --- /dev/null +++ b/buildSrc/settings.gradle.kts @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Properties + +pluginManagement { + val parentProperties = java.util.Properties().apply { + val file = file("../gradle.properties") + if (file.exists()) { + file.inputStream().use { load(it) } + } + } + + val mavenCentralMirrorUrl = parentProperties.getProperty("mavenCentralMirrorUrl") + val isCi = System.getenv("GITHUB_ACTIONS") != null || System.getenv("JENKINS_HOME") != null + val useMirror = isCi && !mavenCentralMirrorUrl.isNullOrBlank() + + if (useMirror) { + logger.lifecycle("Running in CI. Mirroring Maven Central repositories via Google Maven Mirror for buildSrc plugins.") + } + + repositories { + if (useMirror) { + maven { url = uri(mavenCentralMirrorUrl!!) } + } + gradlePluginPortal() + } +} diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/Repositories.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/Repositories.groovy index 12e48356898a..20c878c06a94 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/Repositories.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/Repositories.groovy @@ -19,10 +19,16 @@ package org.apache.beam.gradle import org.gradle.api.Project - class Repositories { static void register(Project project) { + def mirrorUrl = project.findProperty("mavenCentralMirrorUrl") + boolean isCi = System.getenv("GITHUB_ACTIONS") != null || System.getenv("JENKINS_HOME") != null + def useMirror = isCi && mirrorUrl + + if (useMirror) { + project.logger.lifecycle("Running in CI. Mirroring Maven Central repositories via Google Maven Mirror.") + } project.repositories { maven { url project.offlineRepositoryRoot } @@ -36,7 +42,11 @@ class Repositories { return } - mavenCentral() + if (!useMirror) { + mavenCentral() + } else { + maven { url mirrorUrl } + } mavenLocal() // For Confluent Kafka dependencies @@ -60,7 +70,7 @@ class Repositories { // Apply a plugin which provides the 'updateOfflineRepository' task that creates an offline // repository. This offline repository satisfies all Gradle build dependencies and Java // project dependencies. The offline repository is placed within $rootDir/offline-repo - // but can be overridden by specifying '-PofflineRepositoryRoot=/path/to/repo'. + // but can be overridden by specifying '-Pareas/offline-repo'. // Note that parallel build must be disabled when executing 'updateOfflineRepository' // by specifying '--no-parallel', see // https://github.com/mdietrichstein/gradle-offline-dependencies-plugin/issues/3 @@ -68,7 +78,11 @@ class Repositories { project.offlineDependencies { repositories { mavenLocal() - mavenCentral() + if (!useMirror) { + mavenCentral() + } else { + maven { url mirrorUrl } + } maven { url "https://plugins.gradle.org/m2/" } maven { url "https://repo.spring.io/plugins-release" } maven { url "https://packages.confluent.io/maven/" } diff --git a/gradle.properties b/gradle.properties index 1d40c7ed4669..95e50105a494 100644 --- a/gradle.properties +++ b/gradle.properties @@ -44,3 +44,6 @@ flink_versions=1.17,1.18,1.19,1.20,2.0 spark_versions=3,4 # supported python versions python_versions=3.10,3.11,3.12,3.13,3.14 + +# Maven Central fallback mirror URL +mavenCentralMirrorUrl=https://maven-central.storage-download.googleapis.com/maven2/ diff --git a/settings.gradle.kts b/settings.gradle.kts index 9c6bd2b30594..9e83157686a5 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -18,6 +18,20 @@ import com.gradle.enterprise.gradleplugin.internal.extension.BuildScanExtensionWithHiddenFeatures pluginManagement { + val mavenCentralMirrorUrl = settings.providers.gradleProperty("mavenCentralMirrorUrl").orNull + val isCi = System.getenv("GITHUB_ACTIONS") != null || System.getenv("JENKINS_HOME") != null + val useMirror = isCi && !mavenCentralMirrorUrl.isNullOrBlank() + + if (useMirror) { + logger.lifecycle("Running in CI. Mirroring Maven Central repositories via Google Maven Mirror.") + } + + repositories { + if (useMirror) { + maven { url = uri(mavenCentralMirrorUrl!!) } + } + gradlePluginPortal() + } plugins { id("org.javacc.javacc") version "4.0.3" // enable the JavaCC parser generator } From 197963a0f79568b8a899eacda0950cd327aee430 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Thu, 21 May 2026 12:52:05 -0400 Subject: [PATCH 2/5] Fix subproject buildscript classpath resolution for beam-test-gha using CI mirror fallback --- .github/build.gradle | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/build.gradle b/.github/build.gradle index 6effa8742f60..d5a9a6c9dc75 100644 --- a/.github/build.gradle +++ b/.github/build.gradle @@ -18,7 +18,15 @@ buildscript { repositories { - mavenCentral() + def mirrorUrl = project.findProperty("mavenCentralMirrorUrl") + boolean isCi = System.getenv("GITHUB_ACTIONS") != null || System.getenv("JENKINS_HOME") != null + def useMirror = isCi && mirrorUrl + + if (!useMirror) { + mavenCentral() + } else { + maven { url mirrorUrl } + } } dependencies { classpath group: 'org.yaml', name: 'snakeyaml', version: '2.2' From 1bf83f08bb09c9100285ef27569dfaf9faffc51b Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Thu, 21 May 2026 13:18:31 -0400 Subject: [PATCH 3/5] Remove duplicated definition. --- settings.gradle.kts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index 9e83157686a5..14f990df09d1 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -17,9 +17,14 @@ */ import com.gradle.enterprise.gradleplugin.internal.extension.BuildScanExtensionWithHiddenFeatures +// JENKINS_HOME and BUILD_ID set automatically during Jenkins execution +val isJenkinsBuild = arrayOf("JENKINS_HOME", "BUILD_ID").all { System.getenv(it) != null } +// GITHUB_REPOSITORY and GITHUB_RUN_ID set automatically during Github Actions run +val isGithubActionsBuild = arrayOf("GITHUB_REPOSITORY", "GITHUB_RUN_ID").all { System.getenv(it) != null } +val isCi = isJenkinsBuild || isGithubActionsBuild + pluginManagement { val mavenCentralMirrorUrl = settings.providers.gradleProperty("mavenCentralMirrorUrl").orNull - val isCi = System.getenv("GITHUB_ACTIONS") != null || System.getenv("JENKINS_HOME") != null val useMirror = isCi && !mavenCentralMirrorUrl.isNullOrBlank() if (useMirror) { @@ -42,13 +47,6 @@ plugins { id("com.gradle.common-custom-user-data-gradle-plugin") version "2.6.0" } - -// JENKINS_HOME and BUILD_ID set automatically during Jenkins execution -val isJenkinsBuild = arrayOf("JENKINS_HOME", "BUILD_ID").all { System.getenv(it) != null } -// GITHUB_REPOSITORY and GITHUB_RUN_ID set automatically during Github Actions run -val isGithubActionsBuild = arrayOf("GITHUB_REPOSITORY", "GITHUB_RUN_ID").all { System.getenv(it) != null } -val isCi = isJenkinsBuild || isGithubActionsBuild - develocity { server = "https://develocity.apache.org" projectId = "beam" From 866316146eac35f66b579b563c72619028cdab5e Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Thu, 21 May 2026 13:20:18 -0400 Subject: [PATCH 4/5] Discard the new settings file in buildSrc first. --- buildSrc/settings.gradle.kts | 43 ------------------------------------ 1 file changed, 43 deletions(-) delete mode 100644 buildSrc/settings.gradle.kts diff --git a/buildSrc/settings.gradle.kts b/buildSrc/settings.gradle.kts deleted file mode 100644 index 1fae63f5cc1e..000000000000 --- a/buildSrc/settings.gradle.kts +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.Properties - -pluginManagement { - val parentProperties = java.util.Properties().apply { - val file = file("../gradle.properties") - if (file.exists()) { - file.inputStream().use { load(it) } - } - } - - val mavenCentralMirrorUrl = parentProperties.getProperty("mavenCentralMirrorUrl") - val isCi = System.getenv("GITHUB_ACTIONS") != null || System.getenv("JENKINS_HOME") != null - val useMirror = isCi && !mavenCentralMirrorUrl.isNullOrBlank() - - if (useMirror) { - logger.lifecycle("Running in CI. Mirroring Maven Central repositories via Google Maven Mirror for buildSrc plugins.") - } - - repositories { - if (useMirror) { - maven { url = uri(mavenCentralMirrorUrl!!) } - } - gradlePluginPortal() - } -} From 99d50217a4118c3e171b573049318ab552980762 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Thu, 21 May 2026 13:25:43 -0400 Subject: [PATCH 5/5] Fix compilation error. --- settings.gradle.kts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/settings.gradle.kts b/settings.gradle.kts index 14f990df09d1..5826665f0a06 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -17,14 +17,9 @@ */ import com.gradle.enterprise.gradleplugin.internal.extension.BuildScanExtensionWithHiddenFeatures -// JENKINS_HOME and BUILD_ID set automatically during Jenkins execution -val isJenkinsBuild = arrayOf("JENKINS_HOME", "BUILD_ID").all { System.getenv(it) != null } -// GITHUB_REPOSITORY and GITHUB_RUN_ID set automatically during Github Actions run -val isGithubActionsBuild = arrayOf("GITHUB_REPOSITORY", "GITHUB_RUN_ID").all { System.getenv(it) != null } -val isCi = isJenkinsBuild || isGithubActionsBuild - pluginManagement { val mavenCentralMirrorUrl = settings.providers.gradleProperty("mavenCentralMirrorUrl").orNull + val isCi = System.getenv("GITHUB_ACTIONS") != null || System.getenv("JENKINS_HOME") != null val useMirror = isCi && !mavenCentralMirrorUrl.isNullOrBlank() if (useMirror) { @@ -47,6 +42,12 @@ plugins { id("com.gradle.common-custom-user-data-gradle-plugin") version "2.6.0" } +// JENKINS_HOME and BUILD_ID set automatically during Jenkins execution +val isJenkinsBuild = arrayOf("JENKINS_HOME", "BUILD_ID").all { System.getenv(it) != null } +// GITHUB_REPOSITORY and GITHUB_RUN_ID set automatically during Github Actions run +val isGithubActionsBuild = arrayOf("GITHUB_REPOSITORY", "GITHUB_RUN_ID").all { System.getenv(it) != null } +val isCi = isJenkinsBuild || isGithubActionsBuild + develocity { server = "https://develocity.apache.org" projectId = "beam"