From 5e21615a3c16ec386ca7534b81179f7a6d6fe084 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Thu, 14 May 2026 10:22:06 -0400 Subject: [PATCH 1/2] Fix iceberg gcs dependency after gcsio 3.0 upgrade (#38488) --- .github/trigger_files/beam_PostCommit_SQL.json | 2 +- sdks/java/extensions/sql/iceberg/build.gradle | 1 + sdks/java/io/expansion-service/build.gradle | 10 ++++------ 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/trigger_files/beam_PostCommit_SQL.json b/.github/trigger_files/beam_PostCommit_SQL.json index 833fd9b0d174..5df3841d2363 100644 --- a/.github/trigger_files/beam_PostCommit_SQL.json +++ b/.github/trigger_files/beam_PostCommit_SQL.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run ", - "modification": 2 + "modification": 3 } diff --git a/sdks/java/extensions/sql/iceberg/build.gradle b/sdks/java/extensions/sql/iceberg/build.gradle index 1e319c97a8e2..893a485e7d86 100644 --- a/sdks/java/extensions/sql/iceberg/build.gradle +++ b/sdks/java/extensions/sql/iceberg/build.gradle @@ -48,6 +48,7 @@ dependencies { testImplementation library.java.google_api_services_bigquery testImplementation "org.apache.iceberg:iceberg-api:1.9.2" testImplementation "org.apache.iceberg:iceberg-core:1.9.2" + testRuntimeOnly library.java.bigdataoss_util_hadoop testImplementation project(":sdks:java:io:google-cloud-platform") testImplementation project(":sdks:java:extensions:google-cloud-platform-core") } diff --git a/sdks/java/io/expansion-service/build.gradle b/sdks/java/io/expansion-service/build.gradle index 4e4c15f367cb..77a062c5b9af 100644 --- a/sdks/java/io/expansion-service/build.gradle +++ b/sdks/java/io/expansion-service/build.gradle @@ -83,12 +83,10 @@ dependencies { permitUnusedDeclared project(":sdks:java:extensions:kafka-factories") runtimeOnly project(":sdks:java:io:amazon-web-services2") // FileSystem may be used by Iceberg AddFiles - if (JavaVersion.current().compareTo(JavaVersion.VERSION_11) >= 0 && project.findProperty('testJavaVersion') != '8') { - // iceberg ended support for Java 8 in 1.7.0 - runtimeOnly project(":sdks:java:io:iceberg") - runtimeOnly project(":sdks:java:io:iceberg:hive") - runtimeOnly project(path: ":sdks:java:io:iceberg:bqms", configuration: "shadow") - } + runtimeOnly project(":sdks:java:io:iceberg") + runtimeOnly project(":sdks:java:io:iceberg:hive") + runtimeOnly project(path: ":sdks:java:io:iceberg:bqms", configuration: "shadow") + runtimeOnly library.java.bigdataoss_util_hadoop runtimeOnly library.java.kafka_clients runtimeOnly library.java.slf4j_jdk14 From 6ae3da8260dde72ae435fad07c408bee6183bd27 Mon Sep 17 00:00:00 2001 From: Ahmed Abualsaud <65791736+ahmedabu98@users.noreply.github.com> Date: Thu, 14 May 2026 15:13:28 -0400 Subject: [PATCH 2/2] [Iceberg] pin hadoop to 3.3.6 (#38500) * pin hadoop to 3.3.6 * pin in expansion service too --- .../trigger_files/IO_Iceberg_Integration_Tests.json | 2 +- .../IO_Iceberg_Integration_Tests_Dataflow.json | 2 +- sdks/java/io/expansion-service/build.gradle | 10 ++++++++++ sdks/java/io/iceberg/build.gradle | 11 ++++++++++- .../apache/beam/sdk/io/iceberg/IcebergScanConfig.java | 2 +- 5 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.github/trigger_files/IO_Iceberg_Integration_Tests.json b/.github/trigger_files/IO_Iceberg_Integration_Tests.json index b73af5e61a43..7ab7bcd9a9c6 100644 --- a/.github/trigger_files/IO_Iceberg_Integration_Tests.json +++ b/.github/trigger_files/IO_Iceberg_Integration_Tests.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run.", - "modification": 1 + "modification": 2 } diff --git a/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json b/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json index 5abe02fc09c7..3a009261f4f9 100644 --- a/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json +++ b/.github/trigger_files/IO_Iceberg_Integration_Tests_Dataflow.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run.", - "modification": 1 + "modification": 2 } diff --git a/sdks/java/io/expansion-service/build.gradle b/sdks/java/io/expansion-service/build.gradle index 77a062c5b9af..4683b4fd6f50 100644 --- a/sdks/java/io/expansion-service/build.gradle +++ b/sdks/java/io/expansion-service/build.gradle @@ -57,6 +57,16 @@ configurations.runtimeClasspath { // Pin nimbus-jose-jwt to 9.37.4 to fix CVE-2025-53864 (transitive via hadoop-auth) resolutionStrategy.force 'com.nimbusds:nimbus-jose-jwt:9.37.4' + + // [iceberg] + // bigdataoss:gcs-connector and parquet:parquet-hadoop have conflicts with global hadoop-common:3.4.2 + // upgrading gcs-connector to 4.0.0 would be fine, because it uses hadoop-common 3.4.2 + // but parquet-hadoop is still at 3.3.0 + // so for now we need to pin hadoop to 3.3.6 until parquet-hadoop releases a version that uses hadoop 3.4.2+ + resolutionStrategy.force 'org.apache.hadoop:hadoop-common:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-client:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs-client:3.3.6' } shadowJar { diff --git a/sdks/java/io/iceberg/build.gradle b/sdks/java/io/iceberg/build.gradle index bbd55fee2fc8..2dae0335f9c9 100644 --- a/sdks/java/io/iceberg/build.gradle +++ b/sdks/java/io/iceberg/build.gradle @@ -59,7 +59,7 @@ dependencies { implementation "org.apache.iceberg:iceberg-parquet:$iceberg_version" implementation "org.apache.iceberg:iceberg-orc:$iceberg_version" implementation "org.apache.iceberg:iceberg-data:$iceberg_version" - implementation library.java.hadoop_common + implementation "org.apache.hadoop:hadoop-common:3.3.6" // TODO(https://github.com/apache/beam/issues/21156): Determine how to build without this dependency provided "org.immutables:value:2.8.8" permitUnusedDeclared "org.immutables:value:2.8.8" @@ -70,6 +70,7 @@ dependencies { runtimeOnly "org.apache.iceberg:iceberg-azure:$iceberg_version" runtimeOnly "org.apache.iceberg:iceberg-azure-bundle:$iceberg_version" runtimeOnly library.java.bigdataoss_gcs_connector + runtimeOnly library.java.bigdataoss_util_hadoop runtimeOnly library.java.hadoop_client testImplementation project(":sdks:java:managed") @@ -117,6 +118,14 @@ dependencies { configurations.all { // iceberg-core needs avro:1.12.0 resolutionStrategy.force 'org.apache.avro:avro:1.12.0' + // bigdataoss:gcs-connector and parquet:parquet-hadoop have conflicts with global hadoop-common:3.4.2 + // upgrading gcs-connector to 4.0.0 would be fine, because it uses hadoop-common 3.4.2 + // but parquet-hadoop is still at 3.3.0 + // so for now we need to pin hadoop to 3.3.6 until parquet-hadoop releases a version that uses hadoop 3.4.2+ + resolutionStrategy.force 'org.apache.hadoop:hadoop-common:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-client:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs:3.3.6' + resolutionStrategy.force 'org.apache.hadoop:hadoop-hdfs-client:3.3.6' } hadoopVersions.each {kv -> diff --git a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java index 56f21d2bc8b5..45ecc7cf71c3 100644 --- a/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java +++ b/sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java @@ -19,7 +19,7 @@ import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkArgument; import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull; -import static org.apache.hadoop.util.Sets.newHashSet; +import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.Sets.newHashSet; import com.google.auto.value.AutoValue; import java.io.Serializable;