From 10475c958edc95f089545fb6b465e4d4d68df32f Mon Sep 17 00:00:00 2001 From: CI Builder Date: Thu, 8 Jan 2026 16:12:15 -0800 Subject: [PATCH] Add delete file replication as table and spark-sql property. --- .../main/java/org/apache/iceberg/TableProperties.java | 2 ++ .../org/apache/iceberg/spark/SparkSQLProperties.java | 3 +++ .../java/org/apache/iceberg/spark/SparkWriteConf.java | 11 +++++++++++ .../org/apache/iceberg/spark/SparkWriteOptions.java | 6 ++++++ .../org/apache/iceberg/spark/SparkSQLProperties.java | 3 +++ .../java/org/apache/iceberg/spark/SparkWriteConf.java | 2 ++ .../org/apache/iceberg/spark/SparkWriteOptions.java | 2 ++ .../org/apache/iceberg/spark/SparkSQLProperties.java | 3 +++ .../java/org/apache/iceberg/spark/SparkWriteConf.java | 2 ++ 9 files changed, 34 insertions(+) diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java b/core/src/main/java/org/apache/iceberg/TableProperties.java index 2267ba03fd..1600868e87 100644 --- a/core/src/main/java/org/apache/iceberg/TableProperties.java +++ b/core/src/main/java/org/apache/iceberg/TableProperties.java @@ -371,4 +371,6 @@ private TableProperties() {} public static final int ENCRYPTION_DEK_LENGTH_DEFAULT = 16; public static final int ENCRYPTION_AAD_LENGTH_DEFAULT = 16; + + public static final String DELETE_FILE_REPLICATION = "write.delete-file-replication"; } diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java index b09f668c44..8169bbafe3 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java @@ -69,4 +69,7 @@ private SparkSQLProperties() {} // Controls the spark input split size. public static final String SPLIT_SIZE = "spark.sql.iceberg.split-size"; + + // Controls the replication factor for delete files + public static final String DELETE_FILE_REPLICATION = "spark.sql.iceberg.delete-file-replication"; } diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java index 41777c5155..28b4d7db60 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java @@ -133,6 +133,17 @@ public String wapId() { return sessionConf.get(SparkSQLProperties.WAP_ID, null); } + public short deleteFileReplication() { + return (short) + confParser + .intConf() + .option(SparkWriteOptions.DELETE_FILE_REPLICATION) + .sessionConf(SparkSQLProperties.DELETE_FILE_REPLICATION) + .tableProperty(TableProperties.DELETE_FILE_REPLICATION) + .defaultValue(SparkWriteOptions.DEFAULT_DELETE_FILE_REPLICATION) + .parse(); + } + public boolean mergeSchema() { return confParser .booleanConf() diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java index c4eacb7b98..d1fecf10f4 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java @@ -23,6 +23,12 @@ public class SparkWriteOptions { private SparkWriteOptions() {} + // Configuration to tune the replication factor for delete files. + public static final String DELETE_FILE_REPLICATION = "delete-file-replication"; + + // Configuring the default replication factor. + public static final short DEFAULT_DELETE_FILE_REPLICATION = 3; + // Fileformat for write operations(default: Table write.format.default ) public static final String WRITE_FORMAT = "write-format"; diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java index 908a82c3c4..a856a74da8 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java @@ -100,4 +100,7 @@ private SparkSQLProperties() {} // Controls the spark input split size. public static final String SPLIT_SIZE = "spark.sql.iceberg.split-size"; + + // Controls the replication factor for delete files + public static final String DELETE_FILE_REPLICATION = "spark.sql.iceberg.delete-file-replication"; } diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java index c37c09c95d..1a1dfb4102 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java @@ -137,6 +137,8 @@ public short deleteFileReplication() { confParser .intConf() .option(SparkWriteOptions.DELETE_FILE_REPLICATION) + .sessionConf(SparkSQLProperties.DELETE_FILE_REPLICATION) + .tableProperty(TableProperties.DELETE_FILE_REPLICATION) .defaultValue(SparkWriteOptions.DEFAULT_DELETE_FILE_REPLICATION) .parse(); } diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java index f5f3a4259b..4bd704f62a 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java @@ -23,8 +23,10 @@ public class SparkWriteOptions { private SparkWriteOptions() {} + // Configuration to tune the replication factor for delete files. public static final String DELETE_FILE_REPLICATION = "delete-file-replication"; + // The default replication factor. public static final short DEFAULT_DELETE_FILE_REPLICATION = 3; // Fileformat for write operations(default: Table write.format.default ) diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java index 01648fbd56..9b1155533d 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java @@ -93,4 +93,7 @@ private SparkSQLProperties() {} // Controls the spark input split size. public static final String SPLIT_SIZE = "spark.sql.iceberg.split-size"; + + // Controls the delete file replication + public static final String DELETE_FILE_REPLICATION = "spark.sql.iceberg.delete-file-replication"; } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java index 2bef37a8e2..377bb901e1 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java @@ -136,6 +136,8 @@ public short deleteFileReplication() { return (short) confParser .intConf() + .sessionConf(SparkSQLProperties.DELETE_FILE_REPLICATION) + .tableProperty(TableProperties.DELETE_FILE_REPLICATION) .option(SparkWriteOptions.DELETE_FILE_REPLICATION) .defaultValue(SparkWriteOptions.DEFAULT_DELETE_FILE_REPLICATION) .parse();