diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java b/core/src/main/java/org/apache/iceberg/TableProperties.java index 2267ba03f..1600868e8 100644 --- a/core/src/main/java/org/apache/iceberg/TableProperties.java +++ b/core/src/main/java/org/apache/iceberg/TableProperties.java @@ -371,4 +371,6 @@ private TableProperties() {} public static final int ENCRYPTION_DEK_LENGTH_DEFAULT = 16; public static final int ENCRYPTION_AAD_LENGTH_DEFAULT = 16; + + public static final String DELETE_FILE_REPLICATION = "write.delete-file-replication"; } diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java index b09f668c4..8169bbafe 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java @@ -69,4 +69,7 @@ private SparkSQLProperties() {} // Controls the spark input split size. public static final String SPLIT_SIZE = "spark.sql.iceberg.split-size"; + + // Controls the replication factor for delete files + public static final String DELETE_FILE_REPLICATION = "spark.sql.iceberg.delete-file-replication"; } diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java index 41777c515..28b4d7db6 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java @@ -133,6 +133,17 @@ public String wapId() { return sessionConf.get(SparkSQLProperties.WAP_ID, null); } + public short deleteFileReplication() { + return (short) + confParser + .intConf() + .option(SparkWriteOptions.DELETE_FILE_REPLICATION) + .sessionConf(SparkSQLProperties.DELETE_FILE_REPLICATION) + .tableProperty(TableProperties.DELETE_FILE_REPLICATION) + .defaultValue(SparkWriteOptions.DEFAULT_DELETE_FILE_REPLICATION) + .parse(); + } + public boolean mergeSchema() { return confParser .booleanConf() diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java index c4eacb7b9..d1fecf10f 100644 --- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java +++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java @@ -23,6 +23,12 @@ public class SparkWriteOptions { private SparkWriteOptions() {} + // Configuration to tune the replication factor for delete files. + public static final String DELETE_FILE_REPLICATION = "delete-file-replication"; + + // Configuring the default replication factor. + public static final short DEFAULT_DELETE_FILE_REPLICATION = 3; + // Fileformat for write operations(default: Table write.format.default ) public static final String WRITE_FORMAT = "write-format"; diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java index 908a82c3c..a856a74da 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java @@ -100,4 +100,7 @@ private SparkSQLProperties() {} // Controls the spark input split size. public static final String SPLIT_SIZE = "spark.sql.iceberg.split-size"; + + // Controls the replication factor for delete files + public static final String DELETE_FILE_REPLICATION = "spark.sql.iceberg.delete-file-replication"; } diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java index c37c09c95..1a1dfb410 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java @@ -137,6 +137,8 @@ public short deleteFileReplication() { confParser .intConf() .option(SparkWriteOptions.DELETE_FILE_REPLICATION) + .sessionConf(SparkSQLProperties.DELETE_FILE_REPLICATION) + .tableProperty(TableProperties.DELETE_FILE_REPLICATION) .defaultValue(SparkWriteOptions.DEFAULT_DELETE_FILE_REPLICATION) .parse(); } diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java index f5f3a4259..4bd704f62 100644 --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java @@ -23,8 +23,10 @@ public class SparkWriteOptions { private SparkWriteOptions() {} + // Configuration to tune the replication factor for delete files. public static final String DELETE_FILE_REPLICATION = "delete-file-replication"; + // The default replication factor. public static final short DEFAULT_DELETE_FILE_REPLICATION = 3; // Fileformat for write operations(default: Table write.format.default ) diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java index 01648fbd5..9b1155533 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java @@ -93,4 +93,7 @@ private SparkSQLProperties() {} // Controls the spark input split size. public static final String SPLIT_SIZE = "spark.sql.iceberg.split-size"; + + // Controls the delete file replication + public static final String DELETE_FILE_REPLICATION = "spark.sql.iceberg.delete-file-replication"; } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java index 2bef37a8e..377bb901e 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java @@ -136,6 +136,8 @@ public short deleteFileReplication() { return (short) confParser .intConf() + .sessionConf(SparkSQLProperties.DELETE_FILE_REPLICATION) + .tableProperty(TableProperties.DELETE_FILE_REPLICATION) .option(SparkWriteOptions.DELETE_FILE_REPLICATION) .defaultValue(SparkWriteOptions.DEFAULT_DELETE_FILE_REPLICATION) .parse();