diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index cd4afe6fdc8d..be0e6fd72bdc 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -404,6 +404,365 @@ acceptedBreaks: old: "method org.apache.iceberg.orc.ORC.WriteBuilder org.apache.iceberg.orc.ORC.WriteBuilder::config(java.lang.String,\ \ java.lang.String)" justification: "Removing deprecations for 1.2.0" + "1.10.0": + org.apache.iceberg:iceberg-api: + - code: "java.class.defaultSerializationChanged" + old: "class org.apache.iceberg.encryption.EncryptingFileIO" + new: "class org.apache.iceberg.encryption.EncryptingFileIO" + justification: "New method for Manifest List reading" + org.apache.iceberg:iceberg-core: + - code: "java.class.defaultSerializationChanged" + old: "class org.apache.iceberg.avro.SupportsIndexProjection" + new: "class org.apache.iceberg.avro.SupportsIndexProjection" + justification: "Serialization across versions is not guaranteed" + - code: "java.class.defaultSerializationChanged" + old: "class org.apache.iceberg.hadoop.SerializableConfiguration" + new: "class org.apache.iceberg.hadoop.SerializableConfiguration" + justification: "Serialization across versions is not guaranteed" + - code: "java.class.noLongerInheritsFromClass" + old: "class org.apache.iceberg.rest.auth.OAuth2Manager" + new: "class org.apache.iceberg.rest.auth.OAuth2Manager" + justification: "Removing deprecations for 1.11.0" + - code: "java.class.nowImplementsInterface" + old: "class org.apache.iceberg.rest.auth.OAuth2Manager" + new: "class org.apache.iceberg.rest.auth.OAuth2Manager" + justification: "Removing deprecations for 1.11.0" + - code: "java.class.removed" + old: "class org.apache.iceberg.PartitionStatsUtil" + justification: "Removing deprecated code for 1.11.0" + - code: "java.class.removed" + old: "class org.apache.iceberg.rest.auth.RefreshingAuthManager" + justification: "Removing deprecations for 1.11.0" + - code: "java.field.constantValueChanged" + old: "field org.apache.iceberg.rest.ResourcePaths.V1_TABLE_SCAN_PLAN" + new: "field org.apache.iceberg.rest.ResourcePaths.V1_TABLE_SCAN_PLAN" + justification: "Plan API is table scoped and path constant value should include\ + \ namespace. No actual breakage because it never worked before with incorrect\ + \ value." + - code: "java.field.constantValueChanged" + old: "field org.apache.iceberg.rest.ResourcePaths.V1_TABLE_SCAN_PLAN_SUBMIT" + new: "field org.apache.iceberg.rest.ResourcePaths.V1_TABLE_SCAN_PLAN_SUBMIT" + justification: "Plan API is table scoped and path constant value should include\ + \ namespace. No actual breakage because it never worked before with incorrect\ + \ value." + - code: "java.field.constantValueChanged" + old: "field org.apache.iceberg.rest.ResourcePaths.V1_TABLE_SCAN_PLAN_TASKS" + new: "field org.apache.iceberg.rest.ResourcePaths.V1_TABLE_SCAN_PLAN_TASKS" + justification: "Plan API is table scoped and path constant value should include\ + \ namespace. No actual breakage because it never worked before with incorrect\ + \ value." + - code: "java.method.removed" + old: "method java.lang.String org.apache.iceberg.RewriteTablePathUtil::stagingPath(java.lang.String,\ + \ java.lang.String)" + justification: "Removing deprecated code for 1.11.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.RewriteTablePathUtil.RewriteResult\ + \ org.apache.iceberg.RewriteTablePathUtil::rewriteDataManifest(org.apache.iceberg.ManifestFile,\ + \ org.apache.iceberg.io.OutputFile, org.apache.iceberg.io.FileIO, int, java.util.Map, java.lang.String, java.lang.String) throws\ + \ java.io.IOException" + justification: "Removing deprecated code for 1.11.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.RewriteTablePathUtil.RewriteResult\ + \ org.apache.iceberg.RewriteTablePathUtil::rewriteDeleteManifest(org.apache.iceberg.ManifestFile,\ + \ org.apache.iceberg.io.OutputFile, org.apache.iceberg.io.FileIO, int, java.util.Map, java.lang.String, java.lang.String, java.lang.String)\ + \ throws java.io.IOException" + justification: "Removing deprecated code for 1.11.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.Schema org.apache.iceberg.PartitionStatsHandler::schema(org.apache.iceberg.types.Types.StructType)" + justification: "Removing deprecated code for 1.11.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.TableMetadata org.apache.iceberg.TableMetadataParser::read(org.apache.iceberg.io.FileIO,\ + \ org.apache.iceberg.io.InputFile)" + justification: "Removing deprecated code for 1.11.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.encryption.EncryptionManager org.apache.iceberg.encryption.EncryptionUtil::createEncryptionManager(java.util.Map, org.apache.iceberg.encryption.KeyManagementClient)" + justification: "Removing deprecated code for 1.11.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.rest.responses.OAuthTokenResponse org.apache.iceberg.rest.auth.OAuth2Util::exchangeToken(org.apache.iceberg.rest.RESTClient,\ + \ java.util.Map, java.lang.String, java.lang.String,\ + \ java.lang.String, java.lang.String, java.lang.String)" + justification: "Removing deprecated code for 1.11.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.rest.responses.OAuthTokenResponse org.apache.iceberg.rest.auth.OAuth2Util::exchangeToken(org.apache.iceberg.rest.RESTClient,\ + \ java.util.Map, java.lang.String, java.lang.String,\ + \ java.lang.String, java.lang.String, java.lang.String, java.lang.String)" + justification: "Removing deprecated code for 1.11.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.rest.responses.OAuthTokenResponse org.apache.iceberg.rest.auth.OAuth2Util::fetchToken(org.apache.iceberg.rest.RESTClient,\ + \ java.util.Map, java.lang.String, java.lang.String)" + justification: "Removing deprecated code for 1.11.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.rest.responses.OAuthTokenResponse org.apache.iceberg.rest.auth.OAuth2Util::fetchToken(org.apache.iceberg.rest.RESTClient,\ + \ java.util.Map, java.lang.String, java.lang.String,\ + \ java.lang.String)" + justification: "Removing deprecated code for 1.11.0" + - code: "java.method.visibilityReduced" + old: "method void org.apache.iceberg.PartitionStats::appendStats(org.apache.iceberg.PartitionStats)" + new: "method void org.apache.iceberg.PartitionStats::appendStats(org.apache.iceberg.PartitionStats)" + justification: "Changing deprecated code" + - code: "java.method.visibilityReduced" + old: "method void org.apache.iceberg.PartitionStats::deletedEntry(org.apache.iceberg.Snapshot)" + new: "method void org.apache.iceberg.PartitionStats::deletedEntry(org.apache.iceberg.Snapshot)" + justification: "Changing deprecated code" + - code: "java.method.visibilityReduced" + old: "method void org.apache.iceberg.PartitionStats::liveEntry(org.apache.iceberg.ContentFile,\ + \ org.apache.iceberg.Snapshot)" + new: "method void org.apache.iceberg.PartitionStats::liveEntry(org.apache.iceberg.ContentFile,\ + \ org.apache.iceberg.Snapshot)" + justification: "Changing deprecated code" + org.apache.iceberg:iceberg-data: + - code: "java.class.removed" + old: "class org.apache.iceberg.data.PartitionStatsHandler" + justification: "Removing deprecated code for 1.11.0" + "1.11.0": + org.apache.iceberg:iceberg-core: + - code: "java.class.removed" + old: "class org.apache.iceberg.PartitionStats" + justification: "Removing deprecated code for 1.12.0" + - code: "java.class.removed" + old: "class org.apache.iceberg.SystemProperties" + justification: "Removing deprecated code for 1.12.0" + - code: "java.class.removed" + old: "class org.apache.iceberg.data.avro.DataReader" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.PartitionStatsHandler.DATA_FILE_COUNT" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.PartitionStatsHandler.DATA_RECORD_COUNT" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.PartitionStatsHandler.DV_COUNT" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.PartitionStatsHandler.EQUALITY_DELETE_FILE_COUNT" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.PartitionStatsHandler.EQUALITY_DELETE_RECORD_COUNT" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.PartitionStatsHandler.LAST_UPDATED_AT" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.PartitionStatsHandler.LAST_UPDATED_SNAPSHOT_ID" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.PartitionStatsHandler.POSITION_DELETE_FILE_COUNT" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.PartitionStatsHandler.POSITION_DELETE_RECORD_COUNT" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.PartitionStatsHandler.SPEC_ID" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.PartitionStatsHandler.TOTAL_DATA_FILE_SIZE_IN_BYTES" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.PartitionStatsHandler.TOTAL_RECORD_COUNT" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.SystemConfigs.NETFLIX_UNSAFE_PARQUET_ID_FALLBACK_ENABLED" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.rest.RESTUtil.NAMESPACE_JOINER" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.rest.RESTUtil.NAMESPACE_SPLITTER" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.util.ThreadPools.WORKER_THREAD_POOL_SIZE_PROP" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removedWithConstant" + old: "field org.apache.iceberg.PartitionStatsHandler.PARTITION_FIELD_ID" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removedWithConstant" + old: "field org.apache.iceberg.PartitionStatsHandler.PARTITION_FIELD_NAME" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removedWithConstant" + old: "field org.apache.iceberg.TableProperties.MANIFEST_LISTS_ENABLED" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removedWithConstant" + old: "field org.apache.iceberg.TableProperties.MANIFEST_LISTS_ENABLED_DEFAULT" + justification: "Removing deprecated code for 1.12.0" + - code: "java.field.removedWithConstant" + old: "field org.apache.iceberg.rest.RESTSessionCatalog.REST_PAGE_SIZE" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.noLongerDefault" + old: "method org.apache.iceberg.deletes.PositionDeleteWriter\ + \ org.apache.iceberg.RewriteTablePathUtil.PositionDeleteReaderWriter::writer(org.apache.iceberg.io.OutputFile,\ + \ org.apache.iceberg.FileFormat, org.apache.iceberg.PartitionSpec, org.apache.iceberg.StructLike)\ + \ throws java.io.IOException" + new: "method org.apache.iceberg.deletes.PositionDeleteWriter\ + \ org.apache.iceberg.RewriteTablePathUtil.PositionDeleteReaderWriter::writer(org.apache.iceberg.io.OutputFile,\ + \ org.apache.iceberg.FileFormat, org.apache.iceberg.PartitionSpec, org.apache.iceberg.StructLike)\ + \ throws java.io.IOException" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.nowAbstract" + old: "method org.apache.iceberg.deletes.PositionDeleteWriter\ + \ org.apache.iceberg.RewriteTablePathUtil.PositionDeleteReaderWriter::writer(org.apache.iceberg.io.OutputFile,\ + \ org.apache.iceberg.FileFormat, org.apache.iceberg.PartitionSpec, org.apache.iceberg.StructLike)\ + \ throws java.io.IOException" + new: "method org.apache.iceberg.deletes.PositionDeleteWriter\ + \ org.apache.iceberg.RewriteTablePathUtil.PositionDeleteReaderWriter::writer(org.apache.iceberg.io.OutputFile,\ + \ org.apache.iceberg.FileFormat, org.apache.iceberg.PartitionSpec, org.apache.iceberg.StructLike)\ + \ throws java.io.IOException" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method B org.apache.iceberg.rest.responses.BaseScanTaskResponse.Builder, R>, R\ + \ extends org.apache.iceberg.rest.responses.BaseScanTaskResponse>::withDeleteFiles(java.util.List)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method R org.apache.iceberg.deletes.PositionDelete::row()" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method boolean org.apache.iceberg.util.TableScanUtil::hasDeletes(org.apache.iceberg.CombinedScanTask)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method boolean org.apache.iceberg.util.TableScanUtil::hasEqDeletes(org.apache.iceberg.CombinedScanTask)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method java.lang.String org.apache.iceberg.rest.RESTUtil::encodeNamespace(org.apache.iceberg.catalog.Namespace)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method java.nio.ByteBuffer org.apache.iceberg.encryption.StandardEncryptionManager::unwrapKey(java.nio.ByteBuffer)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method java.nio.ByteBuffer org.apache.iceberg.encryption.StandardEncryptionManager::wrapKey(java.nio.ByteBuffer)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method java.util.List org.apache.iceberg.util.SnapshotUtil::newFiles(java.lang.Long,\ + \ long, java.util.function.Function,\ + \ org.apache.iceberg.io.FileIO)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method java.util.concurrent.ExecutorService org.apache.iceberg.util.ThreadPools::newWorkerPool(java.lang.String)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method java.util.concurrent.ExecutorService org.apache.iceberg.util.ThreadPools::newWorkerPool(java.lang.String,\ + \ int)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.avro.Schema org.apache.iceberg.avro.AvroSchemaUtil::pruneColumns(org.apache.avro.Schema,\ + \ java.util.Set, org.apache.iceberg.mapping.NameMapping)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.ManifestReader\ + \ org.apache.iceberg.ManifestFiles::read(org.apache.iceberg.ManifestFile,\ + \ org.apache.iceberg.io.FileIO)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.MetricsConfig org.apache.iceberg.MetricsConfig::forPositionDelete(org.apache.iceberg.Table)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.Schema org.apache.iceberg.PartitionStatsHandler::schema(org.apache.iceberg.types.Types.StructType,\ + \ int)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.catalog.Namespace org.apache.iceberg.rest.RESTUtil::decodeNamespace(java.lang.String)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.deletes.PositionDelete org.apache.iceberg.deletes.PositionDelete::set(java.lang.CharSequence,\ + \ long, R)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.deletes.PositionDeleteWriter\ + \ org.apache.iceberg.RewriteTablePathUtil.PositionDeleteReaderWriter::writer(org.apache.iceberg.io.OutputFile,\ + \ org.apache.iceberg.FileFormat, org.apache.iceberg.PartitionSpec, org.apache.iceberg.StructLike,\ + \ org.apache.iceberg.Schema) throws java.io.IOException" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.CloseableIterable org.apache.iceberg.ManifestFiles::readPaths(org.apache.iceberg.ManifestFile,\ + \ org.apache.iceberg.io.FileIO)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.CloseableIterable\ + \ org.apache.iceberg.util.SnapshotUtil::newFilesBetween(java.lang.Long, long,\ + \ java.util.function.Function,\ + \ org.apache.iceberg.io.FileIO)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.CloseableIterable\ + \ org.apache.iceberg.PartitionStatsHandler::readPartitionStatsFile(org.apache.iceberg.Schema,\ + \ org.apache.iceberg.io.InputFile)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.AllDataFilesTable.AllDataFilesTableScan" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.AllDeleteFilesTable.AllDeleteFilesTableScan" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.AllFilesTable.AllFilesTableScan" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.AllManifestsTable.AllManifestsTableScan" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.DataFilesTable.DataFilesTableScan" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.DataTableScan" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.DeleteFilesTable.DeleteFilesTableScan" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.FilesTable.FilesTableScan" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.PositionDeletesTable.PositionDeletesBatchScan" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.SnapshotScan>" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.rest.responses.LoadTableResponse org.apache.iceberg.rest.CatalogHandlers::loadTable(org.apache.iceberg.catalog.Catalog,\ + \ org.apache.iceberg.catalog.TableIdentifier)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method void org.apache.iceberg.data.avro.RawDecoder::(org.apache.iceberg.Schema,\ + \ java.util.function.Function>,\ + \ org.apache.avro.Schema)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method void org.apache.iceberg.encryption.StandardEncryptionManager::(java.lang.String,\ + \ int, org.apache.iceberg.encryption.KeyManagementClient)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method void org.apache.iceberg.hadoop.HadoopFileIO::(org.apache.iceberg.util.SerializableSupplier)" + justification: "Removing deprecated code for 1.12.0" + - code: "java.method.removed" + old: "method void org.apache.iceberg.io.ContentCache::invalidateAll()" + justification: "Removing deprecated code for 1.12.0" "1.2.0": org.apache.iceberg:iceberg-api: - code: "java.field.constantValueChanged" @@ -1363,111 +1722,6 @@ acceptedBreaks: old: "method org.apache.iceberg.parquet.ParquetValueWriters.StructWriter\ \ org.apache.iceberg.data.parquet.GenericParquetWriter::createStructWriter(java.util.List>)" justification: "Removing deprecations for 1.10.0" - "1.10.0": - org.apache.iceberg:iceberg-api: - - code: "java.class.defaultSerializationChanged" - old: "class org.apache.iceberg.encryption.EncryptingFileIO" - new: "class org.apache.iceberg.encryption.EncryptingFileIO" - justification: "New method for Manifest List reading" - org.apache.iceberg:iceberg-core: - - code: "java.class.defaultSerializationChanged" - old: "class org.apache.iceberg.avro.SupportsIndexProjection" - new: "class org.apache.iceberg.avro.SupportsIndexProjection" - justification: "Serialization across versions is not guaranteed" - - code: "java.class.defaultSerializationChanged" - old: "class org.apache.iceberg.hadoop.SerializableConfiguration" - new: "class org.apache.iceberg.hadoop.SerializableConfiguration" - justification: "Serialization across versions is not guaranteed" - - code: "java.class.noLongerInheritsFromClass" - old: "class org.apache.iceberg.rest.auth.OAuth2Manager" - new: "class org.apache.iceberg.rest.auth.OAuth2Manager" - justification: "Removing deprecations for 1.11.0" - - code: "java.class.nowImplementsInterface" - old: "class org.apache.iceberg.rest.auth.OAuth2Manager" - new: "class org.apache.iceberg.rest.auth.OAuth2Manager" - justification: "Removing deprecations for 1.11.0" - - code: "java.class.removed" - old: "class org.apache.iceberg.rest.auth.RefreshingAuthManager" - justification: "Removing deprecations for 1.11.0" - - code: "java.field.constantValueChanged" - old: "field org.apache.iceberg.rest.ResourcePaths.V1_TABLE_SCAN_PLAN" - new: "field org.apache.iceberg.rest.ResourcePaths.V1_TABLE_SCAN_PLAN" - justification: "Plan API is table scoped and path constant value should include namespace. No actual breakage because it never worked before with incorrect value." - - code: "java.field.constantValueChanged" - old: "field org.apache.iceberg.rest.ResourcePaths.V1_TABLE_SCAN_PLAN_SUBMIT" - new: "field org.apache.iceberg.rest.ResourcePaths.V1_TABLE_SCAN_PLAN_SUBMIT" - justification: "Plan API is table scoped and path constant value should include namespace. No actual breakage because it never worked before with incorrect value." - - code: "java.field.constantValueChanged" - old: "field org.apache.iceberg.rest.ResourcePaths.V1_TABLE_SCAN_PLAN_TASKS" - new: "field org.apache.iceberg.rest.ResourcePaths.V1_TABLE_SCAN_PLAN_TASKS" - justification: "Plan API is table scoped and path constant value should include namespace. No actual breakage because it never worked before with incorrect value." - - code: "java.class.removed" - old: "class org.apache.iceberg.PartitionStatsUtil" - justification: "Removing deprecated code for 1.11.0" - - code: "java.method.removed" - old: "method java.lang.String org.apache.iceberg.RewriteTablePathUtil::stagingPath(java.lang.String,\ - \ java.lang.String)" - justification: "Removing deprecated code for 1.11.0" - - code: "java.method.removed" - old: "method org.apache.iceberg.RewriteTablePathUtil.RewriteResult\ - \ org.apache.iceberg.RewriteTablePathUtil::rewriteDataManifest(org.apache.iceberg.ManifestFile,\ - \ org.apache.iceberg.io.OutputFile, org.apache.iceberg.io.FileIO, int, java.util.Map, java.lang.String, java.lang.String) throws\ - \ java.io.IOException" - justification: "Removing deprecated code for 1.11.0" - - code: "java.method.removed" - old: "method org.apache.iceberg.RewriteTablePathUtil.RewriteResult\ - \ org.apache.iceberg.RewriteTablePathUtil::rewriteDeleteManifest(org.apache.iceberg.ManifestFile,\ - \ org.apache.iceberg.io.OutputFile, org.apache.iceberg.io.FileIO, int, java.util.Map, java.lang.String, java.lang.String, java.lang.String)\ - \ throws java.io.IOException" - justification: "Removing deprecated code for 1.11.0" - - code: "java.method.removed" - old: "method org.apache.iceberg.Schema org.apache.iceberg.PartitionStatsHandler::schema(org.apache.iceberg.types.Types.StructType)" - justification: "Removing deprecated code for 1.11.0" - - code: "java.method.removed" - old: "method org.apache.iceberg.TableMetadata org.apache.iceberg.TableMetadataParser::read(org.apache.iceberg.io.FileIO,\ - \ org.apache.iceberg.io.InputFile)" - justification: "Removing deprecated code for 1.11.0" - - code: "java.method.removed" - old: "method org.apache.iceberg.encryption.EncryptionManager org.apache.iceberg.encryption.EncryptionUtil::createEncryptionManager(java.util.Map, org.apache.iceberg.encryption.KeyManagementClient)" - justification: "Removing deprecated code for 1.11.0" - - code: "java.method.removed" - old: "method org.apache.iceberg.rest.responses.OAuthTokenResponse org.apache.iceberg.rest.auth.OAuth2Util::exchangeToken(org.apache.iceberg.rest.RESTClient,\ - \ java.util.Map, java.lang.String, java.lang.String,\ - \ java.lang.String, java.lang.String, java.lang.String)" - justification: "Removing deprecated code for 1.11.0" - - code: "java.method.removed" - old: "method org.apache.iceberg.rest.responses.OAuthTokenResponse org.apache.iceberg.rest.auth.OAuth2Util::exchangeToken(org.apache.iceberg.rest.RESTClient,\ - \ java.util.Map, java.lang.String, java.lang.String,\ - \ java.lang.String, java.lang.String, java.lang.String, java.lang.String)" - justification: "Removing deprecated code for 1.11.0" - - code: "java.method.removed" - old: "method org.apache.iceberg.rest.responses.OAuthTokenResponse org.apache.iceberg.rest.auth.OAuth2Util::fetchToken(org.apache.iceberg.rest.RESTClient,\ - \ java.util.Map, java.lang.String, java.lang.String)" - justification: "Removing deprecated code for 1.11.0" - - code: "java.method.removed" - old: "method org.apache.iceberg.rest.responses.OAuthTokenResponse org.apache.iceberg.rest.auth.OAuth2Util::fetchToken(org.apache.iceberg.rest.RESTClient,\ - \ java.util.Map, java.lang.String, java.lang.String,\ - \ java.lang.String)" - justification: "Removing deprecated code for 1.11.0" - - code: "java.method.visibilityReduced" - old: "method void org.apache.iceberg.PartitionStats::liveEntry(org.apache.iceberg.ContentFile, org.apache.iceberg.Snapshot)" - new: "method void org.apache.iceberg.PartitionStats::liveEntry(org.apache.iceberg.ContentFile, org.apache.iceberg.Snapshot)" - justification: "Changing deprecated code" - - code: "java.method.visibilityReduced" - old: "method void org.apache.iceberg.PartitionStats::appendStats(org.apache.iceberg.PartitionStats)" - new: "method void org.apache.iceberg.PartitionStats::appendStats(org.apache.iceberg.PartitionStats)" - justification: "Changing deprecated code" - - code: "java.method.visibilityReduced" - old: "method void org.apache.iceberg.PartitionStats::deletedEntry(org.apache.iceberg.Snapshot)" - new: "method void org.apache.iceberg.PartitionStats::deletedEntry(org.apache.iceberg.Snapshot)" - justification: "Changing deprecated code" - org.apache.iceberg:iceberg-data: - - code: "java.class.removed" - old: "class org.apache.iceberg.data.PartitionStatsHandler" - justification: "Removing deprecated code for 1.11.0" apache-iceberg-0.14.0: org.apache.iceberg:iceberg-api: - code: "java.class.defaultSerializationChanged" diff --git a/core/src/main/java/org/apache/iceberg/BaseDistributedDataScan.java b/core/src/main/java/org/apache/iceberg/BaseDistributedDataScan.java index e951ae830737..e047f3f92854 100644 --- a/core/src/main/java/org/apache/iceberg/BaseDistributedDataScan.java +++ b/core/src/main/java/org/apache/iceberg/BaseDistributedDataScan.java @@ -194,7 +194,7 @@ public CloseableIterable> planTasks() { } private List findMatchingDataManifests(Snapshot snapshot) { - List dataManifests = snapshot.dataManifests(io()); + List dataManifests = snapshot.dataManifests(fileIO().get()); scanMetrics().totalDataManifests().increment(dataManifests.size()); List matchingDataManifests = filterManifests(dataManifests); @@ -205,7 +205,7 @@ private List findMatchingDataManifests(Snapshot snapshot) { } private List findMatchingDeleteManifests(Snapshot snapshot) { - List deleteManifests = snapshot.deleteManifests(io()); + List deleteManifests = snapshot.deleteManifests(fileIO().get()); scanMetrics().totalDeleteManifests().increment(deleteManifests.size()); List matchingDeleteManifests = filterManifests(deleteManifests); @@ -293,7 +293,7 @@ private CompletableFuture newDeletesFuture( } private DeleteFileIndex planDeletesLocally(List deleteManifests) { - DeleteFileIndex.Builder builder = DeleteFileIndex.builderFor(io(), deleteManifests); + DeleteFileIndex.Builder builder = DeleteFileIndex.builderFor(fileIO().get(), deleteManifests); if (shouldPlanWithExecutor() && deleteManifests.size() > 1) { builder.planWith(planExecutor()); diff --git a/core/src/main/java/org/apache/iceberg/BaseScan.java b/core/src/main/java/org/apache/iceberg/BaseScan.java index 242a5aaacc09..3c5692c50d2b 100644 --- a/core/src/main/java/org/apache/iceberg/BaseScan.java +++ b/core/src/main/java/org/apache/iceberg/BaseScan.java @@ -103,14 +103,6 @@ public Table table() { return table; } - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link BaseScan#fileIO()} instead. - */ - @Deprecated - protected FileIO io() { - return table.io(); - } - @Override public Supplier fileIO() { return table::io; diff --git a/core/src/main/java/org/apache/iceberg/DataScan.java b/core/src/main/java/org/apache/iceberg/DataScan.java index 336824e4cc91..087875ab5c64 100644 --- a/core/src/main/java/org/apache/iceberg/DataScan.java +++ b/core/src/main/java/org/apache/iceberg/DataScan.java @@ -49,7 +49,7 @@ protected ManifestGroup newManifestGroup( boolean withColumnStats) { ManifestGroup manifestGroup = - new ManifestGroup(io(), dataManifests, deleteManifests) + new ManifestGroup(fileIO().get(), dataManifests, deleteManifests) .caseSensitive(isCaseSensitive()) .select(withColumnStats ? SCAN_WITH_STATS_COLUMNS : SCAN_COLUMNS) .filterData(filter()) diff --git a/core/src/main/java/org/apache/iceberg/ManifestFiles.java b/core/src/main/java/org/apache/iceberg/ManifestFiles.java index 5ac55f0cf41f..008f60caac62 100644 --- a/core/src/main/java/org/apache/iceberg/ManifestFiles.java +++ b/core/src/main/java/org/apache/iceberg/ManifestFiles.java @@ -108,39 +108,6 @@ public static CloseableIterable readPaths( entry -> entry.file().location()); } - /** - * Returns a {@link CloseableIterable} of file paths in the {@link ManifestFile}. - * - * @param manifest a ManifestFile - * @param io a FileIO - * @return a manifest reader - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link #readPaths(ManifestFile, - * FileIO, Map)} instead. - */ - @Deprecated - public static CloseableIterable readPaths(ManifestFile manifest, FileIO io) { - return readPaths(manifest, io, null); - } - - /** - * Returns a new {@link ManifestReader} for a {@link ManifestFile}. - * - *

Note: Callers should use {@link ManifestFiles#read(ManifestFile, FileIO, Map)} to - * ensure the schema used by filters is the latest table schema. This should be used only when - * reading a manifest without filters. - * - * @param manifest a ManifestFile - * @param io a FileIO - * @return a manifest reader - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link #read(ManifestFile, FileIO, - * Map)} instead. Reading partition specs from manifest file metadata will not be supported - * for non-Avro manifest formats. - */ - @Deprecated - public static ManifestReader read(ManifestFile manifest, FileIO io) { - return read(manifest, io, null); - } - /** * Returns a new {@link ManifestReader} for a {@link ManifestFile}. * @@ -437,15 +404,6 @@ public static ManifestFile decode(byte[] manifestData) throws IOException { return AvroEncoderUtil.decode(manifestData); } - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link #open(ManifestFile, FileIO, - * Map)} instead. - */ - @Deprecated - static ManifestReader open(ManifestFile manifest, FileIO io) { - return open(manifest, io, null); - } - static ManifestReader open( ManifestFile manifest, FileIO io, Map specsById) { switch (manifest.content()) { diff --git a/core/src/main/java/org/apache/iceberg/MetricsConfig.java b/core/src/main/java/org/apache/iceberg/MetricsConfig.java index 2b55bcbeab22..0de04596c29a 100644 --- a/core/src/main/java/org/apache/iceberg/MetricsConfig.java +++ b/core/src/main/java/org/apache/iceberg/MetricsConfig.java @@ -32,7 +32,6 @@ import javax.annotation.concurrent.Immutable; import org.apache.iceberg.MetricsModes.MetricsMode; import org.apache.iceberg.exceptions.ValidationException; -import org.apache.iceberg.relocated.com.google.common.base.Joiner; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; @@ -49,7 +48,6 @@ public final class MetricsConfig implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(MetricsConfig.class); - private static final Joiner DOT = Joiner.on('.'); // Disable metrics by default for wide tables to prevent excessive metadata private static final MetricsMode DEFAULT_MODE = @@ -100,34 +98,6 @@ public static MetricsConfig forTable(Table table) { return from(table.properties(), table.schema(), table.sortOrder()); } - /** - * Creates a metrics config for a position delete file. - * - * @param table an Iceberg table - * @deprecated This method is deprecated as of version 1.11.0 and will be removed in 1.12.0. - * Position deletes that include row data are no longer supported. Use {@link - * #forPositionDelete()} instead. - */ - @Deprecated - public static MetricsConfig forPositionDelete(Table table) { - ImmutableMap.Builder columnModes = ImmutableMap.builder(); - - columnModes.put(MetadataColumns.DELETE_FILE_PATH.name(), MetricsModes.Full.get()); - columnModes.put(MetadataColumns.DELETE_FILE_POS.name(), MetricsModes.Full.get()); - - MetricsConfig tableConfig = forTable(table); - - MetricsMode defaultMode = tableConfig.defaultMode; - tableConfig.columnModes.forEach( - (columnAlias, mode) -> { - String positionDeleteColumnAlias = - DOT.join(MetadataColumns.DELETE_FILE_ROW_FIELD_NAME, columnAlias); - columnModes.put(positionDeleteColumnAlias, mode); - }); - - return new MetricsConfig(columnModes.build(), defaultMode); - } - static Set limitFieldIds(Schema schema, int limit) { return TypeUtil.visit( schema, diff --git a/core/src/main/java/org/apache/iceberg/PartitionStats.java b/core/src/main/java/org/apache/iceberg/PartitionStats.java deleted file mode 100644 index e8a4e18916bc..000000000000 --- a/core/src/main/java/org/apache/iceberg/PartitionStats.java +++ /dev/null @@ -1,317 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg; - -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; - -/** - * Class to hold partition statistics values. - * - * @deprecated will be removed in 1.12.0. Use {@link BasePartitionStatistics instead} - */ -@Deprecated -public class PartitionStats implements StructLike { - - private static final int STATS_COUNT = 13; - - private StructLike partition; - private int specId; - private long dataRecordCount; - private int dataFileCount; - private long totalDataFileSizeInBytes; - private long positionDeleteRecordCount; // also includes dv record count as per spec - private int positionDeleteFileCount; - private long equalityDeleteRecordCount; - private int equalityDeleteFileCount; - private Long totalRecordCount; // null by default - private Long lastUpdatedAt; // null by default - private Long lastUpdatedSnapshotId; // null by default - private int dvCount; - - public PartitionStats(StructLike partition, int specId) { - this.partition = partition; - this.specId = specId; - } - - public StructLike partition() { - return partition; - } - - public int specId() { - return specId; - } - - public long dataRecordCount() { - return dataRecordCount; - } - - public int dataFileCount() { - return dataFileCount; - } - - public long totalDataFileSizeInBytes() { - return totalDataFileSizeInBytes; - } - - public long positionDeleteRecordCount() { - return positionDeleteRecordCount; - } - - public int positionDeleteFileCount() { - return positionDeleteFileCount; - } - - public long equalityDeleteRecordCount() { - return equalityDeleteRecordCount; - } - - public int equalityDeleteFileCount() { - return equalityDeleteFileCount; - } - - public Long totalRecords() { - return totalRecordCount; - } - - public Long lastUpdatedAt() { - return lastUpdatedAt; - } - - public Long lastUpdatedSnapshotId() { - return lastUpdatedSnapshotId; - } - - public int dvCount() { - return dvCount; - } - - /** - * Updates the partition stats from the data/delete file. - * - * @param file the {@link ContentFile} from the manifest entry. - * @param snapshot the snapshot corresponding to the live entry. - */ - void liveEntry(ContentFile file, Snapshot snapshot) { - Preconditions.checkArgument(specId == file.specId(), "Spec IDs must match"); - - switch (file.content()) { - case DATA: - this.dataRecordCount += file.recordCount(); - this.dataFileCount += 1; - this.totalDataFileSizeInBytes += file.fileSizeInBytes(); - break; - case POSITION_DELETES: - this.positionDeleteRecordCount += file.recordCount(); - if (file.format() == FileFormat.PUFFIN) { - this.dvCount += 1; - } else { - this.positionDeleteFileCount += 1; - } - - break; - case EQUALITY_DELETES: - this.equalityDeleteRecordCount += file.recordCount(); - this.equalityDeleteFileCount += 1; - break; - default: - throw new UnsupportedOperationException("Unsupported file content type: " + file.content()); - } - - if (snapshot != null) { - updateSnapshotInfo(snapshot.snapshotId(), snapshot.timestampMillis()); - } - - // Note: Not computing the `TOTAL_RECORD_COUNT` for now as it needs scanning the data. - } - - /** - * Updates the modified time and snapshot ID for the deleted manifest entry. - * - * @param snapshot the snapshot corresponding to the deleted manifest entry. - */ - void deletedEntry(Snapshot snapshot) { - if (snapshot != null) { - updateSnapshotInfo(snapshot.snapshotId(), snapshot.timestampMillis()); - } - } - - /** - * Decrement the counters as it was included in the previous stats and updates the modified time - * and snapshot ID for the deleted manifest entry. - * - * @param snapshot the snapshot corresponding to the deleted manifest entry. - */ - void deletedEntryForIncrementalCompute(ContentFile file, Snapshot snapshot) { - Preconditions.checkArgument(specId == file.specId(), "Spec IDs must match"); - - switch (file.content()) { - case DATA: - this.dataRecordCount -= file.recordCount(); - this.dataFileCount -= 1; - this.totalDataFileSizeInBytes -= file.fileSizeInBytes(); - break; - case POSITION_DELETES: - this.positionDeleteRecordCount -= file.recordCount(); - if (file.format() == FileFormat.PUFFIN) { - this.dvCount -= 1; - } else { - this.positionDeleteFileCount -= 1; - } - - break; - case EQUALITY_DELETES: - this.equalityDeleteRecordCount -= file.recordCount(); - this.equalityDeleteFileCount -= 1; - break; - default: - throw new UnsupportedOperationException("Unsupported file content type: " + file.content()); - } - - if (snapshot != null) { - updateSnapshotInfo(snapshot.snapshotId(), snapshot.timestampMillis()); - } - } - - /** - * Appends statistics from given entry to current entry. - * - * @param entry the entry from which statistics will be sourced. - */ - void appendStats(PartitionStats entry) { - Preconditions.checkArgument(specId == entry.specId(), "Spec IDs must match"); - - this.dataRecordCount += entry.dataRecordCount; - this.dataFileCount += entry.dataFileCount; - this.totalDataFileSizeInBytes += entry.totalDataFileSizeInBytes; - this.positionDeleteRecordCount += entry.positionDeleteRecordCount; - this.positionDeleteFileCount += entry.positionDeleteFileCount; - this.equalityDeleteRecordCount += entry.equalityDeleteRecordCount; - this.equalityDeleteFileCount += entry.equalityDeleteFileCount; - this.dvCount += entry.dvCount; - - if (entry.totalRecordCount != null) { - if (totalRecordCount == null) { - this.totalRecordCount = entry.totalRecordCount; - } else { - this.totalRecordCount += entry.totalRecordCount; - } - } - - if (entry.lastUpdatedAt != null) { - updateSnapshotInfo(entry.lastUpdatedSnapshotId, entry.lastUpdatedAt); - } - } - - private void updateSnapshotInfo(long snapshotId, long updatedAt) { - if (lastUpdatedAt == null || lastUpdatedAt < updatedAt) { - this.lastUpdatedAt = updatedAt; - this.lastUpdatedSnapshotId = snapshotId; - } - } - - @Override - public int size() { - return STATS_COUNT; - } - - @Override - public T get(int pos, Class javaClass) { - switch (pos) { - case 0: - return javaClass.cast(partition); - case 1: - return javaClass.cast(specId); - case 2: - return javaClass.cast(dataRecordCount); - case 3: - return javaClass.cast(dataFileCount); - case 4: - return javaClass.cast(totalDataFileSizeInBytes); - case 5: - return javaClass.cast(positionDeleteRecordCount); - case 6: - return javaClass.cast(positionDeleteFileCount); - case 7: - return javaClass.cast(equalityDeleteRecordCount); - case 8: - return javaClass.cast(equalityDeleteFileCount); - case 9: - return javaClass.cast(totalRecordCount); - case 10: - return javaClass.cast(lastUpdatedAt); - case 11: - return javaClass.cast(lastUpdatedSnapshotId); - case 12: - return javaClass.cast(dvCount); - default: - throw new UnsupportedOperationException("Unknown position: " + pos); - } - } - - @Override - public void set(int pos, T value) { - switch (pos) { - case 0: - this.partition = (StructLike) value; - break; - case 1: - this.specId = (int) value; - break; - case 2: - this.dataRecordCount = (long) value; - break; - case 3: - this.dataFileCount = (int) value; - break; - case 4: - this.totalDataFileSizeInBytes = (long) value; - break; - case 5: - // optional field as per spec, implementation initialize to 0 for counters - this.positionDeleteRecordCount = value == null ? 0L : (long) value; - break; - case 6: - // optional field as per spec, implementation initialize to 0 for counters - this.positionDeleteFileCount = value == null ? 0 : (int) value; - break; - case 7: - // optional field as per spec, implementation initialize to 0 for counters - this.equalityDeleteRecordCount = value == null ? 0L : (long) value; - break; - case 8: - // optional field as per spec, implementation initialize to 0 for counters - this.equalityDeleteFileCount = value == null ? 0 : (int) value; - break; - case 9: - this.totalRecordCount = (Long) value; - break; - case 10: - this.lastUpdatedAt = (Long) value; - break; - case 11: - this.lastUpdatedSnapshotId = (Long) value; - break; - case 12: - this.dvCount = value == null ? 0 : (int) value; - break; - default: - throw new UnsupportedOperationException("Unknown position: " + pos); - } - } -} diff --git a/core/src/main/java/org/apache/iceberg/PartitionStatsHandler.java b/core/src/main/java/org/apache/iceberg/PartitionStatsHandler.java index 9420095f94a3..ffd9aedce571 100644 --- a/core/src/main/java/org/apache/iceberg/PartitionStatsHandler.java +++ b/core/src/main/java/org/apache/iceberg/PartitionStatsHandler.java @@ -33,20 +33,14 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.apache.iceberg.data.GenericRecord; -import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.FileAppender; -import org.apache.iceberg.io.InputFile; import org.apache.iceberg.io.OutputFile; import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Queues; import org.apache.iceberg.types.Comparators; -import org.apache.iceberg.types.Types; -import org.apache.iceberg.types.Types.IntegerType; -import org.apache.iceberg.types.Types.LongType; -import org.apache.iceberg.types.Types.NestedField; import org.apache.iceberg.types.Types.StructType; import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.PartitionMap; @@ -67,180 +61,6 @@ private PartitionStatsHandler() {} private static final Logger LOG = LoggerFactory.getLogger(PartitionStatsHandler.class); - // schema of the partition stats file as per spec - /** - * @deprecated will be removed in 1.12.0. Use {@link PartitionStatistics#EMPTY_PARTITION_FIELD} - */ - @Deprecated public static final int PARTITION_FIELD_ID = 1; - - /** - * @deprecated will be removed in 1.12.0. Use {@link PartitionStatistics#EMPTY_PARTITION_FIELD} - */ - @Deprecated public static final String PARTITION_FIELD_NAME = "partition"; - - /** - * @deprecated will be removed in 1.12.0. Use {@link PartitionStatistics#SPEC_ID} - */ - @Deprecated - public static final NestedField SPEC_ID = NestedField.required(2, "spec_id", IntegerType.get()); - - /** - * @deprecated will be removed in 1.12.0. Use {@link PartitionStatistics#DATA_RECORD_COUNT} - */ - @Deprecated - public static final NestedField DATA_RECORD_COUNT = - NestedField.required(3, "data_record_count", LongType.get()); - - /** - * @deprecated will be removed in 1.12.0. Use {@link PartitionStatistics#DATA_FILE_COUNT} - */ - @Deprecated - public static final NestedField DATA_FILE_COUNT = - NestedField.required(4, "data_file_count", IntegerType.get()); - - /** - * @deprecated will be removed in 1.12.0. Use {@link - * PartitionStatistics#TOTAL_DATA_FILE_SIZE_IN_BYTES} - */ - @Deprecated - public static final NestedField TOTAL_DATA_FILE_SIZE_IN_BYTES = - NestedField.required(5, "total_data_file_size_in_bytes", LongType.get()); - - /** - * @deprecated will be removed in 1.12.0. Use {@link - * PartitionStatistics#POSITION_DELETE_RECORD_COUNT} - */ - @Deprecated - public static final NestedField POSITION_DELETE_RECORD_COUNT = - NestedField.optional(6, "position_delete_record_count", LongType.get()); - - /** - * @deprecated will be removed in 1.12.0. Use {@link - * PartitionStatistics#POSITION_DELETE_FILE_COUNT} - */ - @Deprecated - public static final NestedField POSITION_DELETE_FILE_COUNT = - NestedField.optional(7, "position_delete_file_count", IntegerType.get()); - - /** - * @deprecated will be removed in 1.12.0. Use {@link - * PartitionStatistics#EQUALITY_DELETE_RECORD_COUNT} - */ - @Deprecated - public static final NestedField EQUALITY_DELETE_RECORD_COUNT = - NestedField.optional(8, "equality_delete_record_count", LongType.get()); - - /** - * @deprecated will be removed in 1.12.0. Use {@link - * PartitionStatistics#EQUALITY_DELETE_FILE_COUNT} - */ - @Deprecated - public static final NestedField EQUALITY_DELETE_FILE_COUNT = - NestedField.optional(9, "equality_delete_file_count", IntegerType.get()); - - /** - * @deprecated will be removed in 1.12.0. Use {@link PartitionStatistics#TOTAL_RECORD_COUNT} - */ - @Deprecated - public static final NestedField TOTAL_RECORD_COUNT = - NestedField.optional(10, "total_record_count", LongType.get()); - - /** - * @deprecated will be removed in 1.12.0. Use {@link PartitionStatistics#LAST_UPDATED_AT} - */ - @Deprecated - public static final NestedField LAST_UPDATED_AT = - NestedField.optional(11, "last_updated_at", LongType.get()); - - /** - * @deprecated will be removed in 1.12.0. Use {@link PartitionStatistics#LAST_UPDATED_SNAPSHOT_ID} - */ - @Deprecated - public static final NestedField LAST_UPDATED_SNAPSHOT_ID = - NestedField.optional(12, "last_updated_snapshot_id", LongType.get()); - - /** - * @deprecated will be removed in 1.12.0. Use {@link PartitionStatistics#DV_COUNT} - */ - @Deprecated - public static final NestedField DV_COUNT = - NestedField.required("dv_count") - .withId(13) - .ofType(Types.IntegerType.get()) - .withInitialDefault(Literal.of(0)) - .withWriteDefault(Literal.of(0)) - .build(); - - /** - * Generates the partition stats file schema for a given format version based on a combined - * partition type which considers all specs in a table. - * - * @param unifiedPartitionType unified partition schema type. Could be calculated by {@link - * Partitioning#partitionType(Table)}. - * @return a schema that corresponds to the provided unified partition type. - * @deprecated will be removed in 1.12.0. Use {@link PartitionStatistics#schema(StructType, int)} - * instead. - */ - @Deprecated - public static Schema schema(StructType unifiedPartitionType, int formatVersion) { - Preconditions.checkState(!unifiedPartitionType.fields().isEmpty(), "Table must be partitioned"); - Preconditions.checkState( - formatVersion > 0 && formatVersion <= TableMetadata.SUPPORTED_TABLE_FORMAT_VERSION, - "Invalid format version: %d", - formatVersion); - - if (formatVersion <= 2) { - return v2Schema(unifiedPartitionType); - } - - return v3Schema(unifiedPartitionType); - } - - private static Schema v2Schema(StructType unifiedPartitionType) { - return new Schema( - NestedField.required(PARTITION_FIELD_ID, PARTITION_FIELD_NAME, unifiedPartitionType), - SPEC_ID, - DATA_RECORD_COUNT, - DATA_FILE_COUNT, - TOTAL_DATA_FILE_SIZE_IN_BYTES, - POSITION_DELETE_RECORD_COUNT, - POSITION_DELETE_FILE_COUNT, - EQUALITY_DELETE_RECORD_COUNT, - EQUALITY_DELETE_FILE_COUNT, - TOTAL_RECORD_COUNT, - LAST_UPDATED_AT, - LAST_UPDATED_SNAPSHOT_ID); - } - - private static Schema v3Schema(StructType unifiedPartitionType) { - return new Schema( - NestedField.required(PARTITION_FIELD_ID, PARTITION_FIELD_NAME, unifiedPartitionType), - SPEC_ID, - DATA_RECORD_COUNT, - DATA_FILE_COUNT, - TOTAL_DATA_FILE_SIZE_IN_BYTES, - NestedField.required( - POSITION_DELETE_RECORD_COUNT.fieldId(), - POSITION_DELETE_RECORD_COUNT.name(), - LongType.get()), - NestedField.required( - POSITION_DELETE_FILE_COUNT.fieldId(), - POSITION_DELETE_FILE_COUNT.name(), - IntegerType.get()), - NestedField.required( - EQUALITY_DELETE_RECORD_COUNT.fieldId(), - EQUALITY_DELETE_RECORD_COUNT.name(), - LongType.get()), - NestedField.required( - EQUALITY_DELETE_FILE_COUNT.fieldId(), - EQUALITY_DELETE_FILE_COUNT.name(), - IntegerType.get()), - TOTAL_RECORD_COUNT, - LAST_UPDATED_AT, - LAST_UPDATED_SNAPSHOT_ID, - DV_COUNT); - } - /** * Computes the stats incrementally after the snapshot that has partition stats file till the * current snapshot and writes the combined result into a {@link PartitionStatisticsFile} after @@ -343,28 +163,6 @@ static PartitionStatisticsFile writePartitionStatsFile( .build(); } - /** - * Reads partition statistics from the specified {@link InputFile} using given schema. - * - * @param schema The {@link Schema} of the partition statistics file. - * @param inputFile An {@link InputFile} pointing to the partition stats file. - * @deprecated will be removed in 1.12.0, use {@link PartitionStatisticsScan} instead - */ - @Deprecated - public static CloseableIterable readPartitionStatsFile( - Schema schema, InputFile inputFile) { - Preconditions.checkArgument(schema != null, "Invalid schema: null"); - Preconditions.checkArgument(inputFile != null, "Invalid input file: null"); - - FileFormat fileFormat = FileFormat.fromFileName(inputFile.location()); - Preconditions.checkArgument( - fileFormat != null, "Unable to determine format of file: %s", inputFile.location()); - - CloseableIterable records = - InternalData.read(fileFormat, inputFile).project(schema).build(); - return CloseableIterable.transform(records, PartitionStatsHandler::recordToPartitionStats); - } - private static OutputFile newPartitionStatsFile( Table table, FileFormat fileFormat, long snapshotId) { Preconditions.checkArgument( @@ -382,19 +180,6 @@ private static OutputFile newPartitionStatsFile( Locale.ROOT, "partition-stats-%d-%s", snapshotId, UUID.randomUUID())))); } - private static PartitionStats recordToPartitionStats(StructLike record) { - int pos = 0; - PartitionStats stats = - new PartitionStats( - record.get(pos++, StructLike.class), // partition - record.get(pos++, Integer.class)); // spec id - for (; pos < record.size(); pos++) { - stats.set(pos, record.get(pos, Object.class)); - } - - return stats; - } - private static Collection computeAndMergeStatsIncremental( Table table, Snapshot snapshot, long lastSnapshotWithStats) { PartitionMap statsMap = PartitionMap.create(table.specs()); diff --git a/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java b/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java index 69f82931833e..5f97a39cef98 100644 --- a/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java +++ b/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java @@ -576,24 +576,8 @@ private static String rewriteReferencedDataFilePathForDV( public interface PositionDeleteReaderWriter extends Serializable { CloseableIterable reader(InputFile inputFile, FileFormat format, PartitionSpec spec); - default PositionDeleteWriter writer( - OutputFile outputFile, FileFormat format, PartitionSpec spec, StructLike partition) - throws IOException { - return writer(outputFile, format, spec, partition, null); - } - - /** - * @deprecated This method is deprecated as of version 1.11.0 and will be removed in 1.12.0. - * Position deletes that include row data are no longer supported. Use {@link - * #writer(OutputFile, FileFormat, PartitionSpec, StructLike)} instead. - */ - @Deprecated PositionDeleteWriter writer( - OutputFile outputFile, - FileFormat format, - PartitionSpec spec, - StructLike partition, - Schema rowSchema) + OutputFile outputFile, FileFormat format, PartitionSpec spec, StructLike partition) throws IOException; } @@ -634,18 +618,16 @@ public static void rewritePositionDeleteFile( try (CloseableIterable reader = posDeleteReaderWriter.reader(sourceFile, deleteFile.format(), spec)) { Record record = null; - Schema rowSchema = null; CloseableIterator recordIt = reader.iterator(); if (recordIt.hasNext()) { record = recordIt.next(); - rowSchema = record.get(2) != null ? spec.schema() : null; } if (record != null) { try (PositionDeleteWriter writer = posDeleteReaderWriter.writer( - outputFile, deleteFile.format(), spec, deleteFile.partition(), rowSchema)) { + outputFile, deleteFile.format(), spec, deleteFile.partition())) { writer.write(newPositionDeleteRecord(record, sourcePrefix, targetPrefix)); @@ -723,7 +705,7 @@ private static PositionDelete newPositionDeleteRecord( + oldPath); } String newPath = newPath(oldPath, sourcePrefix, targetPrefix); - delete.set(newPath, (Long) record.get(1), record.get(2)); + delete.set(newPath, (Long) record.get(1)); return delete; } diff --git a/core/src/main/java/org/apache/iceberg/SystemConfigs.java b/core/src/main/java/org/apache/iceberg/SystemConfigs.java index be59424992ee..66b122003c3b 100644 --- a/core/src/main/java/org/apache/iceberg/SystemConfigs.java +++ b/core/src/main/java/org/apache/iceberg/SystemConfigs.java @@ -80,21 +80,6 @@ private SystemConfigs() {} 8, Integer::parseUnsignedInt); - /** - * @deprecated will be removed in 1.12.0; use name mapping instead - */ - @Deprecated - public static final ConfigEntry NETFLIX_UNSAFE_PARQUET_ID_FALLBACK_ENABLED = - new ConfigEntry<>( - "iceberg.netflix.unsafe-parquet-id-fallback.enabled", - "ICEBERG_NETFLIX_UNSAFE_PARQUET_ID_FALLBACK_ENABLED", - true, - s -> { - LOG.warn( - "Fallback ID assignment in Parquet is UNSAFE and will be removed in 1.12.0. Use name mapping instead."); - return Boolean.parseBoolean(s); - }); - public static class ConfigEntry { private final String propertyKey; private final String envKey; diff --git a/core/src/main/java/org/apache/iceberg/SystemProperties.java b/core/src/main/java/org/apache/iceberg/SystemProperties.java deleted file mode 100644 index 484879bb21b1..000000000000 --- a/core/src/main/java/org/apache/iceberg/SystemProperties.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg; - -/** - * Configuration properties that are controlled by Java system properties. - * - * @deprecated Use {@link SystemConfigs} instead; will be removed in 1.12.0 - */ -@Deprecated -public class SystemProperties { - - private SystemProperties() {} - - /** - * Sets the size of the worker pool. The worker pool limits the number of tasks concurrently - * processing manifests in the base table implementation across all concurrent planning or commit - * operations. - */ - public static final String WORKER_THREAD_POOL_SIZE_PROP = "iceberg.worker.num-threads"; - - /** Whether to use the shared worker pool when planning table scans. */ - public static final String SCAN_THREAD_POOL_ENABLED = "iceberg.scan.plan-in-worker-pool"; - - /** - * Maximum number of distinct {@link org.apache.iceberg.io.FileIO} that is allowed to have - * associated {@link org.apache.iceberg.io.ContentCache} in memory at a time. - */ - public static final String IO_MANIFEST_CACHE_MAX_FILEIO = "iceberg.io.manifest.cache.fileio-max"; - - public static final int IO_MANIFEST_CACHE_MAX_FILEIO_DEFAULT = 8; - - static boolean getBoolean(String systemProperty, boolean defaultValue) { - String value = System.getProperty(systemProperty); - if (value != null) { - return Boolean.parseBoolean(value); - } - - return defaultValue; - } -} diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java b/core/src/main/java/org/apache/iceberg/TableProperties.java index 021ef95d9122..dbd9295a064e 100644 --- a/core/src/main/java/org/apache/iceberg/TableProperties.java +++ b/core/src/main/java/org/apache/iceberg/TableProperties.java @@ -309,16 +309,6 @@ private TableProperties() {} public static final String WRITE_PARTITION_SUMMARY_LIMIT = "write.summary.partition-limit"; public static final int WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT = 0; - /** - * @deprecated will be removed in 1.12.0, writing manifest lists is always enabled - */ - @Deprecated public static final String MANIFEST_LISTS_ENABLED = "write.manifest-lists.enabled"; - - /** - * @deprecated will be removed in 1.12.0, writing manifest lists is always enabled - */ - @Deprecated public static final boolean MANIFEST_LISTS_ENABLED_DEFAULT = true; - public static final String METADATA_COMPRESSION = "write.metadata.compression-codec"; public static final String METADATA_COMPRESSION_DEFAULT = "none"; diff --git a/core/src/main/java/org/apache/iceberg/avro/Avro.java b/core/src/main/java/org/apache/iceberg/avro/Avro.java index 4a5136f58e71..14472fd50718 100644 --- a/core/src/main/java/org/apache/iceberg/avro/Avro.java +++ b/core/src/main/java/org/apache/iceberg/avro/Avro.java @@ -538,26 +538,10 @@ public PositionDeleteWriter buildPositionWriter() throws IOException { Preconditions.checkArgument( spec.isUnpartitioned() || partition != null, "Partition must not be null for partitioned writes"); - Preconditions.checkArgument( - rowSchema == null || createWriterFunc != null, - "Create function should be provided if we write row data"); - meta("delete-type", "position"); - if (rowSchema != null && createWriterFunc != null) { - // the appender uses the row schema wrapped with position fields - appenderBuilder.schema(DeleteSchemaUtil.posDeleteSchema(rowSchema)); - - appenderBuilder.createWriterFunc( - avroSchema -> new PositionAndRowDatumWriter<>(createWriterFunc.apply(avroSchema))); - - } else { - appenderBuilder.schema(DeleteSchemaUtil.pathPosSchema()); - - // We ignore the 'createWriterFunc' and 'rowSchema' even if is provided, since we do not - // write row data itself - appenderBuilder.createWriterFunc(ignored -> new PositionDatumWriter()); - } + appenderBuilder.schema(DeleteSchemaUtil.pathPosSchema()); + appenderBuilder.createWriterFunc(ignored -> new PositionDatumWriter()); appenderBuilder.createContextFunc(WriteBuilder.Context::deleteContext); @@ -586,44 +570,6 @@ public Stream metrics() { } } - /** - * A {@link DatumWriter} implementation that wraps another to produce position deletes with row - * data. - * - * @param the type of datum written as a deleted row - */ - private static class PositionAndRowDatumWriter - implements MetricsAwareDatumWriter> { - private static final ValueWriter PATH_WRITER = ValueWriters.strings(); - private static final ValueWriter POS_WRITER = ValueWriters.longs(); - - private final DatumWriter rowWriter; - - private PositionAndRowDatumWriter(DatumWriter rowWriter) { - this.rowWriter = rowWriter; - } - - @Override - public void setSchema(Schema schema) { - Schema.Field rowField = schema.getField("row"); - if (rowField != null) { - rowWriter.setSchema(rowField.schema()); - } - } - - @Override - public void write(PositionDelete delete, Encoder out) throws IOException { - PATH_WRITER.write(delete.path(), out); - POS_WRITER.write(delete.pos(), out); - rowWriter.write(delete.row(), out); - } - - @Override - public Stream metrics() { - return Stream.concat(PATH_WRITER.metrics(), POS_WRITER.metrics()); - } - } - public static ReadBuilder read(InputFile file) { return new ReadBuilder(file); } diff --git a/core/src/main/java/org/apache/iceberg/avro/AvroSchemaUtil.java b/core/src/main/java/org/apache/iceberg/avro/AvroSchemaUtil.java index c67a3089a6bf..e0e940622342 100644 --- a/core/src/main/java/org/apache/iceberg/avro/AvroSchemaUtil.java +++ b/core/src/main/java/org/apache/iceberg/avro/AvroSchemaUtil.java @@ -128,16 +128,6 @@ public static Schema pruneColumns(Schema schema, Set selectedIds) { return new PruneColumns(selectedIds, null).rootSchema(schema); } - /** - * @deprecated will be removed in 1.12.0; use applyNameMapping and pruneColumns(Schema, Set) - * instead. - */ - @Deprecated - public static Schema pruneColumns( - Schema schema, Set selectedIds, NameMapping nameMapping) { - return new PruneColumns(selectedIds, nameMapping).rootSchema(schema); - } - public static Schema buildAvroProjection( Schema schema, org.apache.iceberg.Schema expected, Map renames) { return AvroCustomOrderSchemaVisitor.visit(schema, new BuildAvroProjection(expected, renames)); diff --git a/core/src/main/java/org/apache/iceberg/data/avro/DataReader.java b/core/src/main/java/org/apache/iceberg/data/avro/DataReader.java deleted file mode 100644 index 5f813f8db576..000000000000 --- a/core/src/main/java/org/apache/iceberg/data/avro/DataReader.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.data.avro; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.function.Supplier; -import org.apache.avro.LogicalType; -import org.apache.avro.LogicalTypes; -import org.apache.avro.Schema; -import org.apache.avro.io.DatumReader; -import org.apache.avro.io.Decoder; -import org.apache.iceberg.avro.AvroSchemaUtil; -import org.apache.iceberg.avro.AvroSchemaWithTypeVisitor; -import org.apache.iceberg.avro.SupportsRowPosition; -import org.apache.iceberg.avro.ValueReader; -import org.apache.iceberg.avro.ValueReaders; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.Types; - -/** - * @deprecated will be removed in 1.12.0; use {@link PlannedDataReader} instead. - */ -@Deprecated -public class DataReader implements DatumReader, SupportsRowPosition { - - public static DataReader create( - org.apache.iceberg.Schema expectedSchema, Schema readSchema) { - return create(expectedSchema, readSchema, ImmutableMap.of()); - } - - public static DataReader create( - org.apache.iceberg.Schema expectedSchema, Schema readSchema, Map idToConstant) { - return new DataReader<>(expectedSchema, readSchema, idToConstant); - } - - private final Schema readSchema; - private final ValueReader reader; - private Schema fileSchema = null; - - @SuppressWarnings("unchecked") - protected DataReader( - org.apache.iceberg.Schema expectedSchema, Schema readSchema, Map idToConstant) { - this.readSchema = readSchema; - this.reader = - (ValueReader) - AvroSchemaWithTypeVisitor.visit( - expectedSchema, readSchema, new ReadBuilder(idToConstant)); - } - - @Override - public void setSchema(Schema newFileSchema) { - this.fileSchema = Schema.applyAliases(newFileSchema, readSchema); - } - - @Override - public T read(T reuse, Decoder decoder) throws IOException { - return DecoderResolver.resolveAndRead(decoder, readSchema, fileSchema, reader, reuse); - } - - @Override - public void setRowPositionSupplier(Supplier posSupplier) { - if (reader instanceof SupportsRowPosition) { - ((SupportsRowPosition) reader).setRowPositionSupplier(posSupplier); - } - } - - protected ValueReader createStructReader( - Types.StructType struct, List> fields, Map idToConstant) { - return GenericReaders.struct(struct, fields, idToConstant); - } - - private class ReadBuilder extends AvroSchemaWithTypeVisitor> { - private final Map idToConstant; - - private ReadBuilder(Map idToConstant) { - this.idToConstant = idToConstant; - } - - @Override - public ValueReader record( - Types.StructType struct, Schema record, List names, List> fields) { - return createStructReader(struct, fields, idToConstant); - } - - @Override - public ValueReader union(Type ignored, Schema union, List> options) { - return ValueReaders.union(options); - } - - @Override - public ValueReader array( - Types.ListType ignored, Schema array, ValueReader elementReader) { - return ValueReaders.array(elementReader); - } - - @Override - public ValueReader map( - Types.MapType iMap, Schema map, ValueReader keyReader, ValueReader valueReader) { - return ValueReaders.arrayMap(keyReader, valueReader); - } - - @Override - public ValueReader map(Types.MapType ignored, Schema map, ValueReader valueReader) { - return ValueReaders.map(ValueReaders.strings(), valueReader); - } - - @Override - public ValueReader primitive(Type.PrimitiveType ignored, Schema primitive) { - LogicalType logicalType = primitive.getLogicalType(); - if (logicalType != null) { - switch (logicalType.getName()) { - case "date": - return GenericReaders.dates(); - - case "time-micros": - return GenericReaders.times(); - - case "timestamp-micros": - if (AvroSchemaUtil.isTimestamptz(primitive)) { - return GenericReaders.timestamptz(); - } - return GenericReaders.timestamps(); - - case "timestamp-nanos": - if (AvroSchemaUtil.isTimestamptz(primitive)) { - return GenericReaders.timestamptzNanos(); - } - return GenericReaders.timestampNanos(); - - case "timestamp-millis": - if (AvroSchemaUtil.isTimestamptz(primitive)) { - return GenericReaders.timestamptzMillis(); - } - return GenericReaders.timestampMillis(); - - case "decimal": - return ValueReaders.decimal( - ValueReaders.decimalBytesReader(primitive), - ((LogicalTypes.Decimal) logicalType).getScale()); - - case "uuid": - return ValueReaders.uuids(); - - default: - throw new IllegalArgumentException("Unknown logical type: " + logicalType); - } - } - - switch (primitive.getType()) { - case NULL: - return ValueReaders.nulls(); - case BOOLEAN: - return ValueReaders.booleans(); - case INT: - return ValueReaders.ints(); - case LONG: - return ValueReaders.longs(); - case FLOAT: - return ValueReaders.floats(); - case DOUBLE: - return ValueReaders.doubles(); - case STRING: - // might want to use a binary-backed container like Utf8 - return ValueReaders.strings(); - case FIXED: - return ValueReaders.fixed(primitive.getFixedSize()); - case BYTES: - return ValueReaders.byteBuffers(); - default: - throw new IllegalArgumentException("Unsupported type: " + primitive); - } - } - } -} diff --git a/core/src/main/java/org/apache/iceberg/data/avro/RawDecoder.java b/core/src/main/java/org/apache/iceberg/data/avro/RawDecoder.java index 9f3cdfdad5f4..eba06489d9a4 100644 --- a/core/src/main/java/org/apache/iceberg/data/avro/RawDecoder.java +++ b/core/src/main/java/org/apache/iceberg/data/avro/RawDecoder.java @@ -27,8 +27,6 @@ import org.apache.avro.io.DatumReader; import org.apache.avro.io.DecoderFactory; import org.apache.avro.message.MessageDecoder; -import org.apache.iceberg.avro.ProjectionDatumReader; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; public class RawDecoder extends MessageDecoder.BaseDecoder { private static final ThreadLocal DECODER = new ThreadLocal<>(); @@ -56,26 +54,6 @@ public static RawDecoder create( private final DatumReader reader; - /** - * Creates a new {@link MessageDecoder} that constructs datum instances described by the {@link - * Schema readSchema}. - * - *

The {@code readSchema} is used for the expected schema and the {@code writeSchema} is the - * schema used to decode buffers. The {@code writeSchema} must be the schema that was used to - * encode all buffers decoded by this class. - * - * @deprecated will be removed in 1.12.0; use {@link #create(org.apache.iceberg.Schema, Function, - * Schema)} instead - */ - @Deprecated - public RawDecoder( - org.apache.iceberg.Schema readSchema, - Function> readerFunction, - Schema writeSchema) { - this.reader = new ProjectionDatumReader<>(readerFunction, readSchema, ImmutableMap.of(), null); - this.reader.setSchema(writeSchema); - } - /** * Creates a new {@link MessageDecoder} that constructs datum instances using the {@code reader}. */ diff --git a/core/src/main/java/org/apache/iceberg/deletes/PositionDelete.java b/core/src/main/java/org/apache/iceberg/deletes/PositionDelete.java index c3b6cbaa9bff..8de3be0c04a1 100644 --- a/core/src/main/java/org/apache/iceberg/deletes/PositionDelete.java +++ b/core/src/main/java/org/apache/iceberg/deletes/PositionDelete.java @@ -43,19 +43,6 @@ public PositionDelete set(CharSequence newPath, long newPos) { return this; } - /** - * @deprecated This method is deprecated as of version 1.11.0 and will be removed in 1.12.0. - * Position deletes that include row data are no longer supported. Use {@link - * #set(CharSequence, long)} instead. - */ - @Deprecated - public PositionDelete set(CharSequence newPath, long newPos, R newRow) { - this.path = newPath; - this.pos = newPos; - this.row = newRow; - return this; - } - @Override public int size() { return 3; @@ -69,15 +56,6 @@ public long pos() { return pos; } - /** - * @deprecated This method is deprecated as of version 1.11.0 and will be removed in 1.12.0. - * Position deletes that include row data are no longer supported. - */ - @Deprecated - public R row() { - return row; - } - @Override @SuppressWarnings("unchecked") public T get(int colPos, Class javaClass) { diff --git a/core/src/main/java/org/apache/iceberg/encryption/StandardEncryptionManager.java b/core/src/main/java/org/apache/iceberg/encryption/StandardEncryptionManager.java index bb5126d23e3f..f1b19fa30489 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/StandardEncryptionManager.java +++ b/core/src/main/java/org/apache/iceberg/encryption/StandardEncryptionManager.java @@ -53,15 +53,6 @@ public class StandardEncryptionManager implements EncryptionManager { private transient volatile LoadingCache unwrappedKeyCache; private transient volatile SecureRandom lazyRNG = null; - /** - * @deprecated will be removed in 1.12.0. - */ - @Deprecated - public StandardEncryptionManager( - String tableKeyId, int dataKeyLength, KeyManagementClient kmsClient) { - this(List.of(), tableKeyId, dataKeyLength, kmsClient); - } - /** * @param keys encryption keys from table metadata * @param tableKeyId table encryption key id @@ -137,22 +128,6 @@ private SecureRandom workerRNG() { return lazyRNG; } - /** - * @deprecated will be removed in 1.12.0. - */ - @Deprecated - public ByteBuffer wrapKey(ByteBuffer secretKey) { - return kmsClient.wrapKey(secretKey, tableKeyId); - } - - /** - * @deprecated will be removed in 1.12.0. - */ - @Deprecated - public ByteBuffer unwrapKey(ByteBuffer wrappedSecretKey) { - return kmsClient.unwrapKey(wrappedSecretKey, tableKeyId); - } - Map encryptionKeys() { return encryptionKeys; } diff --git a/core/src/main/java/org/apache/iceberg/hadoop/HadoopFileIO.java b/core/src/main/java/org/apache/iceberg/hadoop/HadoopFileIO.java index 877290f48e3f..fbb50abb85a7 100644 --- a/core/src/main/java/org/apache/iceberg/hadoop/HadoopFileIO.java +++ b/core/src/main/java/org/apache/iceberg/hadoop/HadoopFileIO.java @@ -66,16 +66,7 @@ public class HadoopFileIO implements HadoopConfigurable, DelegateFileIO { public HadoopFileIO() {} public HadoopFileIO(Configuration hadoopConf) { - this(new SerializableConfiguration(hadoopConf)); - } - - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link - * HadoopFileIO#HadoopFileIO(Configuration)} instead. - */ - @Deprecated - public HadoopFileIO(SerializableSupplier hadoopConf) { - this.hadoopConf = hadoopConf; + this.hadoopConf = new SerializableConfiguration(hadoopConf); } public Configuration conf() { @@ -138,10 +129,6 @@ public Configuration getConf() { return hadoopConf.get(); } - /** - * @deprecated since 1.11.0, will be removed in 1.12.0. - */ - @Deprecated @Override public void serializeConfWith( Function> confSerializer) { diff --git a/core/src/main/java/org/apache/iceberg/io/BasePositionDeltaWriter.java b/core/src/main/java/org/apache/iceberg/io/BasePositionDeltaWriter.java index 859d339bd993..9e84a47f08e7 100644 --- a/core/src/main/java/org/apache/iceberg/io/BasePositionDeltaWriter.java +++ b/core/src/main/java/org/apache/iceberg/io/BasePositionDeltaWriter.java @@ -67,7 +67,7 @@ public void update(T row, PartitionSpec spec, StructLike partition) { @Override public void delete(CharSequence path, long pos, T row, PartitionSpec spec, StructLike partition) { - positionDelete.set(path, pos, row); + positionDelete.set(path, pos); deleteWriter.write(positionDelete, spec, partition); } diff --git a/core/src/main/java/org/apache/iceberg/io/BaseTaskWriter.java b/core/src/main/java/org/apache/iceberg/io/BaseTaskWriter.java index b3d260c91ea2..4bde586a0f3b 100644 --- a/core/src/main/java/org/apache/iceberg/io/BaseTaskWriter.java +++ b/core/src/main/java/org/apache/iceberg/io/BaseTaskWriter.java @@ -261,7 +261,7 @@ private EncryptedOutputFile newOutputFile(StructLike partition) { } private void writePosDelete(PathOffset pathOffset) { - positionDelete.set(pathOffset.path, pathOffset.rowOffset, null); + positionDelete.set(pathOffset.path, pathOffset.rowOffset); posDeleteWriter.write(positionDelete, spec, partitionKey); } diff --git a/core/src/main/java/org/apache/iceberg/io/ContentCache.java b/core/src/main/java/org/apache/iceberg/io/ContentCache.java index 484306690c67..22bb8f1e455b 100644 --- a/core/src/main/java/org/apache/iceberg/io/ContentCache.java +++ b/core/src/main/java/org/apache/iceberg/io/ContentCache.java @@ -139,17 +139,6 @@ public void invalidate(String key) { cache.invalidate(key); } - /** - * @deprecated since 1.7.0, will be removed in 1.12.0; This method does only best-effort - * invalidation and is susceptible to a race condition. If the caller changed the state that - * could be cached (perhaps files on the storage) and calls this method, there is no guarantee - * that the cache will not contain stale entries some time after this method returns. - */ - @Deprecated - public void invalidateAll() { - cache.invalidateAll(); - } - public void cleanUp() { cache.cleanUp(); } diff --git a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java index 3a1e62260aae..226ea67d6a21 100644 --- a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java +++ b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java @@ -502,15 +502,6 @@ public static void tableExists(Catalog catalog, TableIdentifier ident) { } } - /** - * @deprecated since 1.11.0, will be removed in 1.12.0. Use {@link #loadTable(Catalog, - * TableIdentifier, SnapshotMode)} instead. - */ - @Deprecated - public static LoadTableResponse loadTable(Catalog catalog, TableIdentifier ident) { - return loadTable(catalog, ident, SnapshotMode.ALL); - } - public static LoadTableResponse loadTable( Catalog catalog, TableIdentifier ident, SnapshotMode mode) { Table table = catalog.loadTable(ident); diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java index 9effb875e05f..57f93829e9bb 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java @@ -116,12 +116,6 @@ public class RESTSessionCatalog extends BaseViewSessionCatalog private static final Logger LOG = LoggerFactory.getLogger(RESTSessionCatalog.class); private static final String DEFAULT_FILE_IO_IMPL = "org.apache.iceberg.io.ResolvingFileIO"; - /** - * @deprecated will be removed in 1.12.0. Use {@link - * org.apache.iceberg.rest.RESTCatalogProperties#PAGE_SIZE} instead. - */ - @Deprecated public static final String REST_PAGE_SIZE = "rest-page-size"; - // these default endpoints must not be updated in order to maintain backwards compatibility with // legacy servers private static final Set DEFAULT_ENDPOINTS = diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTUtil.java b/core/src/main/java/org/apache/iceberg/rest/RESTUtil.java index f4fdf3af26e7..f1ffe58374a3 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTUtil.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTUtil.java @@ -40,20 +40,6 @@ public class RESTUtil { /** The namespace separator as url encoded UTF-8 character */ static final String NAMESPACE_SEPARATOR_URLENCODED_UTF_8 = "%1F"; - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link - * RESTUtil#namespaceToQueryParam(Namespace)}} instead. - */ - @Deprecated - public static final Joiner NAMESPACE_JOINER = Joiner.on(NAMESPACE_SEPARATOR_AS_UNICODE); - - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link - * RESTUtil#namespaceFromQueryParam(String)} instead. - */ - @Deprecated - public static final Splitter NAMESPACE_SPLITTER = Splitter.on(NAMESPACE_SEPARATOR_AS_UNICODE); - public static final String IDEMPOTENCY_KEY_HEADER = "Idempotency-Key"; private RESTUtil() {} @@ -253,24 +239,6 @@ public static Namespace namespaceFromQueryParam( return Namespace.of(splitter.splitToStream(namespace).toArray(String[]::new)); } - /** - * Returns a String representation of a namespace that is suitable for use in a URL / URI. - * - *

This function needs to be called when a namespace is used as a path variable (or query - * parameter etc.), to format the namespace per the spec. - * - *

{@link #decodeNamespace} should be used to parse the namespace from a URL parameter. - * - * @param ns namespace to encode - * @return UTF-8 encoded string representing the namespace, suitable for use as a URL parameter - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link - * RESTUtil#encodeNamespace(Namespace, String)} instead. - */ - @Deprecated - public static String encodeNamespace(Namespace ns) { - return encodeNamespace(ns, NAMESPACE_SEPARATOR_URLENCODED_UTF_8); - } - /** * Returns a String representation of a namespace that is suitable for use in a URL / URI. * @@ -299,22 +267,6 @@ public static String encodeNamespace(Namespace namespace, String separator) { return Joiner.on(separator).join(encodedLevels); } - /** - * Takes in a string representation of a namespace as used for a URL parameter and returns the - * corresponding namespace. - * - *

See also {@link #encodeNamespace} for generating correctly formatted URLs. - * - * @param encodedNs a namespace to decode - * @return a namespace - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link - * RESTUtil#decodeNamespace(String, String)} instead. - */ - @Deprecated - public static Namespace decodeNamespace(String encodedNs) { - return decodeNamespace(encodedNs, NAMESPACE_SEPARATOR_URLENCODED_UTF_8); - } - /** * Takes in a string representation of a namespace as used for a URL parameter and returns the * corresponding namespace. diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/BaseScanTaskResponse.java b/core/src/main/java/org/apache/iceberg/rest/responses/BaseScanTaskResponse.java index e83cacc48459..3d42ffce4726 100644 --- a/core/src/main/java/org/apache/iceberg/rest/responses/BaseScanTaskResponse.java +++ b/core/src/main/java/org/apache/iceberg/rest/responses/BaseScanTaskResponse.java @@ -93,15 +93,6 @@ public B withFileScanTasks(List tasks) { return self(); } - /** - * @deprecated since 1.11.0, will be removed in 1.12.0. - */ - @Deprecated - public B withDeleteFiles(List deleteFilesList) { - this.deleteFiles = DeleteFileSet.of(deleteFilesList); - return self(); - } - /** * @deprecated since 1.11.0, visibility will be reduced in 1.12.0. */ diff --git a/core/src/main/java/org/apache/iceberg/util/SnapshotUtil.java b/core/src/main/java/org/apache/iceberg/util/SnapshotUtil.java index 370bbfed336e..e34cd34dd8b8 100644 --- a/core/src/main/java/org/apache/iceberg/util/SnapshotUtil.java +++ b/core/src/main/java/org/apache/iceberg/util/SnapshotUtil.java @@ -21,20 +21,14 @@ import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; -import java.util.Objects; import java.util.function.Function; import org.apache.iceberg.BaseMetadataTable; -import org.apache.iceberg.DataFile; import org.apache.iceberg.HistoryEntry; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; -import org.apache.iceberg.SnapshotChanges; import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.exceptions.ValidationException; -import org.apache.iceberg.io.CloseableIterable; -import org.apache.iceberg.io.FileIO; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; @@ -284,66 +278,6 @@ private static Iterable toIds(Iterable snapshots) { return Iterables.transform(snapshots, Snapshot::snapshotId); } - /** - * @deprecated will be removed in 1.12.0, use {@link SnapshotChanges} with {@link - * #ancestorsBetween(long, Long, Function)} instead. - */ - @Deprecated - public static List newFiles( - Long baseSnapshotId, long latestSnapshotId, Function lookup, FileIO io) { - List newFiles = Lists.newArrayList(); - Snapshot lastSnapshot = null; - for (Snapshot currentSnapshot : ancestorsOf(latestSnapshotId, lookup)) { - lastSnapshot = currentSnapshot; - if (Objects.equals(currentSnapshot.snapshotId(), baseSnapshotId)) { - return newFiles; - } - - Iterables.addAll(newFiles, currentSnapshot.addedDataFiles(io)); - } - - ValidationException.check( - Objects.equals(lastSnapshot.parentId(), baseSnapshotId), - "Cannot determine history between read snapshot %s and the last known ancestor %s", - baseSnapshotId, - lastSnapshot.snapshotId()); - - return newFiles; - } - - /** - * @deprecated will be removed in 1.12.0, use {@link SnapshotChanges} with {@link - * #ancestorsBetween(long, Long, Function)} instead. - */ - @Deprecated - public static CloseableIterable newFilesBetween( - Long startSnapshotId, long endSnapshotId, Function lookup, FileIO io) { - - List snapshots = Lists.newArrayList(); - Snapshot lastSnapshot = null; - for (Snapshot currentSnapshot : ancestorsOf(endSnapshotId, lookup)) { - lastSnapshot = currentSnapshot; - if (Objects.equals(currentSnapshot.snapshotId(), startSnapshotId)) { - break; - } - - snapshots.add(currentSnapshot); - } - - if (lastSnapshot != null) { - ValidationException.check( - Objects.equals(lastSnapshot.snapshotId(), startSnapshotId) - || Objects.equals(lastSnapshot.parentId(), startSnapshotId), - "Cannot determine history between read snapshot %s and the last known ancestor %s", - startSnapshotId, - lastSnapshot.snapshotId()); - } - - return new ParallelIterable<>( - Iterables.transform(snapshots, snapshot -> snapshot.addedDataFiles(io)), - ThreadPools.getWorkerPool()); - } - /** * Traverses the history of the table's current snapshot and finds the snapshot with the given * snapshot id as its parent. diff --git a/core/src/main/java/org/apache/iceberg/util/TableScanUtil.java b/core/src/main/java/org/apache/iceberg/util/TableScanUtil.java index 6291acbf9c73..f143a300b87c 100644 --- a/core/src/main/java/org/apache/iceberg/util/TableScanUtil.java +++ b/core/src/main/java/org/apache/iceberg/util/TableScanUtil.java @@ -25,7 +25,6 @@ import org.apache.iceberg.BaseCombinedScanTask; import org.apache.iceberg.BaseScanTaskGroup; import org.apache.iceberg.CombinedScanTask; -import org.apache.iceberg.FileContent; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.MergeableScanTask; import org.apache.iceberg.PartitionData; @@ -51,30 +50,6 @@ public class TableScanUtil { private TableScanUtil() {} - /** - * @deprecated since 1.11.0 and will be removed in 1.12.0 - */ - @Deprecated - public static boolean hasDeletes(CombinedScanTask task) { - return task.files().stream().anyMatch(TableScanUtil::hasDeletes); - } - - /** - * This is temporarily introduced since we plan to support pos-delete vectorized read first, then - * get to the equality-delete support. We will remove this method once both are supported. - * - * @deprecated since 1.11.0 and will be removed in 1.12.0 - */ - @Deprecated - public static boolean hasEqDeletes(CombinedScanTask task) { - return task.files().stream() - .anyMatch( - t -> - t.deletes().stream() - .anyMatch( - deleteFile -> deleteFile.content().equals(FileContent.EQUALITY_DELETES))); - } - public static boolean hasDeletes(FileScanTask task) { return !task.deletes().isEmpty(); } diff --git a/core/src/main/java/org/apache/iceberg/util/ThreadPools.java b/core/src/main/java/org/apache/iceberg/util/ThreadPools.java index bb508295ecfa..3026a87a0a1e 100644 --- a/core/src/main/java/org/apache/iceberg/util/ThreadPools.java +++ b/core/src/main/java/org/apache/iceberg/util/ThreadPools.java @@ -34,14 +34,6 @@ public class ThreadPools { private ThreadPools() {} - /** - * @deprecated Use {@link SystemConfigs#WORKER_THREAD_POOL_SIZE} instead. will be removed in - * 1.12.0 - */ - @Deprecated - public static final String WORKER_THREAD_POOL_SIZE_PROP = - SystemConfigs.WORKER_THREAD_POOL_SIZE.propertyKey(); - public static final int WORKER_THREAD_POOL_SIZE = SystemConfigs.WORKER_THREAD_POOL_SIZE.value(); private static final ExecutorService WORKER_POOL = @@ -101,48 +93,6 @@ private static class AuthRefreshPoolHolder { "auth-session-refresh", AUTH_REFRESH_THREAD_POOL_SIZE, Duration.ZERO); } - /** - * Creates a fixed-size thread pool that uses daemon threads. The pool is wrapped with {@link - * MoreExecutors#getExitingExecutorService(ThreadPoolExecutor)}, which registers a shutdown hook - * to ensure the pool terminates when the JVM exits. Important: Even if the pool is - * explicitly shut down using {@link ExecutorService#shutdown()}, the shutdown hook is not - * removed. This can lead to accumulation of shutdown hooks if this method is used repeatedly for - * short-lived thread pools. - * - *

For clarity and to avoid potential issues with shutdown hook accumulation, prefer using - * either {@link #newExitingWorkerPool(String, int)} or {@link #newFixedThreadPool(String, int)}, - * depending on the intended lifecycle of the thread pool. - * - * @deprecated will be removed in 1.12.0. Use {@link #newExitingWorkerPool(String, int)} for - * long-lived thread pools that require a shutdown hook, or {@link #newFixedThreadPool(String, - * int)} for short-lived thread pools where you manage the lifecycle. - */ - @Deprecated - public static ExecutorService newWorkerPool(String namePrefix) { - return newExitingWorkerPool(namePrefix, WORKER_THREAD_POOL_SIZE); - } - - /** - * Creates a fixed-size thread pool that uses daemon threads. The pool is wrapped with {@link - * MoreExecutors#getExitingExecutorService(ThreadPoolExecutor)}, which registers a shutdown hook - * to ensure the pool terminates when the JVM exits. Important: Even if the pool is - * explicitly shut down using {@link ExecutorService#shutdown()}, the shutdown hook is not - * removed. This can lead to accumulation of shutdown hooks if this method is used repeatedly for - * short-lived thread pools. - * - *

For clarity and to avoid potential issues with shutdown hook accumulation, prefer using - * either {@link #newExitingWorkerPool(String, int)} or {@link #newFixedThreadPool(String, int)}, - * depending on the intended lifecycle of the thread pool. - * - * @deprecated will be removed in 1.12.0. Use {@link #newExitingWorkerPool(String, int)} for - * long-lived thread pools that require a shutdown hook, or {@link #newFixedThreadPool(String, - * int)} for short-lived thread pools where you manage the lifecycle. - */ - @Deprecated - public static ExecutorService newWorkerPool(String namePrefix, int poolSize) { - return newExitingWorkerPool(namePrefix, poolSize); - } - /** * Creates a fixed-size thread pool that uses daemon threads and registers a shutdown hook to * ensure the pool terminates when the JVM exits. This is suitable for long-lived thread pools diff --git a/core/src/test/java/org/apache/iceberg/PartitionStatsHandlerTestBase.java b/core/src/test/java/org/apache/iceberg/PartitionStatsHandlerTestBase.java index 7b7e85bfffce..a2d7b7fa6d4c 100644 --- a/core/src/test/java/org/apache/iceberg/PartitionStatsHandlerTestBase.java +++ b/core/src/test/java/org/apache/iceberg/PartitionStatsHandlerTestBase.java @@ -24,7 +24,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import java.io.IOException; import java.math.BigDecimal; import java.nio.ByteBuffer; import java.util.Arrays; @@ -32,7 +31,6 @@ import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.UUID; import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.io.CloseableIterable; @@ -41,7 +39,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Comparators; import org.apache.iceberg.types.Types; -import org.assertj.core.groups.Tuple; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestTemplate; import org.junit.jupiter.api.extension.ExtendWith; @@ -294,185 +291,6 @@ public void testOptionalFieldsWriting() throws Exception { } } - /** - * @deprecated will be removed in 1.12.0 - */ - @SuppressWarnings("checkstyle:MethodLength") - @Test - @Deprecated - public void testPartitionStats() throws Exception { - Table testTable = - TestTables.create( - tempDir("partition_stats_compute"), - "partition_stats_compute", - SCHEMA, - SPEC, - 2, - fileFormatProperty); - - DataFile dataFile1 = - FileGenerationUtil.generateDataFile(testTable, TestHelpers.Row.of("foo", "A")); - DataFile dataFile2 = - FileGenerationUtil.generateDataFile(testTable, TestHelpers.Row.of("foo", "B")); - DataFile dataFile3 = - FileGenerationUtil.generateDataFile(testTable, TestHelpers.Row.of("bar", "A")); - DataFile dataFile4 = - FileGenerationUtil.generateDataFile(testTable, TestHelpers.Row.of("bar", "B")); - - for (int i = 0; i < 3; i++) { - // insert same set of seven records thrice to have a new manifest files - testTable - .newAppend() - .appendFile(dataFile1) - .appendFile(dataFile2) - .appendFile(dataFile3) - .appendFile(dataFile4) - .commit(); - } - - Snapshot snapshot1 = testTable.currentSnapshot(); - Schema recordSchema = PartitionStatistics.schema(Partitioning.partitionType(testTable), 2); - - Types.StructType partitionType = - recordSchema.findField(EMPTY_PARTITION_FIELD.fieldId()).type().asStructType(); - computeAndValidatePartitionStats( - testTable, - recordSchema, - Tuple.tuple( - partitionRecord(partitionType, "foo", "A"), - 0, - 3 * dataFile1.recordCount(), - 3, - 3 * dataFile1.fileSizeInBytes(), - 0L, - 0, - 0L, - 0, - null, - snapshot1.timestampMillis(), - snapshot1.snapshotId(), - 0), - Tuple.tuple( - partitionRecord(partitionType, "foo", "B"), - 0, - 3 * dataFile2.recordCount(), - 3, - 3 * dataFile2.fileSizeInBytes(), - 0L, - 0, - 0L, - 0, - null, - snapshot1.timestampMillis(), - snapshot1.snapshotId(), - 0), - Tuple.tuple( - partitionRecord(partitionType, "bar", "A"), - 0, - 3 * dataFile3.recordCount(), - 3, - 3 * dataFile3.fileSizeInBytes(), - 0L, - 0, - 0L, - 0, - null, - snapshot1.timestampMillis(), - snapshot1.snapshotId(), - 0), - Tuple.tuple( - partitionRecord(partitionType, "bar", "B"), - 0, - 3 * dataFile4.recordCount(), - 3, - 3 * dataFile4.fileSizeInBytes(), - 0L, - 0, - 0L, - 0, - null, - snapshot1.timestampMillis(), - snapshot1.snapshotId(), - 0)); - - DeleteFile posDelete = - FileGenerationUtil.generatePositionDeleteFile(testTable, TestHelpers.Row.of("bar", "A")); - testTable.newRowDelta().addDeletes(posDelete).commit(); - // snapshot2 is unused in the result as same partition was updated by snapshot4 - - DeleteFile eqDelete = - FileGenerationUtil.generateEqualityDeleteFile(testTable, TestHelpers.Row.of("foo", "A")); - testTable.newRowDelta().addDeletes(eqDelete).commit(); - Snapshot snapshot3 = testTable.currentSnapshot(); - - testTable.updateProperties().set(TableProperties.FORMAT_VERSION, "3").commit(); - DeleteFile dv = FileGenerationUtil.generateDV(testTable, dataFile3); - testTable.newRowDelta().addDeletes(dv).commit(); - Snapshot snapshot4 = testTable.currentSnapshot(); - - recordSchema = PartitionStatistics.schema(Partitioning.partitionType(testTable), 3); - - computeAndValidatePartitionStats( - testTable, - recordSchema, - Tuple.tuple( - partitionRecord(partitionType, "foo", "A"), - 0, - 3 * dataFile1.recordCount(), - 3, - 3 * dataFile1.fileSizeInBytes(), - 0L, - 0, - eqDelete.recordCount(), - 1, - null, - snapshot3.timestampMillis(), - snapshot3.snapshotId(), - 0), - Tuple.tuple( - partitionRecord(partitionType, "foo", "B"), - 0, - 3 * dataFile2.recordCount(), - 3, - 3 * dataFile2.fileSizeInBytes(), - 0L, - 0, - 0L, - 0, - null, - snapshot1.timestampMillis(), - snapshot1.snapshotId(), - 0), - Tuple.tuple( - partitionRecord(partitionType, "bar", "A"), - 0, - 3 * dataFile3.recordCount(), - 3, - 3 * dataFile3.fileSizeInBytes(), - posDelete.recordCount() + dv.recordCount(), - 1, - 0L, - 0, - null, - snapshot4.timestampMillis(), - snapshot4.snapshotId(), - 1), // dv count - Tuple.tuple( - partitionRecord(partitionType, "bar", "B"), - 0, - 3 * dataFile4.recordCount(), - 3, - 3 * dataFile4.fileSizeInBytes(), - 0L, - 0, - 0L, - 0, - null, - snapshot1.timestampMillis(), - snapshot1.snapshotId(), - 0)); - } - @Test public void testCopyOnWriteDelete() throws Exception { Table testTable = @@ -591,39 +409,6 @@ public void testLatestStatsFileWithBranch() throws Exception { assertThat(PartitionStatsHandler.latestStatsFile(testTable, snapshotBranchBId)).isNull(); } - /** - * @deprecated will be removed in 1.12.0 - */ - @Test - @Deprecated - public void testReadingStatsWithInvalidSchema() throws Exception { - PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).identity("c1").build(); - Table testTable = - TestTables.create(tempDir("old_schema"), "old_schema", SCHEMA, spec, 2, fileFormatProperty); - Types.StructType partitionType = Partitioning.partitionType(testTable); - Schema newSchema = PartitionStatistics.schema(partitionType, 2); - Schema oldSchema = invalidOldSchema(partitionType); - - PartitionStatisticsFile invalidStatisticsFile = - PartitionStatsHandler.writePartitionStatsFile( - testTable, 42L, oldSchema, Collections.singletonList(randomStats(partitionType))); - - try (CloseableIterable recordIterator = - PartitionStatsHandler.readPartitionStatsFile( - newSchema, testTable.io().newInputFile(invalidStatisticsFile.path()))) { - - if (format() == FileFormat.PARQUET) { - assertThatThrownBy(() -> Lists.newArrayList(recordIterator)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Not a primitive type: struct"); - } else if (format() == FileFormat.AVRO) { - assertThatThrownBy(() -> Lists.newArrayList(recordIterator)) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("Not an instance of org.apache.iceberg.StructLike"); - } - } - } - @Test public void testFullComputeFallbackWithInvalidStats() throws Exception { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).identity("c1").build(); @@ -660,106 +445,4 @@ public void testFullComputeFallbackWithInvalidStats() throws Exception { // should include stats from both the appends. assertThat(partitionStats.get(0).dataFileCount()).isEqualTo(2); } - - /** - * @deprecated will be removed in 1.12.0 - */ - @Test - @Deprecated - public void testV2toV3SchemaEvolution() throws Exception { - Table testTable = - TestTables.create( - tempDir("schema_evolution"), "schema_evolution", SCHEMA, SPEC, 2, fileFormatProperty); - - // write stats file using v2 schema - DataFile dataFile = - FileGenerationUtil.generateDataFile(testTable, TestHelpers.Row.of("foo", "A")); - testTable.newAppend().appendFile(dataFile).commit(); - PartitionStatisticsFile statisticsFile = - PartitionStatsHandler.computeAndWriteStatsFile( - testTable, testTable.currentSnapshot().snapshotId()); - - Types.StructType partitionSchema = Partitioning.partitionType(testTable); - - // read with v2 schema - Schema v2Schema = PartitionStatistics.schema(partitionSchema, 2); - List partitionStatsV2; - try (CloseableIterable recordIterator = - PartitionStatsHandler.readPartitionStatsFile( - v2Schema, testTable.io().newInputFile(statisticsFile.path()))) { - partitionStatsV2 = Lists.newArrayList(recordIterator); - } - - // read with v3 schema - Schema v3Schema = PartitionStatistics.schema(partitionSchema, 3); - List partitionStatsV3; - try (CloseableIterable recordIterator = - PartitionStatsHandler.readPartitionStatsFile( - v3Schema, testTable.io().newInputFile(statisticsFile.path()))) { - partitionStatsV3 = Lists.newArrayList(recordIterator); - } - - assertThat(partitionStatsV2).hasSameSizeAs(partitionStatsV3); - Comparator comparator = Comparators.forType(partitionSchema); - for (int i = 0; i < partitionStatsV2.size(); i++) { - assertThat(isEqual(comparator, partitionStatsV2.get(i), partitionStatsV3.get(i))).isTrue(); - } - } - - private static void computeAndValidatePartitionStats( - Table testTable, Schema recordSchema, Tuple... expectedValues) throws IOException { - // compute and commit partition stats file - Snapshot currentSnapshot = testTable.currentSnapshot(); - PartitionStatisticsFile result = PartitionStatsHandler.computeAndWriteStatsFile(testTable); - testTable.updatePartitionStatistics().setPartitionStatistics(result).commit(); - assertThat(result.snapshotId()).isEqualTo(currentSnapshot.snapshotId()); - - // read the partition entries from the stats file - List partitionStats; - try (CloseableIterable recordIterator = - PartitionStatsHandler.readPartitionStatsFile( - recordSchema, testTable.io().newInputFile(result.path()))) { - partitionStats = Lists.newArrayList(recordIterator); - } - - assertThat(partitionStats) - .extracting( - PartitionStats::partition, - PartitionStats::specId, - PartitionStats::dataRecordCount, - PartitionStats::dataFileCount, - PartitionStats::totalDataFileSizeInBytes, - PartitionStats::positionDeleteRecordCount, - PartitionStats::positionDeleteFileCount, - PartitionStats::equalityDeleteRecordCount, - PartitionStats::equalityDeleteFileCount, - PartitionStats::totalRecords, - PartitionStats::lastUpdatedAt, - PartitionStats::lastUpdatedSnapshotId, - PartitionStats::dvCount) - .containsExactlyInAnyOrder(expectedValues); - } - - @SuppressWarnings("checkstyle:CyclomaticComplexity") - private static boolean isEqual( - Comparator partitionComparator, PartitionStats stats1, PartitionStats stats2) { - if (stats1 == stats2) { - return true; - } else if (stats1 == null || stats2 == null) { - return false; - } - - return partitionComparator.compare(stats1.partition(), stats2.partition()) == 0 - && stats1.specId() == stats2.specId() - && stats1.dataRecordCount() == stats2.dataRecordCount() - && stats1.dataFileCount() == stats2.dataFileCount() - && stats1.totalDataFileSizeInBytes() == stats2.totalDataFileSizeInBytes() - && stats1.positionDeleteRecordCount() == stats2.positionDeleteRecordCount() - && stats1.positionDeleteFileCount() == stats2.positionDeleteFileCount() - && stats1.equalityDeleteRecordCount() == stats2.equalityDeleteRecordCount() - && stats1.equalityDeleteFileCount() == stats2.equalityDeleteFileCount() - && Objects.equals(stats1.totalRecords(), stats2.totalRecords()) - && Objects.equals(stats1.lastUpdatedAt(), stats2.lastUpdatedAt()) - && Objects.equals(stats1.lastUpdatedSnapshotId(), stats2.lastUpdatedSnapshotId()); - } } diff --git a/core/src/test/java/org/apache/iceberg/TestBase.java b/core/src/test/java/org/apache/iceberg/TestBase.java index 27b8a49d0497..100de1b8db8b 100644 --- a/core/src/test/java/org/apache/iceberg/TestBase.java +++ b/core/src/test/java/org/apache/iceberg/TestBase.java @@ -777,7 +777,7 @@ protected PositionDelete positionDelete(CharSequence path, long pos) { protected PositionDelete positionDelete(CharSequence path, long pos, T row) { PositionDelete positionDelete = PositionDelete.create(); - return positionDelete.set(path, pos, row); + return positionDelete.set(path, pos); } protected void withUnavailableLocations(Iterable locations, Action action) { diff --git a/core/src/test/java/org/apache/iceberg/TestManifestReader.java b/core/src/test/java/org/apache/iceberg/TestManifestReader.java index 6690a1483e53..643508678e76 100644 --- a/core/src/test/java/org/apache/iceberg/TestManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestManifestReader.java @@ -269,15 +269,4 @@ public void testDataFileSplitOffsetsNullWhenInvalid() throws IOException { assertThat(file.splitOffsets()).isNull(); } } - - @SuppressWarnings("deprecation") - @TestTemplate - public void testDeprecatedReadWithoutSpecsById() throws IOException { - ManifestFile manifest = writeManifest(1000L, manifestEntry(Status.EXISTING, 1000L, FILE_A)); - try (ManifestReader reader = ManifestFiles.read(manifest, FILE_IO)) { - ManifestEntry entry = Iterables.getOnlyElement(reader.entries()); - assertThat(entry.status()).isEqualTo(Status.EXISTING); - assertThat(entry.file().location()).isEqualTo(FILE_A.location()); - } - } } diff --git a/core/src/test/java/org/apache/iceberg/TestPartitionStats.java b/core/src/test/java/org/apache/iceberg/TestPartitionStats.java deleted file mode 100644 index c215fbcb800c..000000000000 --- a/core/src/test/java/org/apache/iceberg/TestPartitionStats.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import org.apache.iceberg.types.Types; -import org.junit.jupiter.api.Test; - -public class TestPartitionStats { - - private static final PartitionData PARTITION = - new PartitionData( - Types.StructType.of(Types.NestedField.required(1, "foo", Types.IntegerType.get()))); - - @Test - public void testAppendWithAllValues() { - PartitionStats stats1 = - createStats(100L, 15, 1000L, 2L, 500, 1L, 200, 15L, 1625077800000L, 12345L); - PartitionStats stats2 = createStats(200L, 7, 500L, 1L, 100, 0L, 50, 7L, 1625077900000L, 12346L); - - stats1.appendStats(stats2); - - validateStats(stats1, 300L, 22, 1500L, 3L, 600, 1L, 250, 22L, 1625077900000L, 12346L); - } - - @Test - public void testAppendWithThisNullOptionalField() { - PartitionStats stats1 = createStats(100L, 15, 1000L, 2L, 500, 1L, 200, null, null, null); - PartitionStats stats2 = createStats(100L, 7, 500L, 1L, 100, 0L, 50, 7L, 1625077900000L, 12346L); - - stats1.appendStats(stats2); - - validateStats(stats1, 200L, 22, 1500L, 3L, 600, 1L, 250, 7L, 1625077900000L, 12346L); - } - - @Test - public void testAppendWithBothNullOptionalFields() { - PartitionStats stats1 = createStats(100L, 15, 1000L, 2L, 500, 1L, 200, null, null, null); - PartitionStats stats2 = createStats(100L, 7, 500L, 1L, 100, 0L, 50, null, null, null); - - stats1.appendStats(stats2); - - validateStats(stats1, 200L, 22, 1500L, 3L, 600, 1L, 250, null, null, null); - } - - @Test - public void testAppendWithOtherNullOptionalFields() { - PartitionStats stats1 = - createStats(100L, 15, 1000L, 2L, 500, 1L, 200, 15L, 1625077900000L, 12346L); - PartitionStats stats2 = createStats(100L, 7, 500L, 1L, 100, 0L, 50, null, null, null); - - stats1.appendStats(stats2); - - validateStats(stats1, 200L, 22, 1500L, 3L, 600, 1L, 250, 15L, 1625077900000L, 12346L); - } - - @Test - public void testAppendEmptyStats() { - PartitionStats stats1 = new PartitionStats(PARTITION, 1); - PartitionStats stats2 = new PartitionStats(PARTITION, 1); - - stats1.appendStats(stats2); - - validateStats(stats1, 0L, 0, 0L, 0L, 0, 0L, 0, null, null, null); - } - - @Test - public void testAppendWithDifferentSpec() { - PartitionStats stats1 = new PartitionStats(PARTITION, 1); - PartitionStats stats2 = new PartitionStats(PARTITION, 2); - - assertThatThrownBy(() -> stats1.appendStats(stats2)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Spec IDs must match"); - } - - private PartitionStats createStats( - long dataRecordCount, - int dataFileCount, - long totalDataFileSizeInBytes, - long positionDeleteRecordCount, - int positionDeleteFileCount, - long equalityDeleteRecordCount, - int equalityDeleteFileCount, - Long totalRecordCount, - Long lastUpdatedAt, - Long lastUpdatedSnapshotId) { - - PartitionStats stats = new PartitionStats(PARTITION, 1); - stats.set(2, dataRecordCount); - stats.set(3, dataFileCount); - stats.set(4, totalDataFileSizeInBytes); - stats.set(5, positionDeleteRecordCount); - stats.set(6, positionDeleteFileCount); - stats.set(7, equalityDeleteRecordCount); - stats.set(8, equalityDeleteFileCount); - stats.set(9, totalRecordCount); - stats.set(10, lastUpdatedAt); - stats.set(11, lastUpdatedSnapshotId); - - return stats; - } - - private void validateStats(PartitionStats stats, Object... expectedValues) { - // Spec id and partition data should be unchanged - assertThat(stats.get(0, PartitionData.class)).isEqualTo(PARTITION); - assertThat(stats.get(1, Integer.class)).isEqualTo(1); - - for (int i = 0; i < expectedValues.length; i++) { - if (expectedValues[i] == null) { - assertThat(stats.get(i + 2, Object.class)).isNull(); - } else { - assertThat(stats.get(i + 2, Object.class)).isEqualTo(expectedValues[i]); - } - } - } -} diff --git a/core/src/test/java/org/apache/iceberg/avro/TestAvroDeleteWriters.java b/core/src/test/java/org/apache/iceberg/avro/TestAvroDeleteWriters.java index 86bb74c5a397..091b3ed4330d 100644 --- a/core/src/test/java/org/apache/iceberg/avro/TestAvroDeleteWriters.java +++ b/core/src/test/java/org/apache/iceberg/avro/TestAvroDeleteWriters.java @@ -138,7 +138,7 @@ public void testPositionDeleteWriter() throws IOException { try (PositionDeleteWriter writer = deleteWriter) { for (int i = 0; i < records.size(); i += 1) { int pos = i * 3 + 2; - writer.write(positionDelete.set(deletePath, pos, records.get(i))); + writer.write(positionDelete.set(deletePath, pos)); expectedDeleteRecords.add( posDelete.copy( ImmutableMap.of( @@ -194,7 +194,7 @@ public void testPositionDeleteWriterWithEmptyRow() throws IOException { try (PositionDeleteWriter writer = deleteWriter) { for (int i = 0; i < records.size(); i += 1) { int pos = i * 3 + 2; - writer.write(positionDelete.set(deletePath, pos, null)); + writer.write(positionDelete.set(deletePath, pos)); expectedDeleteRecords.add( posDelete.copy(ImmutableMap.of("file_path", deletePath, "pos", (long) pos))); } diff --git a/core/src/test/java/org/apache/iceberg/data/avro/TestDataReader.java b/core/src/test/java/org/apache/iceberg/data/avro/TestDataReader.java index 966f01267d67..4ee9c15dd056 100644 --- a/core/src/test/java/org/apache/iceberg/data/avro/TestDataReader.java +++ b/core/src/test/java/org/apache/iceberg/data/avro/TestDataReader.java @@ -69,7 +69,7 @@ public void timestampDataReader() throws IOException { avroSchema.getField("timestamp_micros").addProp("field-id", 2); avroSchema.getField("timestamp_millis").addProp("field-id", 3); - DataReader reader = DataReader.create(icebergSchema, avroSchema); + PlannedDataReader reader = PlannedDataReader.create(icebergSchema); reader.setSchema(avroSchema); // post-epoch timestamps @@ -131,7 +131,7 @@ public void timestampTzDataReader() throws IOException { avroSchema.getField("timestamp_micros_tz").addProp("field-id", 2); avroSchema.getField("timestamp_millis_tz").addProp("field-id", 3); - DataReader reader = DataReader.create(icebergSchema, avroSchema); + PlannedDataReader reader = PlannedDataReader.create(icebergSchema); reader.setSchema(avroSchema); // post-epoch timestamps @@ -186,7 +186,8 @@ public void timestampTzDataReader() throws IOException { .isEqualTo(preEpochTimestampMillis.withOffsetSameInstant(ZoneOffset.UTC)); } - private Record readRecord(DataReader reader, Schema avroSchema, GenericRecord avroRecord) + private Record readRecord( + PlannedDataReader reader, Schema avroSchema, GenericRecord avroRecord) throws IOException { try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); diff --git a/core/src/test/java/org/apache/iceberg/rest/TestRESTUtil.java b/core/src/test/java/org/apache/iceberg/rest/TestRESTUtil.java index 1ed732ebc91a..e618932a6763 100644 --- a/core/src/test/java/org/apache/iceberg/rest/TestRESTUtil.java +++ b/core/src/test/java/org/apache/iceberg/rest/TestRESTUtil.java @@ -109,7 +109,8 @@ public void testRoundTripUrlEncodeDecodeNamespace(String namespaceSeparator) { public void encodeAsOldClientAndDecodeAsNewServer() { Namespace namespace = Namespace.of("first", "second", "third"); // old client would call encodeNamespace without specifying a separator - String encodedNamespace = RESTUtil.encodeNamespace(namespace); + String encodedNamespace = + RESTUtil.encodeNamespace(namespace, RESTUtil.NAMESPACE_SEPARATOR_URLENCODED_UTF_8); assertThat(encodedNamespace).contains(RESTUtil.NAMESPACE_SEPARATOR_URLENCODED_UTF_8); // old client would also call namespaceToQueryParam without specifying a separator @@ -130,11 +131,13 @@ public void encodeAsOldClientAndDecodeAsNewServer() { @Test public void testNamespaceUrlEncodeDecodeDoesNotAllowNull() { assertThatExceptionOfType(IllegalArgumentException.class) - .isThrownBy(() -> RESTUtil.encodeNamespace(null)) + .isThrownBy( + () -> RESTUtil.encodeNamespace(null, RESTUtil.NAMESPACE_SEPARATOR_URLENCODED_UTF_8)) .withMessage("Invalid namespace: null"); assertThatExceptionOfType(IllegalArgumentException.class) - .isThrownBy(() -> RESTUtil.decodeNamespace(null)) + .isThrownBy( + () -> RESTUtil.decodeNamespace(null, RESTUtil.NAMESPACE_SEPARATOR_URLENCODED_UTF_8)) .withMessage("Invalid namespace: null"); } diff --git a/core/src/test/java/org/apache/iceberg/rest/TestResourcePaths.java b/core/src/test/java/org/apache/iceberg/rest/TestResourcePaths.java index a742b89a7627..e5b2980c89d9 100644 --- a/core/src/test/java/org/apache/iceberg/rest/TestResourcePaths.java +++ b/core/src/test/java/org/apache/iceberg/rest/TestResourcePaths.java @@ -120,13 +120,17 @@ public void nestedNamespaceWithLegacySeparator() { ResourcePaths pathsWithLegacySeparator = ResourcePaths.forCatalogProperties(ImmutableMap.of()); // Encode namespace using legacy separator. No need to provide the separator to encodeNamespace - String legacyEncodedNamespace = RESTUtil.encodeNamespace(namespace); + String legacyEncodedNamespace = + RESTUtil.encodeNamespace(namespace, RESTUtil.NAMESPACE_SEPARATOR_URLENCODED_UTF_8); assertThat(pathsWithLegacySeparator.namespace(namespace)) .contains(legacyEncodedNamespace) .contains(legacySeparator); // Decode the namespace containing legacy separator without providing the separator - assertThat(RESTUtil.decodeNamespace(legacyEncodedNamespace)).isEqualTo(namespace); + assertThat( + RESTUtil.decodeNamespace( + legacyEncodedNamespace, RESTUtil.NAMESPACE_SEPARATOR_URLENCODED_UTF_8)) + .isEqualTo(namespace); // Decode the namespace containing legacy separator with providing the new separator assertThat(RESTUtil.decodeNamespace(legacyEncodedNamespace, newSeparator)).isEqualTo(namespace); diff --git a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java index 841083f88baf..690bd5a83a22 100644 --- a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java +++ b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java @@ -145,17 +145,6 @@ public void roundTripSerdeWithInvalidPlanStatusSubmittedWithTasksPresent() { @Test public void roundTripSerdeWithInvalidPlanStatusSubmittedWithDeleteFilesNoFileScanTasksPresent() { PlanStatus planStatus = PlanStatus.fromName("submitted"); - assertThatThrownBy( - () -> { - FetchPlanningResultResponse.builder() - .withPlanStatus(planStatus) - .withDeleteFiles(List.of(FILE_A_DELETES)) - .build(); - }) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "Invalid response: deleteFiles should only be returned with fileScanTasks that reference them"); - String invalidJson = "{\"status\":\"submitted\"," + "\"delete-files\":[{\"spec-id\":0,\"content\":\"position-deletes\"," diff --git a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java index 7b44b3533a5b..b3845bf42398 100644 --- a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java +++ b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java @@ -83,16 +83,6 @@ public void roundTripSerdeWithPlanTasks() { @Test public void roundTripSerdeWithDeleteFilesNoFileScanTasksPresent() { - assertThatThrownBy( - () -> - FetchScanTasksResponse.builder() - .withPlanTasks(List.of("task1", "task2")) - .withDeleteFiles(List.of(FILE_A_DELETES)) - .build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "Invalid response: deleteFiles should only be returned with fileScanTasks that reference them"); - String invalidJson = "{\"plan-tasks\":[\"task1\",\"task2\"]," + "\"delete-files\":[{\"spec-id\":0,\"content\":\"position-deletes\"," diff --git a/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java b/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java index 6354e7bf246f..c248392e3934 100644 --- a/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java +++ b/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java @@ -196,17 +196,6 @@ public void roundTripSerdeWithInvalidPlanIdWithIncorrectStatus() { @Test public void roundTripSerdeWithInvalidPlanStatusSubmittedWithDeleteFilesNoFileScanTasksPresent() { - assertThatThrownBy( - () -> - PlanTableScanResponse.builder() - .withPlanStatus(PlanStatus.SUBMITTED) - .withPlanId("somePlanId") - .withDeleteFiles(List.of(FILE_A_DELETES)) - .build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "Invalid response: deleteFiles should only be returned with fileScanTasks that reference them"); - String invalidJson = "{\"status\":\"submitted\"," + "\"plan-id\":\"somePlanId\"," @@ -264,7 +253,6 @@ public void roundTripSerdeWithValidStatusAndFileScanTasks() { .withPlanStatus(fromResponse.planStatus()) .withPlanId(fromResponse.planId()) .withPlanTasks(fromResponse.planTasks()) - .withDeleteFiles(fromResponse.deleteFiles()) .withFileScanTasks(fromResponse.fileScanTasks()) .withSpecsById(PARTITION_SPECS_BY_ID) .build(); diff --git a/data/src/main/java/org/apache/iceberg/data/BaseFileWriterFactory.java b/data/src/main/java/org/apache/iceberg/data/BaseFileWriterFactory.java index 444c0d0226bd..f6fbf1ca6b24 100644 --- a/data/src/main/java/org/apache/iceberg/data/BaseFileWriterFactory.java +++ b/data/src/main/java/org/apache/iceberg/data/BaseFileWriterFactory.java @@ -293,8 +293,7 @@ public PositionDeleteWriter newPositionDeleteWriter( EncryptedOutputFile file, PartitionSpec spec, StructLike partition) { EncryptionKeyMetadata keyMetadata = file.keyMetadata(); Map properties = table == null ? ImmutableMap.of() : table.properties(); - MetricsConfig metricsConfig = - table == null ? MetricsConfig.forPositionDelete() : MetricsConfig.forPositionDelete(table); + MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(); try { switch (deleteFileFormat) { diff --git a/data/src/main/java/org/apache/iceberg/data/GenericAppenderFactory.java b/data/src/main/java/org/apache/iceberg/data/GenericAppenderFactory.java index dd78b48858a2..704d0fd82719 100644 --- a/data/src/main/java/org/apache/iceberg/data/GenericAppenderFactory.java +++ b/data/src/main/java/org/apache/iceberg/data/GenericAppenderFactory.java @@ -288,9 +288,7 @@ public EqualityDeleteWriter newEqDeleteWriter( public PositionDeleteWriter newPosDeleteWriter( EncryptedOutputFile file, FileFormat format, StructLike partition) { MetricsConfig metricsConfig = - table != null - ? MetricsConfig.forPositionDelete(table) - : MetricsConfig.fromProperties(config); + table != null ? MetricsConfig.forPositionDelete() : MetricsConfig.fromProperties(config); try { switch (format) { diff --git a/data/src/main/java/org/apache/iceberg/data/GenericFileWriterFactory.java b/data/src/main/java/org/apache/iceberg/data/GenericFileWriterFactory.java index 1e75b9eda961..8219587d36b4 100644 --- a/data/src/main/java/org/apache/iceberg/data/GenericFileWriterFactory.java +++ b/data/src/main/java/org/apache/iceberg/data/GenericFileWriterFactory.java @@ -242,9 +242,7 @@ public PositionDeleteWriter newPositionDeleteWriter( "Deprecated feature used. Position delete row schema is used to create the position delete writer."); Map properties = table == null ? ImmutableMap.of() : table.properties(); MetricsConfig metricsConfig = - table == null - ? MetricsConfig.forPositionDelete() - : MetricsConfig.forPositionDelete(table); + table == null ? MetricsConfig.forPositionDelete() : MetricsConfig.forPositionDelete(); try { return switch (format) { diff --git a/data/src/main/java/org/apache/iceberg/data/RegistryBasedFileWriterFactory.java b/data/src/main/java/org/apache/iceberg/data/RegistryBasedFileWriterFactory.java index 868b41f5840b..da2be7702909 100644 --- a/data/src/main/java/org/apache/iceberg/data/RegistryBasedFileWriterFactory.java +++ b/data/src/main/java/org/apache/iceberg/data/RegistryBasedFileWriterFactory.java @@ -160,8 +160,7 @@ public PositionDeleteWriter newPositionDeleteWriter( EncryptedOutputFile file, PartitionSpec spec, StructLike partition) { EncryptionKeyMetadata keyMetadata = file.keyMetadata(); Map properties = table != null ? table.properties() : ImmutableMap.of(); - MetricsConfig metricsConfig = - table != null ? MetricsConfig.forPositionDelete(table) : MetricsConfig.forPositionDelete(); + MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(); try { FileWriterBuilder, ?> builder = diff --git a/data/src/test/java/org/apache/iceberg/data/FileHelpers.java b/data/src/test/java/org/apache/iceberg/data/FileHelpers.java index 19a5bfa441ea..f9ac0822cbfe 100644 --- a/data/src/test/java/org/apache/iceberg/data/FileHelpers.java +++ b/data/src/test/java/org/apache/iceberg/data/FileHelpers.java @@ -94,7 +94,7 @@ public static Pair writeDeleteFile( PositionDelete posDelete = PositionDelete.create(); try (Closeable toClose = writer) { for (Pair delete : deletes) { - writer.write(posDelete.set(delete.first(), delete.second(), null)); + writer.write(posDelete.set(delete.first(), delete.second())); } } diff --git a/data/src/test/java/org/apache/iceberg/io/TestAppenderFactory.java b/data/src/test/java/org/apache/iceberg/io/TestAppenderFactory.java index 4d3e6a8ac9c6..838d2b8060a0 100644 --- a/data/src/test/java/org/apache/iceberg/io/TestAppenderFactory.java +++ b/data/src/test/java/org/apache/iceberg/io/TestAppenderFactory.java @@ -229,7 +229,7 @@ public void testPosDeleteWriter() throws IOException { PositionDelete posDelete = PositionDelete.create(); try (PositionDeleteWriter closeableWriter = eqDeleteWriter) { for (Pair delete : deletes) { - closeableWriter.write(posDelete.set(delete.first(), delete.second(), null)); + closeableWriter.write(posDelete.set(delete.first(), delete.second())); } } @@ -278,7 +278,7 @@ public void testPosDeleteWriterWithRowSchema() throws IOException { PositionDelete posDelete = PositionDelete.create(); try (PositionDeleteWriter closeableWriter = eqDeleteWriter) { for (PositionDelete delete : deletes) { - closeableWriter.write(posDelete.set(delete.path(), delete.pos(), delete.row())); + closeableWriter.write(posDelete.set(delete.path(), delete.pos())); } } diff --git a/data/src/test/java/org/apache/iceberg/io/TestFileWriterFactory.java b/data/src/test/java/org/apache/iceberg/io/TestFileWriterFactory.java index 8354149bf495..41f04ffd84f0 100644 --- a/data/src/test/java/org/apache/iceberg/io/TestFileWriterFactory.java +++ b/data/src/test/java/org/apache/iceberg/io/TestFileWriterFactory.java @@ -466,7 +466,7 @@ private Pair writePositionDeletes( PositionDelete posDelete = PositionDelete.create(); try (PositionDeleteWriter closableWriter = writer) { for (PositionDelete delete : deletes) { - closableWriter.write(posDelete.set(delete.path(), delete.pos(), delete.row())); + closableWriter.write(posDelete.set(delete.path(), delete.pos())); } } diff --git a/data/src/test/java/org/apache/iceberg/io/TestWriterMetrics.java b/data/src/test/java/org/apache/iceberg/io/TestWriterMetrics.java index 4722ae98950e..4652db0c7b6e 100644 --- a/data/src/test/java/org/apache/iceberg/io/TestWriterMetrics.java +++ b/data/src/test/java/org/apache/iceberg/io/TestWriterMetrics.java @@ -154,7 +154,7 @@ public void testPositionDeleteMetrics() throws IOException { try { T deletedRow = toRow(3, "3", true, 3L); PositionDelete positionDelete = PositionDelete.create(); - positionDelete.set("File A", 1, deletedRow); + positionDelete.set("File A", 1); deleteWriter.write(positionDelete); } finally { deleteWriter.close(); @@ -193,10 +193,10 @@ public void testPositionDeleteMetricsCoveringMultipleDataFiles() throws IOExcept try { PositionDelete positionDelete = PositionDelete.create(); - positionDelete.set("File A", 1, toRow(3, "3", true, 3L)); + positionDelete.set("File A", 1); deleteWriter.write(positionDelete); - positionDelete.set("File B", 1, toRow(3, "3", true, 3L)); + positionDelete.set("File B", 1); deleteWriter.write(positionDelete); } finally { diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java index 07a068391ce6..85dcc3133bd6 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java @@ -238,7 +238,7 @@ public EqualityDeleteWriter newEqDeleteWriter( @Override public PositionDeleteWriter newPosDeleteWriter( EncryptedOutputFile outputFile, FileFormat format, StructLike partition) { - MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(table); + MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(); try { switch (format) { case AVRO: diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/SimpleDataUtil.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/SimpleDataUtil.java index fc5bea17b4a2..ee520109dd74 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/SimpleDataUtil.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/SimpleDataUtil.java @@ -259,7 +259,7 @@ public static DeleteFile writePosDeleteFile( PositionDelete posDelete = PositionDelete.create(); try (PositionDeleteWriter writer = posWriter) { for (Pair p : positions) { - writer.write(posDelete.set(p.first(), p.second(), null)); + writer.write(posDelete.set(p.first(), p.second())); } } return posWriter.toDeleteFile(); @@ -280,7 +280,7 @@ public static DeleteFile writePosDeleteFile( PositionDelete posDelete = PositionDelete.create(); try (PositionDeleteWriter writer = posWriter) { for (Pair p : positions) { - writer.write(posDelete.set(p.first(), p.second(), null)); + writer.write(posDelete.set(p.first(), p.second())); } } diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java index 638da61949c9..8f43cc67fe28 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java @@ -457,7 +457,7 @@ private DeleteFile writePosDelete( GenericRecord nested = GenericRecord.create(table.schema()); nested.set(0, id); nested.set(1, oldData); - posDelete.set(path, pos, nested); + posDelete.set(path, pos); return FileHelpers.writePosDeleteFile( table, Files.localOutput(file), null, Lists.newArrayList(posDelete), formatVersion); } diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java index 07a068391ce6..85dcc3133bd6 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java @@ -238,7 +238,7 @@ public EqualityDeleteWriter newEqDeleteWriter( @Override public PositionDeleteWriter newPosDeleteWriter( EncryptedOutputFile outputFile, FileFormat format, StructLike partition) { - MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(table); + MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(); try { switch (format) { case AVRO: diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/SimpleDataUtil.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/SimpleDataUtil.java index 4bbf30512dbc..30ce3a112fb8 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/SimpleDataUtil.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/SimpleDataUtil.java @@ -259,7 +259,7 @@ public static DeleteFile writePosDeleteFile( PositionDelete posDelete = PositionDelete.create(); try (PositionDeleteWriter writer = posWriter) { for (Pair p : positions) { - writer.write(posDelete.set(p.first(), p.second(), null)); + writer.write(posDelete.set(p.first(), p.second())); } } return posWriter.toDeleteFile(); diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java index 4f394be76f4f..dc4efd22bf92 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java @@ -454,7 +454,7 @@ private DeleteFile writePosDelete( GenericRecord nested = GenericRecord.create(table.schema()); nested.set(0, id); nested.set(1, oldData); - posDelete.set(path, pos, nested); + posDelete.set(path, pos); return FileHelpers.writePosDeleteFile( table, Files.localOutput(file), null, Lists.newArrayList(posDelete), formatVersion); } diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java index 07a068391ce6..85dcc3133bd6 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java @@ -238,7 +238,7 @@ public EqualityDeleteWriter newEqDeleteWriter( @Override public PositionDeleteWriter newPosDeleteWriter( EncryptedOutputFile outputFile, FileFormat format, StructLike partition) { - MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(table); + MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(); try { switch (format) { case AVRO: diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/SimpleDataUtil.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/SimpleDataUtil.java index 542376b06c27..30ce3a112fb8 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/SimpleDataUtil.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/SimpleDataUtil.java @@ -259,7 +259,7 @@ public static DeleteFile writePosDeleteFile( PositionDelete posDelete = PositionDelete.create(); try (PositionDeleteWriter writer = posWriter) { for (Pair p : positions) { - writer.write(posDelete.set(p.first(), p.second(), null)); + writer.write(posDelete.set(p.first(), p.second())); } } return posWriter.toDeleteFile(); @@ -280,7 +280,7 @@ public static DeleteFile writePosDeleteFile( PositionDelete posDelete = PositionDelete.create(); try (PositionDeleteWriter writer = posWriter) { for (Pair p : positions) { - writer.write(posDelete.set(p.first(), p.second(), null)); + writer.write(posDelete.set(p.first(), p.second())); } } diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java index 4f394be76f4f..dc4efd22bf92 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java @@ -454,7 +454,7 @@ private DeleteFile writePosDelete( GenericRecord nested = GenericRecord.create(table.schema()); nested.set(0, id); nested.set(1, oldData); - posDelete.set(path, pos, nested); + posDelete.set(path, pos); return FileHelpers.writePosDeleteFile( table, Files.localOutput(file), null, Lists.newArrayList(posDelete), formatVersion); } diff --git a/orc/src/main/java/org/apache/iceberg/data/orc/GenericOrcWriters.java b/orc/src/main/java/org/apache/iceberg/data/orc/GenericOrcWriters.java index 75192b8a82e6..e2c99a6dea90 100644 --- a/orc/src/main/java/org/apache/iceberg/data/orc/GenericOrcWriters.java +++ b/orc/src/main/java/org/apache/iceberg/data/orc/GenericOrcWriters.java @@ -647,7 +647,7 @@ protected Object get(PositionDelete delete, int index) { case 1: return delete.pos(); case 2: - return delete.row(); + return null; } throw new IllegalArgumentException("Cannot get value for invalid index: " + index); } @@ -656,8 +656,7 @@ protected Object get(PositionDelete delete, int index) { public void write(PositionDelete row, VectorizedRowBatch output) throws IOException { Preconditions.checkArgument(row != null, "value must not be null"); Preconditions.checkArgument( - writers().size() == 2 || row.row() != null, - "The row in PositionDelete must not be null because it was set row schema in position delete."); + writers().size() == 2, "Position delete row data is not supported"); writeRow(row, output); } } diff --git a/orc/src/test/java/org/apache/iceberg/orc/TestOrcDeleteWriters.java b/orc/src/test/java/org/apache/iceberg/orc/TestOrcDeleteWriters.java index 112885bb00a4..468c61009ce9 100644 --- a/orc/src/test/java/org/apache/iceberg/orc/TestOrcDeleteWriters.java +++ b/orc/src/test/java/org/apache/iceberg/orc/TestOrcDeleteWriters.java @@ -132,7 +132,7 @@ public void testPositionDeleteWriter() throws IOException { try (PositionDeleteWriter writer = deleteWriter) { for (int i = 0; i < records.size(); i += 1) { int pos = i * 3 + 2; - positionDelete.set(deletePath, pos, records.get(i)); + positionDelete.set(deletePath, pos); writer.write(positionDelete); expectedDeleteRecords.add( posDelete.copy( @@ -187,7 +187,7 @@ public void testPositionDeleteWriterWithEmptyRow() throws IOException { try (PositionDeleteWriter writer = deleteWriter) { for (int i = 0; i < records.size(); i += 1) { int pos = i * 3 + 2; - positionDelete.set(deletePath, pos, null); + positionDelete.set(deletePath, pos); writer.write(positionDelete); expectedDeleteRecords.add( posDelete.copy(ImmutableMap.of("file_path", deletePath, "pos", (long) pos))); diff --git a/orc/src/test/java/org/apache/iceberg/orc/TestOrcPartitionStatsHandler.java b/orc/src/test/java/org/apache/iceberg/orc/TestOrcPartitionStatsHandler.java index 117293629f58..04d1d212f77a 100644 --- a/orc/src/test/java/org/apache/iceberg/orc/TestOrcPartitionStatsHandler.java +++ b/orc/src/test/java/org/apache/iceberg/orc/TestOrcPartitionStatsHandler.java @@ -43,13 +43,6 @@ public void testOptionalFieldsWriting() throws Exception { .hasMessage("Cannot write using unregistered internal data format: ORC"); } - @Override - public void testPartitionStats() throws Exception { - assertThatThrownBy(super::testPartitionStats) - .isInstanceOf(UnsupportedOperationException.class) - .hasMessage("Cannot write using unregistered internal data format: ORC"); - } - @Override public void testLatestStatsFile() throws Exception { assertThatThrownBy(super::testLatestStatsFile) @@ -71,24 +64,10 @@ public void testCopyOnWriteDelete() throws Exception { .hasMessage("Cannot write using unregistered internal data format: ORC"); } - @Override - public void testReadingStatsWithInvalidSchema() { - assertThatThrownBy(super::testReadingStatsWithInvalidSchema) - .isInstanceOf(UnsupportedOperationException.class) - .hasMessage("Cannot write using unregistered internal data format: ORC"); - } - @Override public void testFullComputeFallbackWithInvalidStats() { assertThatThrownBy(super::testFullComputeFallbackWithInvalidStats) .isInstanceOf(UnsupportedOperationException.class) .hasMessage("Cannot write using unregistered internal data format: ORC"); } - - @Override - public void testV2toV3SchemaEvolution() { - assertThatThrownBy(super::testV2toV3SchemaEvolution) - .isInstanceOf(UnsupportedOperationException.class) - .hasMessage("Cannot write using unregistered internal data format: ORC"); - } } diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java b/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java index f02974d6e79c..9e492dae02d0 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java @@ -75,7 +75,6 @@ import org.apache.iceberg.SchemaParser; import org.apache.iceberg.SortOrder; import org.apache.iceberg.StructLike; -import org.apache.iceberg.SystemConfigs; import org.apache.iceberg.Table; import org.apache.iceberg.avro.AvroSchemaUtil; import org.apache.iceberg.data.parquet.GenericParquetWriter; @@ -1486,14 +1485,7 @@ public CloseableIterable build() { optionsBuilder.withUseHadoopVectoredIo(true); ParquetReadOptions options = optionsBuilder.build(); - NameMapping mapping; - if (nameMapping != null) { - mapping = nameMapping; - } else if (SystemConfigs.NETFLIX_UNSAFE_PARQUET_ID_FALLBACK_ENABLED.value()) { - mapping = null; - } else { - mapping = NameMapping.empty(); - } + NameMapping mapping = nameMapping != null ? nameMapping : NameMapping.empty(); Function> batchedFunc = batchedReaderFuncWithSchema != null diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueWriters.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueWriters.java index 5554c86b4671..b025b23cec98 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueWriters.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetValueWriters.java @@ -703,7 +703,7 @@ protected Object get(PositionDelete delete, int index) { case 1: return delete.pos(); case 2: - return delete.row(); + return null; } throw new IllegalArgumentException("Cannot get value for invalid index: " + index); } diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDeleteWriters.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDeleteWriters.java index 7e91f5d9eb63..4dd694ae2583 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDeleteWriters.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDeleteWriters.java @@ -137,7 +137,7 @@ public void testPositionDeleteWriter() throws IOException { try (PositionDeleteWriter writer = deleteWriter) { for (int i = 0; i < records.size(); i += 1) { int pos = i * 3 + 2; - writer.write(positionDelete.set(deletePath, pos, records.get(i))); + writer.write(positionDelete.set(deletePath, pos)); expectedDeleteRecords.add( posDelete.copy( ImmutableMap.of( @@ -196,7 +196,7 @@ public void testPositionDeleteWriterWithEmptyRow() throws IOException { try (PositionDeleteWriter writer = deleteWriter) { for (int i = 0; i < records.size(); i += 1) { int pos = i * 3 + 2; - writer.write(positionDelete.set(deletePath, pos, null)); + writer.write(positionDelete.set(deletePath, pos)); expectedDeleteRecords.add( posDelete.copy(ImmutableMap.of("file_path", deletePath, "pos", (long) pos))); } diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewritePositionDeleteFiles.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewritePositionDeleteFiles.java index ad49da87bedd..622f1db6ace8 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewritePositionDeleteFiles.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewritePositionDeleteFiles.java @@ -339,7 +339,7 @@ private DeleteFile writeDeleteFile( PositionDelete posDelete = PositionDelete.create(); try (Closeable toClose = writer) { for (Pair delete : deletes) { - writer.write(posDelete.set(delete.first(), delete.second(), null)); + writer.write(posDelete.set(delete.first(), delete.second())); } } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java index 3f242ce228ca..13a575f34379 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java @@ -209,7 +209,7 @@ private DeleteWriteResult writePositionDeletes( for (InternalRow row : rows) { String path = row.getString(0); long pos = row.getLong(1); - positionDelete.set(path, pos, null /* no row */); + positionDelete.set(path, pos); closableWriter.write(positionDelete, table.spec(), null); } } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java index db5789724056..37931017244d 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java @@ -166,7 +166,7 @@ private DeleteWriteResult write( for (InternalRow row : rows) { String path = row.getString(0); long pos = row.getLong(1); - positionDelete.set(path, pos, null /* no row */); + positionDelete.set(path, pos); closableWriter.write(positionDelete, table.spec(), null); } } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceDeleteBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceDeleteBenchmark.java index e42707bf102b..708aaec4410b 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceDeleteBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceDeleteBenchmark.java @@ -254,10 +254,10 @@ protected void writePosDeletes(CharSequence path, List deletedPos, int num PositionDelete positionDelete = PositionDelete.create(); try (ClusteredPositionDeleteWriter closeableWriter = writer) { for (Long pos : deletedPos) { - positionDelete.set(path, pos, null); + positionDelete.set(path, pos); closeableWriter.write(positionDelete, unpartitionedSpec, null); for (int i = 0; i < numNoise; i++) { - positionDelete.set(noisePath(path), pos, null); + positionDelete.set(noisePath(path), pos); closeableWriter.write(positionDelete, unpartitionedSpec, null); } } diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java index 06efb6ba20e4..10785bdc85be 100644 --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java @@ -300,7 +300,7 @@ private void writeUnpartitionedClusteredPositionDeleteWriter( for (InternalRow row : positionDeleteRows) { String path = row.getString(0); long pos = row.getLong(1); - positionDelete.set(path, pos, null); + positionDelete.set(path, pos); closeableWriter.write(positionDelete, unpartitionedSpec, null); } } @@ -339,7 +339,7 @@ private void writeUnpartitionedFanoutPositionDeleteWriterPartition( for (InternalRow row : positionDeleteRows) { String path = row.getString(0); long pos = row.getLong(1); - positionDelete.set(path, pos, null); + positionDelete.set(path, pos); closeableWriter.write(positionDelete, unpartitionedSpec, null); } } @@ -379,7 +379,7 @@ private void writeUnpartitionedFanoutPositionDeleteWriterShuffled( for (InternalRow row : shuffledPositionDeleteRows) { String path = row.getString(0); long pos = row.getLong(1); - positionDelete.set(path, pos, null); + positionDelete.set(path, pos); closeableWriter.write(positionDelete, unpartitionedSpec, null); } } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java index aedb25e4a4a6..b74732ac0a84 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java @@ -714,13 +714,9 @@ public CloseableIterable reader( @Override public PositionDeleteWriter writer( - OutputFile outputFile, - FileFormat format, - PartitionSpec spec, - StructLike partition, - Schema rowSchema) + OutputFile outputFile, FileFormat format, PartitionSpec spec, StructLike partition) throws IOException { - return positionDeletesWriter(outputFile, format, spec, partition, rowSchema); + return positionDeletesWriter(outputFile, format, spec, partition); } } @@ -757,44 +753,13 @@ private static CloseableIterable positionDeletesReader( } private static PositionDeleteWriter positionDeletesWriter( - OutputFile outputFile, - FileFormat format, - PartitionSpec spec, - StructLike partition, - Schema rowSchema) + OutputFile outputFile, FileFormat format, PartitionSpec spec, StructLike partition) throws IOException { - if (rowSchema == null) { - return FormatModelRegistry.positionDeleteWriteBuilder( - format, EncryptedFiles.plainAsEncryptedOutput(outputFile)) - .partition(partition) - .spec(spec) - .build(); - } else { - return switch (format) { - case AVRO -> - Avro.writeDeletes(outputFile) - .createWriterFunc(DataWriter::create) - .withPartition(partition) - .rowSchema(rowSchema) - .withSpec(spec) - .buildPositionWriter(); - case PARQUET -> - Parquet.writeDeletes(outputFile) - .createWriterFunc(GenericParquetWriter::create) - .withPartition(partition) - .rowSchema(rowSchema) - .withSpec(spec) - .buildPositionWriter(); - case ORC -> - ORC.writeDeletes(outputFile) - .createWriterFunc(GenericOrcWriter::buildWriter) - .withPartition(partition) - .rowSchema(rowSchema) - .withSpec(spec) - .buildPositionWriter(); - default -> throw new UnsupportedOperationException("Unsupported file format: " + format); - }; - } + return FormatModelRegistry.positionDeleteWriteBuilder( + format, EncryptedFiles.plainAsEncryptedOutput(outputFile)) + .partition(partition) + .spec(spec) + .build(); } private Set snapshotSet(TableMetadata metadata) { diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java index 39110f0b0597..95c9e8b33e38 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java @@ -169,7 +169,7 @@ public PositionDeleteWriter newPositionDeleteWriter( MetricsConfig metricsConfig = table == null ? MetricsConfig.forPositionDelete() - : MetricsConfig.forPositionDelete(table); + : MetricsConfig.forPositionDelete(); try { return switch (deleteFormat) { diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeletesRewrite.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeletesRewrite.java index 0ec7084bfd1b..ced64861e915 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeletesRewrite.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeletesRewrite.java @@ -368,10 +368,10 @@ public void write(InternalRow record) { long position = record.getLong(positionOrdinal); InternalRow row = record.getStruct(rowOrdinal, rowSize); if (row != null) { - positionDelete.set(file, position, row); + positionDelete.set(file, position); lazyWriterWithRow().write(positionDelete, spec, partition); } else { - positionDelete.set(file, position, null); + positionDelete.set(file, position); lazyWriterWithoutRow().write(positionDelete, spec, partition); } } @@ -477,7 +477,7 @@ private static class DVWriter implements DataWriter { public void write(InternalRow record) { String file = record.getString(fileOrdinal); long position = record.getLong(positionOrdinal); - positionDelete.set(file, position, null); + positionDelete.set(file, position); dvWriter.write(positionDelete, spec, partition); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java index d74d8a29f994..60db61e280bb 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java @@ -2453,7 +2453,7 @@ private List writePosDeletes( .newPositionDeleteWriter(encrypt(outputFile), table.spec(), partition); PositionDelete posDelete = PositionDelete.create(); - posDeleteWriter.write(posDelete.set(path, rowPosition, null)); + posDeleteWriter.write(posDelete.set(path, rowPosition)); try { posDeleteWriter.close(); } catch (IOException e) { @@ -2497,7 +2497,7 @@ private List writePosDeletes( for (int position = file * positionsPerDeleteFile; position < (file + 1) * positionsPerDeleteFile; position++) { - posDeleteWriter.write(posDelete.set(path, position, null)); + posDeleteWriter.write(posDelete.set(path, position)); } try { diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java index dae721b1d73d..53bee1e6db51 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java @@ -1647,7 +1647,7 @@ private PositionDelete positionDelete( for (int i = 0; i < values.length; i++) { nested.set(i, values[i]); } - posDelete.set(path, position, nested); + posDelete.set(path, position); return posDelete; } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java index 88113be2ddd9..f390acf1ff0d 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java @@ -2437,7 +2437,7 @@ private DeleteFile writePosDeleteFile(Table table, long pos) { StructLike dataFilePartition = dataFile.partition(); PositionDelete delete = PositionDelete.create(); - delete.set(dataFile.location(), pos, null); + delete.set(dataFile.location(), pos); return writePositionDeletes(table, dataFileSpec, dataFilePartition, ImmutableList.of(delete)); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java index 3cfd9c848fcb..e811a26e853f 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java @@ -1573,7 +1573,7 @@ protected void dropTable(String name) { private PositionDelete positionDelete(CharSequence path, Long position) { PositionDelete posDelete = PositionDelete.create(); - posDelete.set(path, position, null); + posDelete.set(path, position); return posDelete; } @@ -1584,7 +1584,7 @@ private PositionDelete positionDelete( for (int i = 0; i < values.length; i++) { nested.set(i, values[i]); } - posDelete.set(path, position, nested); + posDelete.set(path, position); return posDelete; } @@ -1623,7 +1623,7 @@ private StructLikeSet expected( GenericRecord record = GenericRecord.create(finalSchema); record.setField("file_path", p.path()); record.setField("pos", p.pos()); - record.setField("row", formatVersion >= 3 ? null : p.row()); + record.setField("row", null); if (partitionStruct != null) { record.setField("partition", partitionStruct); } diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewritePositionDeleteFiles.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewritePositionDeleteFiles.java index ad49da87bedd..622f1db6ace8 100644 --- a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewritePositionDeleteFiles.java +++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewritePositionDeleteFiles.java @@ -339,7 +339,7 @@ private DeleteFile writeDeleteFile( PositionDelete posDelete = PositionDelete.create(); try (Closeable toClose = writer) { for (Pair delete : deletes) { - writer.write(posDelete.set(delete.first(), delete.second(), null)); + writer.write(posDelete.set(delete.first(), delete.second())); } } diff --git a/spark/v4.0/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v4.0/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java index 3f242ce228ca..13a575f34379 100644 --- a/spark/v4.0/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java +++ b/spark/v4.0/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java @@ -209,7 +209,7 @@ private DeleteWriteResult writePositionDeletes( for (InternalRow row : rows) { String path = row.getString(0); long pos = row.getLong(1); - positionDelete.set(path, pos, null /* no row */); + positionDelete.set(path, pos); closableWriter.write(positionDelete, table.spec(), null); } } diff --git a/spark/v4.0/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v4.0/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java index db5789724056..37931017244d 100644 --- a/spark/v4.0/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java +++ b/spark/v4.0/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java @@ -166,7 +166,7 @@ private DeleteWriteResult write( for (InternalRow row : rows) { String path = row.getString(0); long pos = row.getLong(1); - positionDelete.set(path, pos, null /* no row */); + positionDelete.set(path, pos); closableWriter.write(positionDelete, table.spec(), null); } } diff --git a/spark/v4.0/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceDeleteBenchmark.java b/spark/v4.0/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceDeleteBenchmark.java index e42707bf102b..708aaec4410b 100644 --- a/spark/v4.0/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceDeleteBenchmark.java +++ b/spark/v4.0/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceDeleteBenchmark.java @@ -254,10 +254,10 @@ protected void writePosDeletes(CharSequence path, List deletedPos, int num PositionDelete positionDelete = PositionDelete.create(); try (ClusteredPositionDeleteWriter closeableWriter = writer) { for (Long pos : deletedPos) { - positionDelete.set(path, pos, null); + positionDelete.set(path, pos); closeableWriter.write(positionDelete, unpartitionedSpec, null); for (int i = 0; i < numNoise; i++) { - positionDelete.set(noisePath(path), pos, null); + positionDelete.set(noisePath(path), pos); closeableWriter.write(positionDelete, unpartitionedSpec, null); } } diff --git a/spark/v4.0/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java b/spark/v4.0/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java index 06efb6ba20e4..10785bdc85be 100644 --- a/spark/v4.0/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java +++ b/spark/v4.0/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java @@ -300,7 +300,7 @@ private void writeUnpartitionedClusteredPositionDeleteWriter( for (InternalRow row : positionDeleteRows) { String path = row.getString(0); long pos = row.getLong(1); - positionDelete.set(path, pos, null); + positionDelete.set(path, pos); closeableWriter.write(positionDelete, unpartitionedSpec, null); } } @@ -339,7 +339,7 @@ private void writeUnpartitionedFanoutPositionDeleteWriterPartition( for (InternalRow row : positionDeleteRows) { String path = row.getString(0); long pos = row.getLong(1); - positionDelete.set(path, pos, null); + positionDelete.set(path, pos); closeableWriter.write(positionDelete, unpartitionedSpec, null); } } @@ -379,7 +379,7 @@ private void writeUnpartitionedFanoutPositionDeleteWriterShuffled( for (InternalRow row : shuffledPositionDeleteRows) { String path = row.getString(0); long pos = row.getLong(1); - positionDelete.set(path, pos, null); + positionDelete.set(path, pos); closeableWriter.write(positionDelete, unpartitionedSpec, null); } } diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java index aedb25e4a4a6..b74732ac0a84 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java @@ -714,13 +714,9 @@ public CloseableIterable reader( @Override public PositionDeleteWriter writer( - OutputFile outputFile, - FileFormat format, - PartitionSpec spec, - StructLike partition, - Schema rowSchema) + OutputFile outputFile, FileFormat format, PartitionSpec spec, StructLike partition) throws IOException { - return positionDeletesWriter(outputFile, format, spec, partition, rowSchema); + return positionDeletesWriter(outputFile, format, spec, partition); } } @@ -757,44 +753,13 @@ private static CloseableIterable positionDeletesReader( } private static PositionDeleteWriter positionDeletesWriter( - OutputFile outputFile, - FileFormat format, - PartitionSpec spec, - StructLike partition, - Schema rowSchema) + OutputFile outputFile, FileFormat format, PartitionSpec spec, StructLike partition) throws IOException { - if (rowSchema == null) { - return FormatModelRegistry.positionDeleteWriteBuilder( - format, EncryptedFiles.plainAsEncryptedOutput(outputFile)) - .partition(partition) - .spec(spec) - .build(); - } else { - return switch (format) { - case AVRO -> - Avro.writeDeletes(outputFile) - .createWriterFunc(DataWriter::create) - .withPartition(partition) - .rowSchema(rowSchema) - .withSpec(spec) - .buildPositionWriter(); - case PARQUET -> - Parquet.writeDeletes(outputFile) - .createWriterFunc(GenericParquetWriter::create) - .withPartition(partition) - .rowSchema(rowSchema) - .withSpec(spec) - .buildPositionWriter(); - case ORC -> - ORC.writeDeletes(outputFile) - .createWriterFunc(GenericOrcWriter::buildWriter) - .withPartition(partition) - .rowSchema(rowSchema) - .withSpec(spec) - .buildPositionWriter(); - default -> throw new UnsupportedOperationException("Unsupported file format: " + format); - }; - } + return FormatModelRegistry.positionDeleteWriteBuilder( + format, EncryptedFiles.plainAsEncryptedOutput(outputFile)) + .partition(partition) + .spec(spec) + .build(); } private Set snapshotSet(TableMetadata metadata) { diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java index 39110f0b0597..95c9e8b33e38 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java @@ -169,7 +169,7 @@ public PositionDeleteWriter newPositionDeleteWriter( MetricsConfig metricsConfig = table == null ? MetricsConfig.forPositionDelete() - : MetricsConfig.forPositionDelete(table); + : MetricsConfig.forPositionDelete(); try { return switch (deleteFormat) { diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeletesRewrite.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeletesRewrite.java index 3bbea7b5c98a..d7cef9a39237 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeletesRewrite.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeletesRewrite.java @@ -389,10 +389,10 @@ public void write(InternalRow record) { long position = record.getLong(positionOrdinal); InternalRow row = record.getStruct(rowOrdinal, rowSize); if (row != null) { - positionDelete.set(file, position, row); + positionDelete.set(file, position); lazyWriterWithRow().write(positionDelete, spec, partition); } else { - positionDelete.set(file, position, null); + positionDelete.set(file, position); lazyWriterWithoutRow().write(positionDelete, spec, partition); } } @@ -498,7 +498,7 @@ private static class DVWriter implements DataWriter { public void write(InternalRow record) { String file = record.getString(fileOrdinal); long position = record.getLong(positionOrdinal); - positionDelete.set(file, position, null); + positionDelete.set(file, position); dvWriter.write(positionDelete, spec, partition); } diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java index 38ddefd26a45..fdd8e9ede082 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java @@ -2503,7 +2503,7 @@ private List writePosDeletes( .newPositionDeleteWriter(encrypt(outputFile), table.spec(), partition); PositionDelete posDelete = PositionDelete.create(); - posDeleteWriter.write(posDelete.set(path, rowPosition, null)); + posDeleteWriter.write(posDelete.set(path, rowPosition)); try { posDeleteWriter.close(); } catch (IOException e) { @@ -2547,7 +2547,7 @@ private List writePosDeletes( for (int position = file * positionsPerDeleteFile; position < (file + 1) * positionsPerDeleteFile; position++) { - posDeleteWriter.write(posDelete.set(path, position, null)); + posDeleteWriter.write(posDelete.set(path, position)); } try { diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java index dae721b1d73d..53bee1e6db51 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java @@ -1647,7 +1647,7 @@ private PositionDelete positionDelete( for (int i = 0; i < values.length; i++) { nested.set(i, values[i]); } - posDelete.set(path, position, nested); + posDelete.set(path, position); return posDelete; } diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java index 88113be2ddd9..f390acf1ff0d 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java @@ -2437,7 +2437,7 @@ private DeleteFile writePosDeleteFile(Table table, long pos) { StructLike dataFilePartition = dataFile.partition(); PositionDelete delete = PositionDelete.create(); - delete.set(dataFile.location(), pos, null); + delete.set(dataFile.location(), pos); return writePositionDeletes(table, dataFileSpec, dataFilePartition, ImmutableList.of(delete)); } diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java index f9a2fea30cf0..859ea6d3049a 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java @@ -1573,7 +1573,7 @@ protected void dropTable(String name) { private PositionDelete positionDelete(CharSequence path, Long position) { PositionDelete posDelete = PositionDelete.create(); - posDelete.set(path, position, null); + posDelete.set(path, position); return posDelete; } @@ -1584,7 +1584,7 @@ private PositionDelete positionDelete( for (int i = 0; i < values.length; i++) { nested.set(i, values[i]); } - posDelete.set(path, position, nested); + posDelete.set(path, position); return posDelete; } @@ -1623,7 +1623,7 @@ private StructLikeSet expected( GenericRecord record = GenericRecord.create(finalSchema); record.setField("file_path", p.path()); record.setField("pos", p.pos()); - record.setField("row", formatVersion >= 3 ? null : p.row()); + record.setField("row", null); if (partitionStruct != null) { record.setField("partition", partitionStruct); } diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewritePositionDeleteFiles.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewritePositionDeleteFiles.java index ad49da87bedd..622f1db6ace8 100644 --- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewritePositionDeleteFiles.java +++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewritePositionDeleteFiles.java @@ -339,7 +339,7 @@ private DeleteFile writeDeleteFile( PositionDelete posDelete = PositionDelete.create(); try (Closeable toClose = writer) { for (Pair delete : deletes) { - writer.write(posDelete.set(delete.first(), delete.second(), null)); + writer.write(posDelete.set(delete.first(), delete.second())); } } diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java index 3f242ce228ca..13a575f34379 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java @@ -209,7 +209,7 @@ private DeleteWriteResult writePositionDeletes( for (InternalRow row : rows) { String path = row.getString(0); long pos = row.getLong(1); - positionDelete.set(path, pos, null /* no row */); + positionDelete.set(path, pos); closableWriter.write(positionDelete, table.spec(), null); } } diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java index db5789724056..37931017244d 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java @@ -166,7 +166,7 @@ private DeleteWriteResult write( for (InternalRow row : rows) { String path = row.getString(0); long pos = row.getLong(1); - positionDelete.set(path, pos, null /* no row */); + positionDelete.set(path, pos); closableWriter.write(positionDelete, table.spec(), null); } } diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceDeleteBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceDeleteBenchmark.java index e42707bf102b..708aaec4410b 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceDeleteBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceDeleteBenchmark.java @@ -254,10 +254,10 @@ protected void writePosDeletes(CharSequence path, List deletedPos, int num PositionDelete positionDelete = PositionDelete.create(); try (ClusteredPositionDeleteWriter closeableWriter = writer) { for (Long pos : deletedPos) { - positionDelete.set(path, pos, null); + positionDelete.set(path, pos); closeableWriter.write(positionDelete, unpartitionedSpec, null); for (int i = 0; i < numNoise; i++) { - positionDelete.set(noisePath(path), pos, null); + positionDelete.set(noisePath(path), pos); closeableWriter.write(positionDelete, unpartitionedSpec, null); } } diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java index 06efb6ba20e4..10785bdc85be 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java +++ b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java @@ -300,7 +300,7 @@ private void writeUnpartitionedClusteredPositionDeleteWriter( for (InternalRow row : positionDeleteRows) { String path = row.getString(0); long pos = row.getLong(1); - positionDelete.set(path, pos, null); + positionDelete.set(path, pos); closeableWriter.write(positionDelete, unpartitionedSpec, null); } } @@ -339,7 +339,7 @@ private void writeUnpartitionedFanoutPositionDeleteWriterPartition( for (InternalRow row : positionDeleteRows) { String path = row.getString(0); long pos = row.getLong(1); - positionDelete.set(path, pos, null); + positionDelete.set(path, pos); closeableWriter.write(positionDelete, unpartitionedSpec, null); } } @@ -379,7 +379,7 @@ private void writeUnpartitionedFanoutPositionDeleteWriterShuffled( for (InternalRow row : shuffledPositionDeleteRows) { String path = row.getString(0); long pos = row.getLong(1); - positionDelete.set(path, pos, null); + positionDelete.set(path, pos); closeableWriter.write(positionDelete, unpartitionedSpec, null); } } diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java index aedb25e4a4a6..9905aa66af9e 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java @@ -39,7 +39,6 @@ import org.apache.iceberg.RewriteTablePathUtil; import org.apache.iceberg.RewriteTablePathUtil.PositionDeleteReaderWriter; import org.apache.iceberg.RewriteTablePathUtil.RewriteResult; -import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.StaticTableOperations; import org.apache.iceberg.StatisticsFile; @@ -50,11 +49,7 @@ import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.actions.ImmutableRewriteTablePath; import org.apache.iceberg.actions.RewriteTablePath; -import org.apache.iceberg.avro.Avro; import org.apache.iceberg.data.Record; -import org.apache.iceberg.data.avro.DataWriter; -import org.apache.iceberg.data.orc.GenericOrcWriter; -import org.apache.iceberg.data.parquet.GenericParquetWriter; import org.apache.iceberg.deletes.PositionDeleteWriter; import org.apache.iceberg.encryption.EncryptedFiles; import org.apache.iceberg.exceptions.RuntimeIOException; @@ -64,8 +59,6 @@ import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; import org.apache.iceberg.io.OutputFile; -import org.apache.iceberg.orc.ORC; -import org.apache.iceberg.parquet.Parquet; import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; @@ -714,13 +707,9 @@ public CloseableIterable reader( @Override public PositionDeleteWriter writer( - OutputFile outputFile, - FileFormat format, - PartitionSpec spec, - StructLike partition, - Schema rowSchema) + OutputFile outputFile, FileFormat format, PartitionSpec spec, StructLike partition) throws IOException { - return positionDeletesWriter(outputFile, format, spec, partition, rowSchema); + return positionDeletesWriter(outputFile, format, spec, partition); } } @@ -757,44 +746,13 @@ private static CloseableIterable positionDeletesReader( } private static PositionDeleteWriter positionDeletesWriter( - OutputFile outputFile, - FileFormat format, - PartitionSpec spec, - StructLike partition, - Schema rowSchema) + OutputFile outputFile, FileFormat format, PartitionSpec spec, StructLike partition) throws IOException { - if (rowSchema == null) { - return FormatModelRegistry.positionDeleteWriteBuilder( - format, EncryptedFiles.plainAsEncryptedOutput(outputFile)) - .partition(partition) - .spec(spec) - .build(); - } else { - return switch (format) { - case AVRO -> - Avro.writeDeletes(outputFile) - .createWriterFunc(DataWriter::create) - .withPartition(partition) - .rowSchema(rowSchema) - .withSpec(spec) - .buildPositionWriter(); - case PARQUET -> - Parquet.writeDeletes(outputFile) - .createWriterFunc(GenericParquetWriter::create) - .withPartition(partition) - .rowSchema(rowSchema) - .withSpec(spec) - .buildPositionWriter(); - case ORC -> - ORC.writeDeletes(outputFile) - .createWriterFunc(GenericOrcWriter::buildWriter) - .withPartition(partition) - .rowSchema(rowSchema) - .withSpec(spec) - .buildPositionWriter(); - default -> throw new UnsupportedOperationException("Unsupported file format: " + format); - }; - } + return FormatModelRegistry.positionDeleteWriteBuilder( + format, EncryptedFiles.plainAsEncryptedOutput(outputFile)) + .partition(partition) + .spec(spec) + .build(); } private Set snapshotSet(TableMetadata metadata) { diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java index 39110f0b0597..b63af5c256d0 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java @@ -167,9 +167,7 @@ public PositionDeleteWriter newPositionDeleteWriter( LOG.warn("Position deletes with deleted rows are deprecated and will be removed in 1.12.0."); Map properties = table == null ? ImmutableMap.of() : table.properties(); MetricsConfig metricsConfig = - table == null - ? MetricsConfig.forPositionDelete() - : MetricsConfig.forPositionDelete(table); + table == null ? MetricsConfig.forPositionDelete() : MetricsConfig.forPositionDelete(); try { return switch (deleteFormat) { diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeletesRewrite.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeletesRewrite.java index 6b8be68c62b9..e6080003282d 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeletesRewrite.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeletesRewrite.java @@ -391,10 +391,10 @@ public void write(InternalRow record) { long position = record.getLong(positionOrdinal); InternalRow row = record.getStruct(rowOrdinal, rowSize); if (row != null) { - positionDelete.set(file, position, row); + positionDelete.set(file, position); lazyWriterWithRow().write(positionDelete, spec, partition); } else { - positionDelete.set(file, position, null); + positionDelete.set(file, position); lazyWriterWithoutRow().write(positionDelete, spec, partition); } } @@ -500,7 +500,7 @@ private static class DVWriter implements DataWriter { public void write(InternalRow record) { String file = record.getString(fileOrdinal); long position = record.getLong(positionOrdinal); - positionDelete.set(file, position, null); + positionDelete.set(file, position); dvWriter.write(positionDelete, spec, partition); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java index 110e43ede1f9..ce9cf416fc33 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java @@ -2503,7 +2503,7 @@ private List writePosDeletes( .newPositionDeleteWriter(encrypt(outputFile), table.spec(), partition); PositionDelete posDelete = PositionDelete.create(); - posDeleteWriter.write(posDelete.set(path, rowPosition, null)); + posDeleteWriter.write(posDelete.set(path, rowPosition)); try { posDeleteWriter.close(); } catch (IOException e) { @@ -2547,7 +2547,7 @@ private List writePosDeletes( for (int position = file * positionsPerDeleteFile; position < (file + 1) * positionsPerDeleteFile; position++) { - posDeleteWriter.write(posDelete.set(path, position, null)); + posDeleteWriter.write(posDelete.set(path, position)); } try { diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java index dae721b1d73d..53bee1e6db51 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java @@ -1647,7 +1647,7 @@ private PositionDelete positionDelete( for (int i = 0; i < values.length; i++) { nested.set(i, values[i]); } - posDelete.set(path, position, nested); + posDelete.set(path, position); return posDelete; } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java index c322730440d1..16e1754c1b48 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java @@ -2445,7 +2445,7 @@ private DeleteFile writePosDeleteFile(Table table, long pos) { StructLike dataFilePartition = dataFile.partition(); PositionDelete delete = PositionDelete.create(); - delete.set(dataFile.location(), pos, null); + delete.set(dataFile.location(), pos); return writePositionDeletes(table, dataFileSpec, dataFilePartition, ImmutableList.of(delete)); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java index 14ad107e50e3..79186dd96826 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java @@ -1535,7 +1535,7 @@ protected void dropTable(String name) { private PositionDelete positionDelete(CharSequence path, Long position) { PositionDelete posDelete = PositionDelete.create(); - posDelete.set(path, position, null); + posDelete.set(path, position); return posDelete; } @@ -1546,7 +1546,7 @@ private PositionDelete positionDelete( for (int i = 0; i < values.length; i++) { nested.set(i, values[i]); } - posDelete.set(path, position, nested); + posDelete.set(path, position); return posDelete; } @@ -1585,7 +1585,7 @@ private StructLikeSet expected( GenericRecord record = GenericRecord.create(finalSchema); record.setField("file_path", p.path()); record.setField("pos", p.pos()); - record.setField("row", formatVersion >= 3 ? null : p.row()); + record.setField("row", null); if (partitionStruct != null) { record.setField("partition", partitionStruct); }