From 4d52af2166706845fddb9efa34a2544541cf16d9 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Fri, 27 Feb 2026 09:09:37 -0800 Subject: [PATCH] [SPARK-55748][SQL] Use `DSv2` for `avro|csv|json|kafka|orc|parquet|text` by default --- docs/sql-migration-guide.md | 1 + .../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md index 6eab844a6fd00..473d791e19f98 100644 --- a/docs/sql-migration-guide.md +++ b/docs/sql-migration-guide.md @@ -26,6 +26,7 @@ license: | - Since Spark 4.2, Spark enables order-independent checksums for shuffle outputs by default to detect data inconsistencies during indeterminate shuffle stage retries. If a checksum mismatch is detected, Spark rolls back and re-executes all succeeding stages that depend on the shuffle output. If rolling back is not possible for some succeeding stages, the job will fail. To restore the previous behavior, set `spark.sql.shuffle.orderIndependentChecksum.enabled` and `spark.sql.shuffle.orderIndependentChecksum.enableFullRetryOnMismatch` to `false`. - Since Spark 4.2, support for Derby JDBC datasource is deprecated. +- Since Spark 4.2, Spark uses `DataSourceV2` implementations of `avro`, `csv`, `json`, `kafka`, `orc`, `parquet`, and `text` data sources by default. To restore the previous behavior, set `spark.sql.sources.useV1SourceList=avro,csv,json,kafka,orc,parquet,text`. ## Upgrading from Spark SQL 4.0 to 4.1 diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 1874ff195516b..9b306124bb83f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -4650,7 +4650,7 @@ object SQLConf { "sources will fallback to Data Source V1 code path.") .version("3.0.0") .stringConf - .createWithDefault("avro,csv,json,kafka,orc,parquet,text") + .createWithDefault("") val ALLOW_EMPTY_SCHEMAS_FOR_WRITES = buildConf("spark.sql.legacy.allowEmptySchemaWrite") .internal()