From d85b90c257aa89c10b9796e3263a9e28e06e9fba Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Mon, 13 Apr 2026 19:51:19 +0800 Subject: [PATCH 1/2] [SPARK-56459][SQL] Fix MatchError in FileDataSourceV2.attachFilePath for fatal errors attachFilePath uses NonFatal pattern matching which does not match fatal errors like OutOfMemoryError. When OOM occurs during file reading, the incomplete match causes a MatchError that masks the original error. Add a catch-all case to rethrow fatal errors directly. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../spark/sql/execution/datasources/v2/FileDataSourceV2.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala index 4242fc5d8510a..98273a5faf85e 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2.scala @@ -140,6 +140,7 @@ object FileDataSourceV2 { throw QueryExecutionErrors.fileNotExistError(filePath, e) case NonFatal(e) => throw QueryExecutionErrors.cannotReadFilesError(e, filePath) + case other => throw other } } } From 440649fdc54a53b940358fb9b7ea2e435cbfa3c5 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Tue, 14 Apr 2026 15:31:16 +0000 Subject: [PATCH 2/2] [SPARK-56459][SQL][TEST] Add unit tests for FileDataSourceV2.attachFilePath Add comprehensive unit tests for all branches of FileDataSourceV2.attachFilePath: - SparkUpgradeException is rethrown directly - FAILED_READ_FILE.CANNOT_READ_FILE_FOOTER is rethrown directly - SchemaColumnConvertNotSupportedException is wrapped - FileNotFoundException is wrapped - NonFatal exceptions are wrapped - Fatal errors (OutOfMemoryError, StackOverflowError) are rethrown directly Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../v2/FileDataSourceV2Suite.scala | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2Suite.scala diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2Suite.scala new file mode 100644 index 0000000000000..ff177d676c0e5 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/FileDataSourceV2Suite.scala @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.v2 + +import java.io.FileNotFoundException + +import org.apache.spark.{SparkException, SparkFunSuite, SparkUpgradeException} +import org.apache.spark.sql.execution.datasources.SchemaColumnConvertNotSupportedException + +class FileDataSourceV2Suite extends SparkFunSuite { + + private val testFilePath = "/test/path/file.parquet" + + test("SPARK-56459: attachFilePath rethrows SparkUpgradeException directly") { + val cause = new SparkUpgradeException( + "INCONSISTENT_BEHAVIOR_CROSS_VERSION.READ_ANCIENT_DATETIME", + Map("format" -> "Parquet", "config" -> "config", "option" -> "option"), + null) + val thrown = intercept[SparkUpgradeException] { + FileDataSourceV2.attachFilePath(testFilePath, cause) + } + assert(thrown eq cause) + } + + test("SPARK-56459: attachFilePath rethrows FAILED_READ_FILE.CANNOT_READ_FILE_FOOTER") { + val cause = new SparkException( + errorClass = "FAILED_READ_FILE.CANNOT_READ_FILE_FOOTER", + messageParameters = Map("path" -> testFilePath), + cause = null) + val thrown = intercept[SparkException] { + FileDataSourceV2.attachFilePath(testFilePath, cause) + } + assert(thrown eq cause) + } + + test("SPARK-56459: attachFilePath wraps SchemaColumnConvertNotSupportedException") { + val cause = new SchemaColumnConvertNotSupportedException("col1", "INT32", "STRING") + val thrown = intercept[SparkException] { + FileDataSourceV2.attachFilePath(testFilePath, cause) + } + assert(thrown.getCondition == "FAILED_READ_FILE.PARQUET_COLUMN_DATA_TYPE_MISMATCH") + assert(thrown.getCause eq cause) + } + + test("SPARK-56459: attachFilePath wraps FileNotFoundException") { + val cause = new FileNotFoundException("file not found") + val thrown = intercept[SparkException] { + FileDataSourceV2.attachFilePath(testFilePath, cause) + } + assert(thrown.getCondition == "FAILED_READ_FILE.FILE_NOT_EXIST") + assert(thrown.getCause eq cause) + } + + test("SPARK-56459: attachFilePath wraps NonFatal exceptions") { + val cause = new RuntimeException("something went wrong") + val thrown = intercept[SparkException] { + FileDataSourceV2.attachFilePath(testFilePath, cause) + } + assert(thrown.getCondition == "FAILED_READ_FILE.NO_HINT") + assert(thrown.getCause eq cause) + } + + test("SPARK-56459: attachFilePath rethrows fatal errors directly") { + val oom = new OutOfMemoryError("Java heap space") + val thrown = intercept[OutOfMemoryError] { + FileDataSourceV2.attachFilePath(testFilePath, oom) + } + assert(thrown eq oom) + } + + test("SPARK-56459: attachFilePath rethrows StackOverflowError directly") { + val soe = new StackOverflowError("stack overflow") + val thrown = intercept[StackOverflowError] { + FileDataSourceV2.attachFilePath(testFilePath, soe) + } + assert(thrown eq soe) + } +}