diff --git a/sql/api/src/main/scala/org/apache/spark/sql/Column.scala b/sql/api/src/main/scala/org/apache/spark/sql/Column.scala index 56a9787db092e..4befebd8418b5 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/Column.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/Column.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql import scala.jdk.CollectionConverters._ -import org.apache.spark.annotation.Stable +import org.apache.spark.annotation.{DeveloperApi, Stable} import org.apache.spark.internal.Logging import org.apache.spark.internal.LogKeys.{LEFT_EXPR, RIGHT_EXPR} import org.apache.spark.sql.catalyst.parser.DataTypeParser @@ -30,11 +30,19 @@ import org.apache.spark.sql.internal.{ColumnNode, TableValuedFunctionArgument} import org.apache.spark.sql.types._ import org.apache.spark.util.ArrayImplicits._ -private[spark] object Column { +/** + * The companion object is public so that the `Column` type can be referenced as a value. This + * allows an implementation to add a `Column(expression)` factory through an extension method. All + * of its members are internal to Spark. + * + * @since 4.4.0 + */ +@DeveloperApi +object Column { - def apply(colName: String): Column = new Column(colName) + private[spark] def apply(colName: String): Column = new Column(colName) - def apply(node: => ColumnNode): Column = withOrigin(new Column(node)) + private[spark] def apply(node: => ColumnNode): Column = withOrigin(new Column(node)) /** * Invoke a function with an options map as its last argument. If there are no options, its diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/conversions.scala b/sql/core/src/main/scala/org/apache/spark/sql/classic/conversions.scala index 3cfdace73c458..fd5771778b277 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/classic/conversions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/conversions.scala @@ -59,7 +59,21 @@ trait ClassicConversions { } @DeveloperApi -object ClassicConversions extends ClassicConversions +object ClassicConversions extends ClassicConversions { + /** + * Convert an [[Expression]] into a [[Column]]. This is the counterpart of + * [[ColumnConversions.expression]], for callers that would rather name the conversion than rely + * on the implicit [[ClassicConversions.ColumnConstructorExt]]. + * + * This is intentionally defined on the object rather than on the [[ClassicConversions]] trait: + * on the trait it would shadow `functions.column(colName: String)` for everyone who mixes the + * trait in, which is the trait's documented use case. + * + * @since 4.4.0 + */ + @DeveloperApi + def column(e: Expression): Column = ExpressionUtils.column(e) +} /** * Conversions from a [[Column]] to an [[Expression]]. diff --git a/sql/core/src/test/scala/test/org/apache/spark/sql/ExpressionToColumnSuite.scala b/sql/core/src/test/scala/test/org/apache/spark/sql/ExpressionToColumnSuite.scala new file mode 100644 index 0000000000000..c03453b4856a4 --- /dev/null +++ b/sql/core/src/test/scala/test/org/apache/spark/sql/ExpressionToColumnSuite.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package test.org.apache.spark.sql + +import org.apache.spark.sql.{Column, QueryTest, Row} +import org.apache.spark.sql.catalyst.expressions.{Expression, Literal} +import org.apache.spark.sql.classic.{ClassicConversions, ColumnConversions} +import org.apache.spark.sql.classic.ClassicConversions._ +import org.apache.spark.sql.test.SharedSparkSession + +/** + * Tests the public Expression <-> Column conversions from a package outside of + * `org.apache.spark`, which is the only way to catch that a step of the path is package-private. + * Compiling this suite is as much a part of the test as running it. + */ +class ExpressionToColumnSuite extends QueryTest with SharedSparkSession { + + test("SPARK-49828: build a Column from an Expression via the Column companion") { + val e: Expression = Literal(1) + val c: Column = Column(e) + assert(ColumnConversions.expression(c) == e) + } + + test("SPARK-49828: build a Column from an Expression via ClassicConversions.column") { + val e: Expression = Literal(1) + val c: Column = ClassicConversions.column(e) + assert(ColumnConversions.expression(c) == e) + } + + test("SPARK-49828: a Column built from an Expression is usable in a query") { + val df = spark.range(2).select(Column(Literal(1)).as("one")) + checkAnswer(df, Seq(Row(1), Row(1))) + } +}