diff --git a/internal/recipe/Dockerfile.spark4 b/internal/recipe/Dockerfile.spark4 index 1361a8f03..6a47e8d59 100644 --- a/internal/recipe/Dockerfile.spark4 +++ b/internal/recipe/Dockerfile.spark4 @@ -17,7 +17,7 @@ FROM apache/spark:4.0.2-python3 USER root -ARG ICEBERG_VERSION=1.10.0 +ARG ICEBERG_VERSION=1.11.0 ARG SCALA_VERSION=2.13 RUN curl --progress-bar -L --retry 3 \ diff --git a/internal/recipe/provision.py b/internal/recipe/provision.py index 0cec3f2a0..8a8227163 100644 --- a/internal/recipe/provision.py +++ b/internal/recipe/provision.py @@ -24,6 +24,9 @@ spark = SparkSession.builder.getOrCreate() +# Spark 4 (Iceberg 1.11+) warns on a bare "default." in ALTER extensions; Spark 3.5 (Iceberg 1.8) can't plan them with an explicit catalog. Qualify only on 4+. +alter_ns = "spark_catalog.default" if int(spark.version.split(".")[0]) >= 4 else "default" + catalogs = { 'rest': load_catalog( "rest", @@ -138,7 +141,7 @@ # Partitioning is not really needed, but there is a bug: # https://github.com/apache/iceberg/pull/7685 - spark.sql(f"ALTER TABLE default.test_positional_mor_deletes ADD PARTITION FIELD years(dt) AS dt_years") + spark.sql(f"ALTER TABLE {alter_ns}.test_positional_mor_deletes ADD PARTITION FIELD years(dt) AS dt_years") spark.sql( f""" @@ -159,9 +162,9 @@ """ ) - spark.sql(f"ALTER TABLE default.test_positional_mor_deletes CREATE TAG tag_12") + spark.sql(f"ALTER TABLE {alter_ns}.test_positional_mor_deletes CREATE TAG tag_12") - spark.sql(f"ALTER TABLE default.test_positional_mor_deletes CREATE BRANCH without_5") + spark.sql(f"ALTER TABLE {alter_ns}.test_positional_mor_deletes CREATE BRANCH without_5") # WAP must be enabled for spark.wap.branch to scope DML to the branch spark.sql(f"ALTER TABLE default.test_positional_mor_deletes SET TBLPROPERTIES ('write.wap.enabled'='true')") @@ -189,7 +192,7 @@ """ ) - spark.sql(f"ALTER TABLE default.test_positional_mor_double_deletes ADD PARTITION FIELD years(dt) AS dt_years") + spark.sql(f"ALTER TABLE {alter_ns}.test_positional_mor_double_deletes ADD PARTITION FIELD years(dt) AS dt_years") spark.sql( f""" @@ -272,7 +275,7 @@ """ ) - spark.sql(f"ALTER TABLE default.{table_name} ADD PARTITION FIELD {partition}") + spark.sql(f"ALTER TABLE {alter_ns}.{table_name} ADD PARTITION FIELD {partition}") spark.sql( f"""