diff --git a/CHANGELOG.md b/CHANGELOG.md index 33dd2a8f..6a157f1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file. - Add `SparkApplication.spec.job.retryOnFailureCount` field with a default of `0`. This has the effect that applications where the `spark-submit` Pod fails are not resubmitted. Previously, Jobs were retried at most 6 times by default ([#647]). +- Support for Spark `3.5.8` ([#650]). ### Changed @@ -24,12 +25,14 @@ All notable changes to this project will be documented in this file. ### Removed - Support for Spark `3.5.6` ([#642]). +- Deprecated support for Spark `3.5.7` ([#650]). [#640]: https://github.com/stackabletech/spark-k8s-operator/pull/640 [#642]: https://github.com/stackabletech/spark-k8s-operator/pull/642 [#647]: https://github.com/stackabletech/spark-k8s-operator/pull/647 [#648]: https://github.com/stackabletech/spark-k8s-operator/pull/648 [#649]: https://github.com/stackabletech/spark-k8s-operator/pull/649 +[#650]: https://github.com/stackabletech/spark-k8s-operator/pull/650 [#651]: https://github.com/stackabletech/spark-k8s-operator/pull/651 ## [25.11.0] - 2025-11-07 diff --git a/docs/modules/spark-k8s/examples/example-history-app.yaml b/docs/modules/spark-k8s/examples/example-history-app.yaml index e963dada..8a5f78f4 100644 --- a/docs/modules/spark-k8s/examples/example-history-app.yaml +++ b/docs/modules/spark-k8s/examples/example-history-app.yaml @@ -5,7 +5,7 @@ metadata: name: spark-pi-s3-1 spec: sparkImage: - productVersion: 3.5.7 + productVersion: 3.5.8 pullPolicy: IfNotPresent mode: cluster mainClass: org.apache.spark.examples.SparkPi diff --git a/docs/modules/spark-k8s/examples/example-history-server.yaml b/docs/modules/spark-k8s/examples/example-history-server.yaml index b303f4fd..8dbb61f1 100644 --- a/docs/modules/spark-k8s/examples/example-history-server.yaml +++ b/docs/modules/spark-k8s/examples/example-history-server.yaml @@ -5,7 +5,7 @@ metadata: name: spark-history spec: image: - productVersion: 3.5.7 + productVersion: 3.5.8 logFileDirectory: # <1> s3: prefix: eventlogs/ # <2> diff --git a/docs/modules/spark-k8s/examples/example-spark-connect.yaml b/docs/modules/spark-k8s/examples/example-spark-connect.yaml index db18b1b8..18b8c207 100644 --- a/docs/modules/spark-k8s/examples/example-spark-connect.yaml +++ b/docs/modules/spark-k8s/examples/example-spark-connect.yaml @@ -5,7 +5,7 @@ metadata: name: spark-connect # <1> spec: image: - productVersion: "3.5.7" # <2> + productVersion: "3.5.8" # <2> pullPolicy: IfNotPresent args: - "--package org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1" # <3> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml index 92415a02..e12fa1fd 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.7 + productVersion: 3.5.8 mode: cluster mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.1.0.jar # <3> mainClass: tech.stackable.demo.spark.NYTLCReport diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml index 8ce2cb8c..4bbe3e0e 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml @@ -7,7 +7,7 @@ metadata: spec: image: oci.stackable.tech/stackable/ny-tlc-report:0.2.0 # <1> sparkImage: - productVersion: 3.5.7 + productVersion: 3.5.8 mode: cluster mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py # <2> args: diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml index bf80d0bf..9759aa90 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.7 + productVersion: 3.5.8 mode: cluster mainApplicationFile: s3a://my-bucket/app.jar # <1> mainClass: org.example.App # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml index 6e05fbf4..3c360239 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml @@ -5,7 +5,7 @@ metadata: name: example-sparkapp-s3-private spec: sparkImage: - productVersion: 3.5.7 + productVersion: 3.5.8 mode: cluster mainApplicationFile: s3a://my-bucket/spark-examples.jar # <1> mainClass: org.apache.spark.examples.SparkPi # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml index 3302b8f8..f3c8a5f3 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.7 + productVersion: 3.5.8 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py args: diff --git a/docs/modules/spark-k8s/examples/getting_started/application.yaml b/docs/modules/spark-k8s/examples/getting_started/application.yaml index 54e7194f..5057495b 100644 --- a/docs/modules/spark-k8s/examples/getting_started/application.yaml +++ b/docs/modules/spark-k8s/examples/getting_started/application.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: # <2> - productVersion: 3.5.7 + productVersion: 3.5.8 mode: cluster # <3> mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py # <4> job: # <5> diff --git a/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc b/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc index 9b7bc4f9..c74485ac 100644 --- a/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc +++ b/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc @@ -29,7 +29,7 @@ Below is an example of a custom image that includes a JDBC driver: [source, Dockerfile] ---- -FROM oci.stackable.tech/sdp/spark-k8s:3.5.7-stackable0.0.0-dev # <1> +FROM oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev # <1> RUN curl --fail -o /stackable/spark/jars/postgresql-42.6.0.jar "https://jdbc.postgresql.org/download/postgresql-42.6.0.jar" # <2> ---- @@ -41,8 +41,8 @@ Build your custom image and push it to your container registry. [source, bash] ---- -docker build -t my-registry/spark-k8s:3.5.7-psql . -docker push my-registry/spark-k8s:3.5.7-psql +docker build -t my-registry/spark-k8s:3.5.8-psql . +docker push my-registry/spark-k8s:3.5.8-psql ---- And the following snippet showcases an application that uses the custom image: @@ -55,8 +55,8 @@ metadata: name: spark-jdbc spec: sparkImage: - custom: "my-registry/spark-k8s:3.5.7-psql" # <1> - productVersion: "3.5.7" # <2> + custom: "my-registry/spark-k8s:3.5.8-psql" # <1> + productVersion: "3.5.8" # <2> ... ---- diff --git a/docs/modules/spark-k8s/partials/supported-versions.adoc b/docs/modules/spark-k8s/partials/supported-versions.adoc index eb04407c..72c53867 100644 --- a/docs/modules/spark-k8s/partials/supported-versions.adoc +++ b/docs/modules/spark-k8s/partials/supported-versions.adoc @@ -3,11 +3,12 @@ // Stackable Platform documentation. // Please sort the versions in descending order (newest first) -- 4.1.1 (Hadoop 3.4.2, Scala 2.13, Python 3.12, Java 21) (Experimental) -- 4.0.1 (Hadoop 3.4.2, Scala 2.13, Python 3.12, Java 21) -- 3.5.7 (Hadoop 3.4.2, Scala 2.12, Python 3.11, Java 17) (LTS) +- 4.1.1 (Hadoop 3.4.2, Scala 2.13, Python 3.12, Java 21) +- 4.0.1 (Hadoop 3.4.2, Scala 2.13, Python 3.12, Java 21) (Deprecated) +- 3.5.8 (Hadoop 3.4.2, Scala 2.12, Python 3.11, Java 17) (LTS) +- 3.5.7 (Hadoop 3.4.2, Scala 2.12, Python 3.11, Java 17) (Deprecated) -Some reasons why Spark 4.1.1 is considered experimental (as of January 2026): +Apache Spark 4.1.1 has the following known issues (as of February 2026): - Missing HBase compatibility (See: https://github.com/apache/hbase-connectors/pull/130) - No Iceberg Spark runtime release with support for Spark 4.1 available yet. diff --git a/examples/README-examples.md b/examples/README-examples.md index f26927b5..aa7eecf4 100644 --- a/examples/README-examples.md +++ b/examples/README-examples.md @@ -50,10 +50,10 @@ Several resources are needed in this store. These can be loaded like this: ```text kubectl exec minio-mc-0 -- sh -c 'mc alias set test-minio http://test-minio:9000/' -kubectl cp tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.7.jar minio-mc-0:/tmp +kubectl cp tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.8.jar minio-mc-0:/tmp kubectl cp apps/ny_tlc_report.py minio-mc-0:/tmp kubectl cp examples/yellow_tripdata_2021-07.csv minio-mc-0:/tmp -kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.5.7.jar test-minio/my-bucket +kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.5.8.jar test-minio/my-bucket kubectl exec minio-mc-0 -- mc cp /tmp/ny_tlc_report.py test-minio/my-bucket kubectl exec minio-mc-0 -- mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket ``` diff --git a/examples/ny-tlc-report-external-dependencies.yaml b/examples/ny-tlc-report-external-dependencies.yaml index cc45bed2..77ecece6 100644 --- a/examples/ny-tlc-report-external-dependencies.yaml +++ b/examples/ny-tlc-report-external-dependencies.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.7 + productVersion: 3.5.8 pullPolicy: IfNotPresent mode: cluster mainApplicationFile: s3a://my-bucket/ny_tlc_report.py diff --git a/examples/ny-tlc-report-image.yaml b/examples/ny-tlc-report-image.yaml index 73daa848..40bbc98d 100644 --- a/examples/ny-tlc-report-image.yaml +++ b/examples/ny-tlc-report-image.yaml @@ -7,7 +7,7 @@ metadata: spec: # everything under /jobs will be copied to /stackable/spark/jobs image: oci.stackable.tech/stackable/ny-tlc-report:0.2.0 - sparkImage: oci.stackable.tech/sdp/spark-k8s:3.5.7-stackable0.0.0-dev + sparkImage: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev sparkImagePullPolicy: IfNotPresent mode: cluster mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py diff --git a/examples/ny-tlc-report.yaml b/examples/ny-tlc-report.yaml index c20ed4a3..64ed648a 100644 --- a/examples/ny-tlc-report.yaml +++ b/examples/ny-tlc-report.yaml @@ -13,9 +13,9 @@ metadata: name: spark-ny-cm spec: sparkImage: - productVersion: 3.5.7 + productVersion: 3.5.8 mode: cluster - mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.5.7.jar + mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.5.8.jar mainClass: tech.stackable.demo.spark.NYTLCReport volumes: - name: cm-job-arguments diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 44faf89e..5defa934 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -47,7 +47,7 @@ mod test { name: spark-history spec: image: - productVersion: 3.5.7 + productVersion: 3.5.8 logFileDirectory: s3: prefix: eventlogs/ diff --git a/rust/operator-binary/src/crd/history.rs b/rust/operator-binary/src/crd/history.rs index 35d372c2..45dbd77f 100644 --- a/rust/operator-binary/src/crd/history.rs +++ b/rust/operator-binary/src/crd/history.rs @@ -452,7 +452,7 @@ mod test { name: spark-history spec: image: - productVersion: 3.5.7 + productVersion: 3.5.8 logFileDirectory: s3: prefix: eventlogs/ diff --git a/rust/operator-binary/src/history/config/jvm.rs b/rust/operator-binary/src/history/config/jvm.rs index ed3fd7ba..b650aedb 100644 --- a/rust/operator-binary/src/history/config/jvm.rs +++ b/rust/operator-binary/src/history/config/jvm.rs @@ -65,7 +65,7 @@ mod tests { name: spark-history spec: image: - productVersion: 3.5.7 + productVersion: 3.5.8 logFileDirectory: s3: prefix: eventlogs/ @@ -98,7 +98,7 @@ mod tests { name: spark-history spec: image: - productVersion: 3.5.7 + productVersion: 3.5.8 logFileDirectory: s3: prefix: eventlogs/ diff --git a/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.7.jar b/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.7.jar index f969e32c..aee1fdb0 100644 Binary files a/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.7.jar and b/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.7.jar differ diff --git a/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.8.jar b/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.8.jar new file mode 100644 index 00000000..436e664e Binary files /dev/null and b/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.8.jar differ diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 8857b14e..ebb0b697 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -6,6 +6,7 @@ dimensions: - name: spark values: - 3.5.7 + - 3.5.8 - 4.0.1 - 4.1.1 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version @@ -14,17 +15,20 @@ dimensions: - name: spark-logging values: - 3.5.7 + - 3.5.8 - 4.0.1 - 4.1.1 - name: spark-hbase-connector values: - 3.5.7 + - 3.5.8 # No hbase-connector release with support for Spark 4 yet. # - 4.0.1 # - 4.1.1 - name: spark-delta-lake values: - 3.5.7 + - 3.5.8 - 4.0.1 # No delta-lake release with support for Spark 4.1 yet # - 4.1.1 @@ -32,12 +36,14 @@ dimensions: - name: spark-iceberg values: - 3.5.7 + - 3.5.8 - 4.0.1 # No iceberg release with support for Spark 4.1 yet # - 4.1.1 - name: spark-connect values: - 3.5.7 + - 3.5.8 - 4.0.1 - 4.1.1 # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev