diff --git a/pom.xml b/pom.xml index 2e30905d..1c3dc5b2 100644 --- a/pom.xml +++ b/pom.xml @@ -16,9 +16,9 @@ 1.8 1.8 UTF-8 - 2.12.15 + 2.12.20 2.12 - 3.1.2 + 3.5.5 3.2.2 @@ -114,6 +114,12 @@ ${spark.version} provided + + org.apache.spark + spark-sql-api_${scala.version.short} + ${spark.version} + provided + org.apache.spark spark-unsafe_${scala.version.short} diff --git a/samples/README.md b/samples/README.md index ea1ae264..fb8d1713 100644 --- a/samples/README.md +++ b/samples/README.md @@ -56,7 +56,7 @@ follow the steps: ``` 4. Create a Dataproc cluster ```sh - gcloud dataproc clusters create $CLUSTER_NAME --region=$REGION --zone=$REGION-$ZONE_ID --image-version=1.5-debian10 --scopes=cloud-platform + gcloud dataproc clusters create $CLUSTER_NAME --region=$REGION --zone=$REGION-$ZONE_ID --image-version=2.3-debian12 --scopes=cloud-platform ``` 5. Package sample jar ```sh diff --git a/samples/simple_read_sample.sh b/samples/simple_read_sample.sh index bf81cadd..b3cb1ca1 100644 --- a/samples/simple_read_sample.sh +++ b/samples/simple_read_sample.sh @@ -25,7 +25,7 @@ if [ "$1" == "run" ]; then gcloud dataproc clusters create $CLUSTER_NAME \ --region=$REGION \ --zone=$REGION-$ZONE_ID \ - --image-version=1.5-debian10 \ + --image-version=2.3-debian12 \ --scopes=cloud-platform # Package sample jar diff --git a/samples/simple_write_sample.sh b/samples/simple_write_sample.sh index 6688b752..47ac0767 100644 --- a/samples/simple_write_sample.sh +++ b/samples/simple_write_sample.sh @@ -21,7 +21,7 @@ if [ "$1" == "run" ]; then gcloud dataproc clusters create $CLUSTER_NAME \ --region=$REGION \ --zone=$REGION-$ZONE_ID \ - --image-version=1.5-debian10 \ + --image-version=2.3-debian12 \ --scopes=cloud-platform # Package sample jar diff --git a/samples/word_count_sample.sh b/samples/word_count_sample.sh index 052eee36..361c25db 100644 --- a/samples/word_count_sample.sh +++ b/samples/word_count_sample.sh @@ -26,7 +26,7 @@ if [ "$1" == "run" ]; then gcloud dataproc clusters create $CLUSTER_NAME \ --region=$REGION \ --zone=$REGION-$ZONE_ID \ - --image-version=1.5-debian10 \ + --image-version=2.3-debian12 \ --scopes=cloud-platform # Package sample jar diff --git a/src/main/java/com/google/cloud/pubsublite/spark/BaseDataStream.java b/src/main/java/com/google/cloud/pubsublite/spark/BaseDataStream.java index d95a1476..e91fbd4a 100644 --- a/src/main/java/com/google/cloud/pubsublite/spark/BaseDataStream.java +++ b/src/main/java/com/google/cloud/pubsublite/spark/BaseDataStream.java @@ -81,6 +81,7 @@ public void commit(org.apache.spark.sql.connector.read.streaming.Offset end) { } @Override + @SuppressWarnings("try") public void stop() { try (AutoCloseable a = committer; AutoCloseable b = cursorClient; diff --git a/src/main/java/com/google/cloud/pubsublite/spark/PslWrite.java b/src/main/java/com/google/cloud/pubsublite/spark/PslWrite.java index c9868107..7f368a9b 100644 --- a/src/main/java/com/google/cloud/pubsublite/spark/PslWrite.java +++ b/src/main/java/com/google/cloud/pubsublite/spark/PslWrite.java @@ -102,4 +102,9 @@ public BatchWrite buildForBatch() { public StreamingWrite buildForStreaming() { return this; } + + @Override + public boolean useCommitCoordinator() { + return true; + } } diff --git a/src/main/java/com/google/cloud/pubsublite/spark/internal/LimitingHeadOffsetReader.java b/src/main/java/com/google/cloud/pubsublite/spark/internal/LimitingHeadOffsetReader.java index a974ba23..48b9602a 100644 --- a/src/main/java/com/google/cloud/pubsublite/spark/internal/LimitingHeadOffsetReader.java +++ b/src/main/java/com/google/cloud/pubsublite/spark/internal/LimitingHeadOffsetReader.java @@ -101,6 +101,7 @@ public PslSourceOffset getHeadOffset() { } @Override + @SuppressWarnings("try") public void close() { try (AutoCloseable a = topicStatsClient; Closeable b = partitionCountReader) { diff --git a/src/main/java/com/google/cloud/pubsublite/spark/internal/MultiPartitionCommitter.java b/src/main/java/com/google/cloud/pubsublite/spark/internal/MultiPartitionCommitter.java index bf6441e8..def26a6e 100644 --- a/src/main/java/com/google/cloud/pubsublite/spark/internal/MultiPartitionCommitter.java +++ b/src/main/java/com/google/cloud/pubsublite/spark/internal/MultiPartitionCommitter.java @@ -29,5 +29,6 @@ interface CommitterFactory { void commit(PslSourceOffset offset); + @Override void close(); }