From 96d2e161b31f404889199170ce2c6e871103f910 Mon Sep 17 00:00:00 2001 From: Ali Ebrahim Date: Wed, 27 May 2026 12:45:30 -0700 Subject: [PATCH] Create temporary dataset with a 24 hour ttl. --- sdks/python/apache_beam/io/gcp/bigquery_tools.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sdks/python/apache_beam/io/gcp/bigquery_tools.py b/sdks/python/apache_beam/io/gcp/bigquery_tools.py index 8dd58cd55a01..731a60b171a3 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery_tools.py +++ b/sdks/python/apache_beam/io/gcp/bigquery_tools.py @@ -125,6 +125,12 @@ "GEOGRAPHY": str, } +# Duplicated logic with io/gcp/bigquery_change_history.py +# Default table expiration for auto-created temp datasets: 24 hours in ms. +# Tables created in the dataset auto-expire after this duration if not +# explicitly deleted, acting as a safety net for orphaned temp tables +# (e.g. pipeline crash before cleanup runs). +_DEFAULT_TABLE_EXPIRATION_MS = 24 * 60 * 60 * 1000 class FileFormat(object): CSV = 'CSV' @@ -952,6 +958,7 @@ def create_temporary_dataset( project_id, self.temp_dataset_id, location=location, + default_table_expiration_ms=_DEFAULT_TABLE_EXPIRATION_MS, labels=labels, kms_key=kms_key)