From 2153a758301b6c532e0a422254ad9081f0651d73 Mon Sep 17 00:00:00 2001 From: Abhay D Date: Mon, 12 Jan 2026 15:47:28 -0800 Subject: [PATCH] Fix race condition in maybe_download --- src/openpi/shared/download.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/openpi/shared/download.py b/src/openpi/shared/download.py index 2276507e6e..582f8392df 100644 --- a/src/openpi/shared/download.py +++ b/src/openpi/shared/download.py @@ -81,18 +81,19 @@ def maybe_download(url: str, *, force_download: bool = False, **kwargs) -> pathl else: local_path.unlink() - # Download the data to a local cache. - logger.info(f"Downloading {url} to {local_path}") - scratch_path = local_path.with_suffix(".partial") - # Route openpi-assets through gsutil to avoid gcsfs auth issues with this bucket. - # All other gs:// URLs (e.g. big_vision) continue to use gcsfs as normal. - if parsed.scheme == "gs" and parsed.netloc == "openpi-assets": - _download_gsutil(url, scratch_path, **kwargs) - else: - _download_fsspec(url, scratch_path, **kwargs) - - shutil.move(scratch_path, local_path) - _ensure_permissions(local_path) + if not local_path.exists(): + # Download the data to a local cache. + logger.info(f"Downloading {url} to {local_path}") + scratch_path = local_path.with_suffix(".partial") + # Route openpi-assets through gsutil to avoid gcsfs auth issues with this bucket. + # All other gs:// URLs (e.g. big_vision) continue to use gcsfs as normal. + if parsed.scheme == "gs" and parsed.netloc == "openpi-assets": + _download_gsutil(url, scratch_path, **kwargs) + else: + _download_fsspec(url, scratch_path, **kwargs) + + shutil.move(scratch_path, local_path) + _ensure_permissions(local_path) except PermissionError as e: msg = (