From e2071cfff8fedb0e48da37233d64c20f0adddc8e Mon Sep 17 00:00:00 2001 From: "MagicMock/mock.effective_git_name/140286118293392" Date: Sat, 13 Jun 2026 19:19:41 +0000 Subject: [PATCH] perf(backup): cut Azure container/list ops in rclone sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add --fast-list and --azureblob-no-check-container to both rclone backup command templates. Azure's "List and Create Container Operations" meter was the single largest line on shardbackupstorage (~3x the actual data-stored cost, ~€15/mo). Two causes: - Without --fast-list, rclone lists the destination hierarchically, issuing one List Blobs op per subdirectory. user_data trees are deep, so this scales with folder count x shards x nightly runs. --fast-list does a single recursive listing (~1 op per 5000 blobs) instead. - Without --azureblob-no-check-container, rclone issues a create/check container op on every invocation (2x per shard per night). The controller already creates the container when minting the SAS, so the check is pure waste. Read-side/listing change only; sync semantics unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- shard_core/service/backup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shard_core/service/backup.py b/shard_core/service/backup.py index b93f428..c792347 100644 --- a/shard_core/service/backup.py +++ b/shard_core/service/backup.py @@ -36,6 +36,7 @@ --crypt-remote :azureblob:{container_name} sync {directory} :crypt:{container_name}/{directory} --create-empty-src-dirs --stats-log-level NOTICE --stats 1000m --use-json-log +--fast-list --azureblob-no-check-container """ CLEARTEXT_COMMAND_TEMPLATE = """ @@ -43,6 +44,7 @@ --azureblob-sas-url {sas_token} sync {directory} :azureblob:{container_name}/{directory} --create-empty-src-dirs --stats-log-level NOTICE --stats 1000m --use-json-log +--fast-list --azureblob-no-check-container """