diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 3d94e1f..0488117 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -14,17 +14,16 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 + - name: Set up uv + uses: astral-sh/setup-uv@v3 with: - python-version: '3.12' - cache: 'pip' + python-version: "3.13" - name: Install dependencies - run: pip install httpx python-dotenv + run: uv sync --frozen - name: Run sync script env: TOKEN: ${{ secrets.TOKEN }} PROFILE: ${{ secrets.PROFILE }} - run: python main.py + run: uv run python main.py diff --git a/README.md b/README.md index d15c930..1b99066 100644 --- a/README.md +++ b/README.md @@ -69,5 +69,5 @@ The included GitHub Actions workflow (`.github/workflows/ci.yml`) runs a dry-run - `PROFILE`: your Control D profile ID(s) ## Requirements -- Python 3.12+ +- Python 3.13+ - `uv` (for dependency management) diff --git a/main.py b/main.py index 5436ebe..32b008b 100644 --- a/main.py +++ b/main.py @@ -18,6 +18,7 @@ import os import logging import time +import concurrent.futures import re from typing import Dict, List, Optional, Any, Set, Sequence @@ -350,14 +351,31 @@ def sync_profile( try: # Fetch all folder data first folder_data_list = [] - for url in folder_urls: - if not validate_folder_url(url): - continue + + # Validate URLs first + valid_urls = [url for url in folder_urls if validate_folder_url(url)] + + invalid_count = len(folder_urls) - len(valid_urls) + if invalid_count > 0: + log.warning(f"Filtered out {invalid_count} invalid URL(s)") + + if not valid_urls: + log.error("No valid folder URLs to fetch") + return False + + def safe_fetch(url): try: - folder_data_list.append(fetch_folder_data(url)) + return fetch_folder_data(url) except (httpx.HTTPError, KeyError) as e: log.error(f"Failed to fetch folder data from {url}: {e}") - continue + return None + + # Fetch folder data in parallel to speed up startup + max_workers = min(10, len(valid_urls)) + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + results = executor.map(safe_fetch, valid_urls) + + folder_data_list = [r for r in results if r is not None] if not folder_data_list: log.error("No valid folder data found")