From 99d3806bf2d8378af779ab057ed0c90a78eb8bd8 Mon Sep 17 00:00:00 2001 From: Sina Zadeh <63080674+sinazadeh@users.noreply.github.com> Date: Tue, 29 Jul 2025 18:36:05 -0500 Subject: [PATCH 01/12] Update main.py --- main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/main.py b/main.py index 817d77b..aad38b8 100644 --- a/main.py +++ b/main.py @@ -45,16 +45,16 @@ # URLs of the JSON block-lists we want to import FOLDER_URLS = [ + "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/ultimate-known_issues-allow-folder.json", + "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/apple-private-relay-allow-folder.json", + "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/spam-tlds-allow-folder.json", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/badware-hoster-folder.json", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-amazon-folder.json", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-microsoft-folder.json", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-tiktok-aggressive-folder.json", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/referral-allow-folder.json", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/spam-idns-folder.json", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/spam-tlds-allow-folder.json", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/spam-tlds-folder.json", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/ultimate-known_issues-allow-folder.json", -] + ] BATCH_SIZE = 500 MAX_RETRIES = 3 From 3ea1b70af7a9c56a72c5626f069ed2d9b927a6ac Mon Sep 17 00:00:00 2001 From: Sina Zadeh Date: Sat, 2 Aug 2025 09:37:13 -0500 Subject: [PATCH 02/12] feat: enhance configuration options for folder syncing and add usage examples --- .env.example | 18 +++++- README.md | 15 ++++- USAGE_EXAMPLES.md | 34 +++++++++++ main.py | 152 ++++++++++++++++++++++++++++++++-------------- 4 files changed, 169 insertions(+), 50 deletions(-) create mode 100644 USAGE_EXAMPLES.md diff --git a/.env.example b/.env.example index b57f93d..7e7194e 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,16 @@ -TOKEN= -PROFILE= +# Control D API Configuration +TOKEN=your_api_token_here + +# Profile IDs to sync (comma-separated) +PROFILE=123456,789012 + +# Profile-specific folder configurations (optional) +# If not specified, default folders will be used for all profiles + +# Example: Profile 123456 uses only 3 specific folders +PROFILE_123456_FOLDERS=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/ultimate-known_issues-allow-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/apple-private-relay-allow-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/spam-tlds-allow-folder.json + +# Example: Profile 789012 uses different folders +PROFILE_789012_FOLDERS=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/badware-hoster-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-amazon-folder.json + +# If PROFILE__FOLDERS is not specified, the profile will use all default folders diff --git a/README.md b/README.md index 934d296..991f974 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,20 @@ https://controld.com/dashboard/profiles/741861frakbm/filters ``` 3. **Configure Folders** - Edit the `FOLDER_URLS` list in `main.py` to include the URLs of the JSON block-lists you want to sync. + + **Option 1: Use default folders for all profiles** + No additional configuration needed. All profiles will use the default folder URLs defined in `main.py`. + + **Option 2: Configure different folders for each profile** + Add profile-specific environment variables to your `.env` file: + ```py + # Profile-specific folder configurations (optional) + PROFILE_123456_FOLDERS=https://example.com/folder1.json,https://example.com/folder2.json + PROFILE_789012_FOLDERS=https://example.com/folder3.json,https://example.com/folder4.json + ``` + + **Option 3: Edit default folders** + Edit the `DEFAULT_FOLDER_URLS` list in `main.py` to change the default folders used when no profile-specific configuration is provided. > [!NOTE] > Currently only Folders with one action are supported. diff --git a/USAGE_EXAMPLES.md b/USAGE_EXAMPLES.md new file mode 100644 index 0000000..7ab9835 --- /dev/null +++ b/USAGE_EXAMPLES.md @@ -0,0 +1,34 @@ +# Example Usage Scenarios + +## Scenario 1: All profiles use the same default folders +```bash +TOKEN=your_token_here +PROFILE=123456,789012 +# No PROFILE_*_FOLDERS variables needed - all profiles will use default folders +``` + +## Scenario 2: Each profile has different folder sets +```bash +TOKEN=your_token_here +PROFILE=123456,789012 + +# Profile 123456 only syncs allow-lists +PROFILE_123456_FOLDERS=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/ultimate-known_issues-allow-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/apple-private-relay-allow-folder.json + +# Profile 789012 only syncs block-lists +PROFILE_789012_FOLDERS=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/badware-hoster-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-amazon-folder.json +``` + +## Scenario 3: Mixed configuration +```bash +TOKEN=your_token_here +PROFILE=123456,789012,345678 + +# Profile 123456 has custom folders +PROFILE_123456_FOLDERS=https://example.com/custom1.json,https://example.com/custom2.json + +# Profile 789012 has different custom folders +PROFILE_789012_FOLDERS=https://example.com/custom3.json + +# Profile 345678 has no PROFILE_*_FOLDERS variable, so it uses default folders +``` diff --git a/main.py b/main.py index aad38b8..19a0b99 100644 --- a/main.py +++ b/main.py @@ -10,6 +10,18 @@ 2. Deletes any existing folders with those names (so we start fresh). 3. Re-creates the folders and pushes all rules in batches. +Configuration: +- TOKEN: Your Control D API token +- PROFILE: Comma-separated list of profile IDs to sync +- PROFILE__FOLDERS: Comma-separated list of folder URLs for specific profile + (if not set, uses default folder URLs) + +Example environment variables: +TOKEN=your_api_token_here +PROFILE=123456,789012 +PROFILE_123456_FOLDERS=https://example.com/folder1.json,https://example.com/folder2.json +PROFILE_789012_FOLDERS=https://example.com/folder3.json + Nothing fancy, just works. """ @@ -43,18 +55,40 @@ # Accept either a single profile id or a comma-separated list PROFILE_IDS = [p.strip() for p in os.getenv("PROFILE", "").split(",") if p.strip()] -# URLs of the JSON block-lists we want to import -FOLDER_URLS = [ +# Default URLs of the JSON block-lists we want to import (used if no profile-specific config) +DEFAULT_FOLDER_URLS = [ "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/ultimate-known_issues-allow-folder.json", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/apple-private-relay-allow-folder.json", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/spam-tlds-allow-folder.json", + "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/referral-allow-folder.json", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/badware-hoster-folder.json", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-amazon-folder.json", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-microsoft-folder.json", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-tiktok-aggressive-folder.json", - "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/referral-allow-folder.json", "https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/spam-idns-folder.json", - ] +] + +# Profile-specific folder configuration +# Format: {profile_id: [list_of_folder_urls]} +PROFILE_FOLDERS = {} + +# Parse profile-specific folder configurations from environment variables +# Format: PROFILE__FOLDERS="url1,url2,url3" +for profile_id in PROFILE_IDS: + env_key = f"PROFILE_{profile_id}_FOLDERS" + folder_urls_str = os.getenv(env_key) + if folder_urls_str: + PROFILE_FOLDERS[profile_id] = [ + url.strip() for url in folder_urls_str.split(",") if url.strip() + ] + log.info( + f"Profile {profile_id}: using {len(PROFILE_FOLDERS[profile_id])} custom folder URLs" + ) + else: + PROFILE_FOLDERS[profile_id] = DEFAULT_FOLDER_URLS + log.info( + f"Profile {profile_id}: using {len(DEFAULT_FOLDER_URLS)} default folder URLs" + ) BATCH_SIZE = 500 MAX_RETRIES = 3 @@ -100,7 +134,13 @@ def _api_post(url: str, data: Dict) -> httpx.Response: def _api_post_form(url: str, data: Dict) -> httpx.Response: """POST helper for form data with retries.""" - return _retry_request(lambda: _api.post(url, data=data, headers={"Content-Type": "application/x-www-form-urlencoded"})) + return _retry_request( + lambda: _api.post( + url, + data=data, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + ) def _retry_request(request_func, max_retries=MAX_RETRIES, delay=RETRY_DELAY): @@ -113,11 +153,13 @@ def _retry_request(request_func, max_retries=MAX_RETRIES, delay=RETRY_DELAY): except (httpx.HTTPError, httpx.TimeoutException) as e: if attempt == max_retries - 1: # Log the response content if available - if hasattr(e, 'response') and e.response is not None: + if hasattr(e, "response") and e.response is not None: log.error(f"Response content: {e.response.text}") raise - wait_time = delay * (2 ** attempt) - log.warning(f"Request failed (attempt {attempt + 1}/{max_retries}): {e}. Retrying in {wait_time}s...") + wait_time = delay * (2**attempt) + log.warning( + f"Request failed (attempt {attempt + 1}/{max_retries}): {e}. Retrying in {wait_time}s..." + ) time.sleep(wait_time) @@ -148,7 +190,7 @@ def list_existing_folders(profile_id: str) -> Dict[str, str]: def get_all_existing_rules(profile_id: str) -> Set[str]: """Get all existing rules from all folders in the profile.""" all_rules = set() - + try: # Get rules from root folder (no folder_id) try: @@ -157,15 +199,15 @@ def get_all_existing_rules(profile_id: str) -> Set[str]: for rule in root_rules: if rule.get("PK"): all_rules.add(rule["PK"]) - + log.debug(f"Found {len(root_rules)} rules in root folder") - + except httpx.HTTPError as e: log.warning(f"Failed to get root folder rules: {e}") - + # Get all folders (including ones we're not managing) folders = list_existing_folders(profile_id) - + # Get rules from each folder for folder_name, folder_id in folders.items(): try: @@ -174,16 +216,16 @@ def get_all_existing_rules(profile_id: str) -> Set[str]: for rule in folder_rules: if rule.get("PK"): all_rules.add(rule["PK"]) - + log.debug(f"Found {len(folder_rules)} rules in folder '{folder_name}'") - + except httpx.HTTPError as e: log.warning(f"Failed to get rules from folder '{folder_name}': {e}") continue - + log.info(f"Total existing rules across all folders: {len(all_rules)}") return all_rules - + except Exception as e: log.error(f"Failed to get existing rules: {e}") return set() @@ -216,7 +258,7 @@ def create_folder(profile_id: str, name: str, do: int, status: int) -> Optional[ f"{API_BASE}/{profile_id}/groups", data={"name": name, "do": do, "status": status}, ) - + # Re-fetch the list and pick the folder we just created data = _api_get(f"{API_BASE}/{profile_id}/groups").json() for grp in data["body"]["groups"]: @@ -224,7 +266,7 @@ def create_folder(profile_id: str, name: str, do: int, status: int) -> Optional[ log.info("Created folder '%s' (ID %s)", name, grp["PK"]) time.sleep(FOLDER_CREATION_DELAY) return str(grp["PK"]) - + log.error(f"Folder '{name}' was not found after creation") return None except (httpx.HTTPError, KeyError) as e: @@ -245,34 +287,36 @@ def push_rules( if not hostnames: log.info("Folder '%s' - no rules to push", folder_name) return True - + # Filter out duplicates original_count = len(hostnames) filtered_hostnames = [h for h in hostnames if h not in existing_rules] duplicates_count = original_count - len(filtered_hostnames) - + if duplicates_count > 0: log.info(f"Folder '{folder_name}': skipping {duplicates_count} duplicate rules") - + if not filtered_hostnames: - log.info(f"Folder '{folder_name}' - no new rules to push after filtering duplicates") + log.info( + f"Folder '{folder_name}' - no new rules to push after filtering duplicates" + ) return True - + successful_batches = 0 total_batches = len(range(0, len(filtered_hostnames), BATCH_SIZE)) - + for i, start in enumerate(range(0, len(filtered_hostnames), BATCH_SIZE), 1): batch = filtered_hostnames[start : start + BATCH_SIZE] - + data = { "do": str(do), "status": str(status), "group": str(folder_id), } - + for j, hostname in enumerate(batch): data[f"hostnames[{j}]"] = hostname - + try: _api_post_form( f"{API_BASE}/{profile_id}/rules", @@ -285,20 +329,26 @@ def push_rules( len(batch), ) successful_batches += 1 - + # Update existing_rules set with the newly added rules existing_rules.update(batch) - + except httpx.HTTPError as e: log.error(f"Failed to push batch {i} for folder '{folder_name}': {e}") - if hasattr(e, 'response') and e.response is not None: + if hasattr(e, "response") and e.response is not None: log.error(f"Response content: {e.response.text}") - + if successful_batches == total_batches: - log.info("Folder '%s' – finished (%d new rules added)", folder_name, len(filtered_hostnames)) + log.info( + "Folder '%s' – finished (%d new rules added)", + folder_name, + len(filtered_hostnames), + ) return True else: - log.error(f"Folder '%s' – only {successful_batches}/{total_batches} batches succeeded") + log.error( + f"Folder '%s' – only {successful_batches}/{total_batches} batches succeeded" + ) return False @@ -308,29 +358,33 @@ def push_rules( def sync_profile(profile_id: str) -> bool: """One-shot sync: delete old, create new, push rules. Returns True if successful.""" try: + # Get folder URLs for this specific profile + folder_urls = PROFILE_FOLDERS.get(profile_id, DEFAULT_FOLDER_URLS) + log.info(f"Profile {profile_id}: syncing {len(folder_urls)} folders") + # Fetch all folder data first folder_data_list = [] - for url in FOLDER_URLS: + for url in folder_urls: try: folder_data_list.append(fetch_folder_data(url)) except (httpx.HTTPError, KeyError) as e: log.error(f"Failed to fetch folder data from {url}: {e}") continue - + if not folder_data_list: log.error("No valid folder data found") return False - + # Get existing folders and delete target folders existing_folders = list_existing_folders(profile_id) for folder_data in folder_data_list: name = folder_data["group"]["group"].strip() if name in existing_folders: delete_folder(profile_id, name, existing_folders[name]) - + # Get all existing rules AFTER deleting target folders existing_rules = get_all_existing_rules(profile_id) - + # Create new folders and push rules success_count = 0 for folder_data in folder_data_list: @@ -339,18 +393,22 @@ def sync_profile(profile_id: str) -> bool: do = grp["action"]["do"] status = grp["action"]["status"] hostnames = [r["PK"] for r in folder_data.get("rules", []) if r.get("PK")] - + folder_id = create_folder(profile_id, name, do, status) - if folder_id and push_rules(profile_id, name, folder_id, do, status, hostnames, existing_rules): + if folder_id and push_rules( + profile_id, name, folder_id, do, status, hostnames, existing_rules + ): success_count += 1 # Note: existing_rules is updated within push_rules function - + # Optional: Refresh existing rules after each folder (more thorough but slower) # existing_rules = get_all_existing_rules(profile_id) - - log.info(f"Sync complete: {success_count}/{len(folder_data_list)} folders processed successfully") + + log.info( + f"Sync complete: {success_count}/{len(folder_data_list)} folders processed successfully" + ) return success_count == len(folder_data_list) - + except Exception as e: log.error(f"Unexpected error during sync for profile {profile_id}: {e}") return False @@ -363,13 +421,13 @@ def main(): if not TOKEN or not PROFILE_IDS: log.error("TOKEN and/or PROFILE missing - check your .env file") exit(1) - + success_count = 0 for profile_id in PROFILE_IDS: log.info("Starting sync for profile %s", profile_id) if sync_profile(profile_id): success_count += 1 - + log.info(f"All profiles processed: {success_count}/{len(PROFILE_IDS)} successful") exit(0 if success_count == len(PROFILE_IDS) else 1) From 083f14f6407d8b854584969f9b1f07face3b411d Mon Sep 17 00:00:00 2001 From: Sina Zadeh Date: Sat, 2 Aug 2025 09:48:30 -0500 Subject: [PATCH 03/12] feat: update environment variable usage and examples for indexed profile configurations --- .env.example | 14 +++---- .github/workflows/sync.yml | 4 +- USAGE_EXAMPLES.md | 26 ++++++------- main.py | 75 +++++++++++++++++++++++++++----------- 4 files changed, 77 insertions(+), 42 deletions(-) diff --git a/.env.example b/.env.example index 7e7194e..9b199c2 100644 --- a/.env.example +++ b/.env.example @@ -2,15 +2,15 @@ TOKEN=your_api_token_here # Profile IDs to sync (comma-separated) -PROFILE=123456,789012 +PROFILE= # Profile-specific folder configurations (optional) -# If not specified, default folders will be used for all profiles +# Use indexed approach: PROFILE_0_FOLDERS for first profile, PROFILE_1_FOLDERS for second, etc. -# Example: Profile 123456 uses only 3 specific folders -PROFILE_123456_FOLDERS=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/ultimate-known_issues-allow-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/apple-private-relay-allow-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/spam-tlds-allow-folder.json +# Profile 0 (first profile in PROFILE list) uses allow-list folders +PROFILE_0_FOLDERS=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/ultimate-known_issues-allow-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/apple-private-relay-allow-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/spam-tlds-allow-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/referral-allow-folder.json -# Example: Profile 789012 uses different folders -PROFILE_789012_FOLDERS=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/badware-hoster-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-amazon-folder.json +# Profile 1 (second profile in PROFILE list) uses block-list folders +PROFILE_1_FOLDERS=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/badware-hoster-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-amazon-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-microsoft-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-tiktok-aggressive-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/spam-idns-folder.json -# If PROFILE__FOLDERS is not specified, the profile will use all default folders +# If PROFILE_X_FOLDERS is not specified, that profile will use all default folders diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 511ed6d..9432193 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -27,4 +27,6 @@ jobs: env: TOKEN: ${{ secrets.TOKEN }} PROFILE: ${{ secrets.PROFILE }} - run: uv run python main.py \ No newline at end of file + PROFILE_0_FOLDERS: ${{ secrets.PROFILE_0_FOLDERS }} + PROFILE_1_FOLDERS: ${{ secrets.PROFILE_1_FOLDERS }} + run: uv run python main.py diff --git a/USAGE_EXAMPLES.md b/USAGE_EXAMPLES.md index 7ab9835..fbedfb8 100644 --- a/USAGE_EXAMPLES.md +++ b/USAGE_EXAMPLES.md @@ -3,32 +3,32 @@ ## Scenario 1: All profiles use the same default folders ```bash TOKEN=your_token_here -PROFILE=123456,789012 +PROFILE=profile1,profile2 # No PROFILE_*_FOLDERS variables needed - all profiles will use default folders ``` -## Scenario 2: Each profile has different folder sets +## Scenario 2: Each profile has different folder sets (indexed approach) ```bash TOKEN=your_token_here -PROFILE=123456,789012 +PROFILE=profile1,profile2 -# Profile 123456 only syncs allow-lists -PROFILE_123456_FOLDERS=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/ultimate-known_issues-allow-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/apple-private-relay-allow-folder.json +# Profile 0 (first profile) only syncs allow-lists +PROFILE_0_FOLDERS=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/ultimate-known_issues-allow-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/apple-private-relay-allow-folder.json -# Profile 789012 only syncs block-lists -PROFILE_789012_FOLDERS=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/badware-hoster-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-amazon-folder.json +# Profile 1 (second profile) only syncs block-lists +PROFILE_1_FOLDERS=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/badware-hoster-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-amazon-folder.json ``` ## Scenario 3: Mixed configuration ```bash TOKEN=your_token_here -PROFILE=123456,789012,345678 +PROFILE=profile1,profile2,profile3 -# Profile 123456 has custom folders -PROFILE_123456_FOLDERS=https://example.com/custom1.json,https://example.com/custom2.json +# Profile 0 has custom folders +PROFILE_0_FOLDERS=https://example.com/custom1.json,https://example.com/custom2.json -# Profile 789012 has different custom folders -PROFILE_789012_FOLDERS=https://example.com/custom3.json +# Profile 1 has different custom folders +PROFILE_1_FOLDERS=https://example.com/custom3.json -# Profile 345678 has no PROFILE_*_FOLDERS variable, so it uses default folders +# Profile 2 has no PROFILE_2_FOLDERS variable, so it uses default folders ``` diff --git a/main.py b/main.py index 19a0b99..b6b9dcd 100644 --- a/main.py +++ b/main.py @@ -13,14 +13,14 @@ Configuration: - TOKEN: Your Control D API token - PROFILE: Comma-separated list of profile IDs to sync -- PROFILE__FOLDERS: Comma-separated list of folder URLs for specific profile - (if not set, uses default folder URLs) +- PROFILE_X_FOLDERS: Comma-separated list of folder URLs for profile at index X + (X=0 for first profile, X=1 for second, etc. If not set, uses default folder URLs) Example environment variables: TOKEN=your_api_token_here -PROFILE=123456,789012 -PROFILE_123456_FOLDERS=https://example.com/folder1.json,https://example.com/folder2.json -PROFILE_789012_FOLDERS=https://example.com/folder3.json +PROFILE=first_profile_id,second_profile_id +PROFILE_0_FOLDERS=https://example.com/folder1.json,https://example.com/folder2.json +PROFILE_1_FOLDERS=https://example.com/folder3.json Nothing fancy, just works. """ @@ -38,8 +38,12 @@ # --------------------------------------------------------------------------- # load_dotenv() +# Enable debug logging if DEBUG environment variable is set +debug_mode = os.getenv("DEBUG", "").lower() in ("1", "true", "yes") +log_level = logging.DEBUG if debug_mode else logging.INFO + logging.basicConfig( - level=logging.INFO, + level=log_level, format="%(asctime)s | %(levelname)-8s | %(message)s", datefmt="%H:%M:%S", ) @@ -72,23 +76,37 @@ # Format: {profile_id: [list_of_folder_urls]} PROFILE_FOLDERS = {} -# Parse profile-specific folder configurations from environment variables -# Format: PROFILE__FOLDERS="url1,url2,url3" -for profile_id in PROFILE_IDS: - env_key = f"PROFILE_{profile_id}_FOLDERS" + +def get_profile_folders(profile_index: int) -> List[str]: + """Get folder URLs for a specific profile by index, with fallback to defaults.""" + env_key = f"PROFILE_{profile_index}_FOLDERS" folder_urls_str = os.getenv(env_key) + + log.debug(f"Looking for environment variable: {env_key}") + log.debug(f"Found value: {folder_urls_str}") + if folder_urls_str: - PROFILE_FOLDERS[profile_id] = [ - url.strip() for url in folder_urls_str.split(",") if url.strip() - ] - log.info( - f"Profile {profile_id}: using {len(PROFILE_FOLDERS[profile_id])} custom folder URLs" - ) + urls = [url.strip() for url in folder_urls_str.split(",") if url.strip()] + log.info(f"Profile {profile_index + 1}: using {len(urls)} custom folder URLs") + log.debug(f"Profile {profile_index + 1} custom URLs: {urls}") + return urls else: - PROFILE_FOLDERS[profile_id] = DEFAULT_FOLDER_URLS log.info( - f"Profile {profile_id}: using {len(DEFAULT_FOLDER_URLS)} default folder URLs" + f"Profile {profile_index + 1}: no custom folders found, using {len(DEFAULT_FOLDER_URLS)} default folder URLs" ) + log.debug(f"Profile {profile_index + 1} default URLs: {DEFAULT_FOLDER_URLS}") + return DEFAULT_FOLDER_URLS + + +# Initialize profile folders for each profile ID using index-based lookup +for i, profile_id in enumerate(PROFILE_IDS): + PROFILE_FOLDERS[profile_id] = get_profile_folders(i) + +# Debug: Print all environment variables that start with PROFILE_ +log.debug("All PROFILE_* environment variables:") +for key, value in os.environ.items(): + if key.startswith("PROFILE_"): + log.debug(f" {key}={value}") BATCH_SIZE = 500 MAX_RETRIES = 3 @@ -358,9 +376,9 @@ def push_rules( def sync_profile(profile_id: str) -> bool: """One-shot sync: delete old, create new, push rules. Returns True if successful.""" try: - # Get folder URLs for this specific profile + # Get folder URLs for this specific profile using the cached lookup folder_urls = PROFILE_FOLDERS.get(profile_id, DEFAULT_FOLDER_URLS) - log.info(f"Profile {profile_id}: syncing {len(folder_urls)} folders") + log.info(f"Profile: syncing {len(folder_urls)} folders") # Fetch all folder data first folder_data_list = [] @@ -422,9 +440,24 @@ def main(): log.error("TOKEN and/or PROFILE missing - check your .env file") exit(1) + log.info(f"Found {len(PROFILE_IDS)} profiles") + + # Debug: Show configuration for each profile + for i, profile_id in enumerate(PROFILE_IDS): + env_key = f"PROFILE_{i}_FOLDERS" + env_value = os.getenv(env_key) + if env_value: + log.info( + f"Profile {i + 1}: Found custom configuration with {len(env_value.split(','))} folders" + ) + else: + log.info( + f"Profile {i + 1}: No custom configuration found, will use defaults" + ) + success_count = 0 for profile_id in PROFILE_IDS: - log.info("Starting sync for profile %s", profile_id) + log.info("Starting sync for profile") if sync_profile(profile_id): success_count += 1 From d0bcedb6f235ff389a3fe234e28809ef11bd5279 Mon Sep 17 00:00:00 2001 From: Sina Zadeh Date: Sat, 2 Aug 2025 09:52:53 -0500 Subject: [PATCH 04/12] fix: improve URL cleaning in profile folder retrieval and enhance logging --- main.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index b6b9dcd..8ebe18f 100644 --- a/main.py +++ b/main.py @@ -86,7 +86,13 @@ def get_profile_folders(profile_index: int) -> List[str]: log.debug(f"Found value: {folder_urls_str}") if folder_urls_str: - urls = [url.strip() for url in folder_urls_str.split(",") if url.strip()] + # Split by comma and clean each URL (strip whitespace and quotes) + urls = [] + for url in folder_urls_str.split(","): + cleaned_url = url.strip().strip('"').strip("'") + if cleaned_url: + urls.append(cleaned_url) + log.info(f"Profile {profile_index + 1}: using {len(urls)} custom folder URLs") log.debug(f"Profile {profile_index + 1} custom URLs: {urls}") return urls @@ -384,9 +390,10 @@ def sync_profile(profile_id: str) -> bool: folder_data_list = [] for url in folder_urls: try: + log.debug(f"Fetching folder data from: '{url}'") folder_data_list.append(fetch_folder_data(url)) except (httpx.HTTPError, KeyError) as e: - log.error(f"Failed to fetch folder data from {url}: {e}") + log.error(f"Failed to fetch folder data from '{url}': {e}") continue if not folder_data_list: From 0784291c5591e05334f5bb65936a64aaec0a1fbe Mon Sep 17 00:00:00 2001 From: Sina Zadeh Date: Sat, 2 Aug 2025 09:54:34 -0500 Subject: [PATCH 05/12] fix: add logging for cleaned URLs in profile folder retrieval --- main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/main.py b/main.py index 8ebe18f..b955800 100644 --- a/main.py +++ b/main.py @@ -95,6 +95,11 @@ def get_profile_folders(profile_index: int) -> List[str]: log.info(f"Profile {profile_index + 1}: using {len(urls)} custom folder URLs") log.debug(f"Profile {profile_index + 1} custom URLs: {urls}") + + # Log each cleaned URL for debugging + for idx, cleaned_url in enumerate(urls): + log.debug(f"Cleaned URL {idx + 1}: {cleaned_url}") + return urls else: log.info( From c100c80f5c2b6912f1f9d99fa35ad70efd0bd524 Mon Sep 17 00:00:00 2001 From: Sina Zadeh Date: Sat, 2 Aug 2025 09:57:12 -0500 Subject: [PATCH 06/12] fix: remove outer quotes from folder URLs string in profile folder retrieval --- main.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/main.py b/main.py index b955800..52640db 100644 --- a/main.py +++ b/main.py @@ -86,6 +86,12 @@ def get_profile_folders(profile_index: int) -> List[str]: log.debug(f"Found value: {folder_urls_str}") if folder_urls_str: + # Remove outer quotes from the entire string if present + if folder_urls_str.startswith(('"', "'")) and folder_urls_str.endswith( + ('"', "'") + ): + folder_urls_str = folder_urls_str[1:-1] + # Split by comma and clean each URL (strip whitespace and quotes) urls = [] for url in folder_urls_str.split(","): From 83926a4a0839fc47d0a66342ae5bf20be5d53ffd Mon Sep 17 00:00:00 2001 From: Sina Zadeh Date: Sat, 2 Aug 2025 09:59:12 -0500 Subject: [PATCH 07/12] fix: enhance URL cleaning by removing all quotes in profile folder retrieval --- main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 52640db..f7e58c8 100644 --- a/main.py +++ b/main.py @@ -95,7 +95,8 @@ def get_profile_folders(profile_index: int) -> List[str]: # Split by comma and clean each URL (strip whitespace and quotes) urls = [] for url in folder_urls_str.split(","): - cleaned_url = url.strip().strip('"').strip("'") + # Be more aggressive with cleaning quotes + cleaned_url = url.strip().replace('"', "").replace("'", "") if cleaned_url: urls.append(cleaned_url) From 7056c287a0f2f94c906a12e04c82949d3c265039 Mon Sep 17 00:00:00 2001 From: Sina Zadeh Date: Sat, 2 Aug 2025 10:01:25 -0500 Subject: [PATCH 08/12] fix: enhance URL cleaning by removing outer quotes and improving string handling in profile folder retrieval --- main.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/main.py b/main.py index f7e58c8..6e755e0 100644 --- a/main.py +++ b/main.py @@ -86,16 +86,9 @@ def get_profile_folders(profile_index: int) -> List[str]: log.debug(f"Found value: {folder_urls_str}") if folder_urls_str: - # Remove outer quotes from the entire string if present - if folder_urls_str.startswith(('"', "'")) and folder_urls_str.endswith( - ('"', "'") - ): - folder_urls_str = folder_urls_str[1:-1] - - # Split by comma and clean each URL (strip whitespace and quotes) + # Split by comma and clean each URL (strip whitespace and all quotes) urls = [] for url in folder_urls_str.split(","): - # Be more aggressive with cleaning quotes cleaned_url = url.strip().replace('"', "").replace("'", "") if cleaned_url: urls.append(cleaned_url) From e25584a926ad075b4de4fb6347eba8634261bfb3 Mon Sep 17 00:00:00 2001 From: Sina Zadeh Date: Sat, 2 Aug 2025 10:03:02 -0500 Subject: [PATCH 09/12] fix: clean URLs before fetching folder data in sync_profile function --- main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 6e755e0..452b568 100644 --- a/main.py +++ b/main.py @@ -395,8 +395,10 @@ def sync_profile(profile_id: str) -> bool: folder_data_list = [] for url in folder_urls: try: - log.debug(f"Fetching folder data from: '{url}'") - folder_data_list.append(fetch_folder_data(url)) + # Clean the URL again right before using it + cleaned_url = url.strip().replace('"', "").replace("'", "") + log.debug(f"Fetching folder data from: '{cleaned_url}'") + folder_data_list.append(fetch_folder_data(cleaned_url)) except (httpx.HTTPError, KeyError) as e: log.error(f"Failed to fetch folder data from '{url}': {e}") continue From 7de0b88b032344e9b6dda7d3c5d4c9482f1bb520 Mon Sep 17 00:00:00 2001 From: Sina Zadeh Date: Sat, 2 Aug 2025 10:15:49 -0500 Subject: [PATCH 10/12] docs: update .env.example and README.md for improved profile folder configuration instructions --- .env.example | 40 +++++++++++++++++++++++++++++++--------- README.md | 14 ++++++++++---- USAGE_EXAMPLES.md | 40 +++++++++++++++++++++++++++++----------- 3 files changed, 70 insertions(+), 24 deletions(-) diff --git a/.env.example b/.env.example index 9b199c2..95a6db5 100644 --- a/.env.example +++ b/.env.example @@ -1,16 +1,38 @@ -# Control D API Configuration +# --------------------------------------------------------------------------- # +# Control D Sync Configuration +# +# Create a copy of this file named .env and fill in your values. +# These variables can also be set as GitHub repository secrets for CI. +# --------------------------------------------------------------------------- # + +# Your Control D API token. +# Required. TOKEN=your_api_token_here -# Profile IDs to sync (comma-separated) +# A comma-separated list of the profile IDs you want to sync. +# Required. Example: PROFILE=abc123def,xyz789ghi PROFILE= -# Profile-specific folder configurations (optional) -# Use indexed approach: PROFILE_0_FOLDERS for first profile, PROFILE_1_FOLDERS for second, etc. +# --------------------------------------------------------------------------- # +# Profile-specific Folder URLs (Optional) +# +# By default, all profiles use the `DEFAULT_FOLDER_URLS` from `main.py`. +# You can override this for specific profiles using the indexed approach below. +# The index corresponds to the order of profile IDs in your `PROFILE` list. +# +# If a `PROFILE_X_FOLDERS` variable is not set for a profile, it will use +# the default folders. +# --------------------------------------------------------------------------- # + +# --- Example --- +# This example assumes: PROFILE=profile_A_id,profile_B_id + +# For the first profile ('profile_A_id'), use a custom list of allow-list folders. +# PROFILE_0_FOLDERS="https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/ultimate-known_issues-allow-folder.json","https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/apple-private-relay-allow-folder.json" -# Profile 0 (first profile in PROFILE list) uses allow-list folders -PROFILE_0_FOLDERS=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/ultimate-known_issues-allow-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/apple-private-relay-allow-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/spam-tlds-allow-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/referral-allow-folder.json +# For the second profile ('profile_B_id'), use a custom list of block-list folders. +# PROFILE_1_FOLDERS="https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/badware-hoster-folder.json","https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-amazon-folder.json" -# Profile 1 (second profile in PROFILE list) uses block-list folders -PROFILE_1_FOLDERS=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/badware-hoster-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-amazon-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-microsoft-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-tiktok-aggressive-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/spam-idns-folder.json +# A third profile in the list would use the default folders, since +# `PROFILE_2_FOLDERS` is not defined. -# If PROFILE_X_FOLDERS is not specified, that profile will use all default folders diff --git a/README.md b/README.md index 991f974..4d03237 100644 --- a/README.md +++ b/README.md @@ -49,11 +49,17 @@ https://controld.com/dashboard/profiles/741861frakbm/filters No additional configuration needed. All profiles will use the default folder URLs defined in `main.py`. **Option 2: Configure different folders for each profile** - Add profile-specific environment variables to your `.env` file: + Add profile-specific environment variables to your `.env` file using the profile's index (starting from 0). For the first profile ID in your `PROFILE` list, use `PROFILE_0_FOLDERS`, for the second use `PROFILE_1_FOLDERS`, and so on. + + Example: ```py - # Profile-specific folder configurations (optional) - PROFILE_123456_FOLDERS=https://example.com/folder1.json,https://example.com/folder2.json - PROFILE_789012_FOLDERS=https://example.com/folder3.json,https://example.com/folder4.json + # .env + PROFILE=first_profile_id,second_profile_id + + # Corresponds to first_profile_id + PROFILE_0_FOLDERS=https://example.com/folder1.json,https://example.com/folder2.json + # Corresponds to second_profile_id + PROFILE_1_FOLDERS=https://example.com/folder3.json,https://example.com/folder4.json ``` **Option 3: Edit default folders** diff --git a/USAGE_EXAMPLES.md b/USAGE_EXAMPLES.md index fbedfb8..2709318 100644 --- a/USAGE_EXAMPLES.md +++ b/USAGE_EXAMPLES.md @@ -1,34 +1,52 @@ # Example Usage Scenarios +This file shows how to configure your `.env` file or GitHub secrets for different use cases. + ## Scenario 1: All profiles use the same default folders + +In this setup, all profiles listed in the `PROFILE` variable will be synced with the `DEFAULT_FOLDER_URLS` defined in `main.py`. + ```bash +# .env file or GitHub repository secrets TOKEN=your_token_here -PROFILE=profile1,profile2 -# No PROFILE_*_FOLDERS variables needed - all profiles will use default folders +PROFILE=abc123def,xyz789ghi + +# No PROFILE_*_FOLDERS variables are set. +# Both profiles will use the default folder URLs. ``` -## Scenario 2: Each profile has different folder sets (indexed approach) +## Scenario 2: All profiles use custom folder lists + +In this case, you want to define a specific list of folders for **every** profile. You must provide a `PROFILE_X_FOLDERS` variable for each profile listed in `PROFILE`. + ```bash +# .env file or GitHub repository secrets TOKEN=your_token_here -PROFILE=profile1,profile2 +PROFILE=abc123def,xyz789ghi -# Profile 0 (first profile) only syncs allow-lists +# For profile 'abc123def' (index 0) PROFILE_0_FOLDERS=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/ultimate-known_issues-allow-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/apple-private-relay-allow-folder.json -# Profile 1 (second profile) only syncs block-lists +# For profile 'xyz789ghi' (index 1) PROFILE_1_FOLDERS=https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/badware-hoster-folder.json,https://raw.githubusercontent.com/hagezi/dns-blocklists/main/controld/native-tracker-amazon-folder.json ``` -## Scenario 3: Mixed configuration +## Scenario 3: Some profiles use custom lists, others use defaults + +This is a hybrid approach. You can define custom folder lists for specific profiles, while letting any others that are not explicitly defined fall back to the `DEFAULT_FOLDER_URLS`. + ```bash +# .env file or GitHub repository secrets TOKEN=your_token_here -PROFILE=profile1,profile2,profile3 +PROFILE=abc123def,xyz789ghi,jkl456mno -# Profile 0 has custom folders +# For profile 'abc123def' (index 0) PROFILE_0_FOLDERS=https://example.com/custom1.json,https://example.com/custom2.json -# Profile 1 has different custom folders +# For profile 'xyz789ghi' (index 1) PROFILE_1_FOLDERS=https://example.com/custom3.json -# Profile 2 has no PROFILE_2_FOLDERS variable, so it uses default folders +# Profile 'jkl456mno' (index 2) has no PROFILE_2_FOLDERS variable, +# so it will fall back to using the default folder URLs. ``` + From 57b9844ff70f9c859942eb250feb4dbebb1b2100 Mon Sep 17 00:00:00 2001 From: Sina Zadeh Date: Sat, 2 Aug 2025 11:44:35 -0500 Subject: [PATCH 11/12] fix: improve logging for folder data fetching and profile sync status --- main.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/main.py b/main.py index 452b568..091bab6 100644 --- a/main.py +++ b/main.py @@ -395,10 +395,8 @@ def sync_profile(profile_id: str) -> bool: folder_data_list = [] for url in folder_urls: try: - # Clean the URL again right before using it - cleaned_url = url.strip().replace('"', "").replace("'", "") - log.debug(f"Fetching folder data from: '{cleaned_url}'") - folder_data_list.append(fetch_folder_data(cleaned_url)) + log.debug(f"Fetching folder data from: '{url}'") + folder_data_list.append(fetch_folder_data(url)) except (httpx.HTTPError, KeyError) as e: log.error(f"Failed to fetch folder data from '{url}': {e}") continue @@ -462,16 +460,16 @@ def main(): env_value = os.getenv(env_key) if env_value: log.info( - f"Profile {i + 1}: Found custom configuration with {len(env_value.split(','))} folders" + f"Profile {i + 1} ({profile_id}): Found custom configuration with {len(env_value.split(','))} folders" ) else: log.info( - f"Profile {i + 1}: No custom configuration found, will use defaults" + f"Profile {i + 1} ({profile_id}): No custom configuration found, will use defaults" ) success_count = 0 for profile_id in PROFILE_IDS: - log.info("Starting sync for profile") + log.info(f"Starting sync for profile {profile_id}") if sync_profile(profile_id): success_count += 1 From bbc4567aab0c05bcfedc40f71b3a4d2180b2eeca Mon Sep 17 00:00:00 2001 From: Sina Zadeh Date: Fri, 17 Oct 2025 11:20:39 -0500 Subject: [PATCH 12/12] fix: update sync workflow to use repository variables for non-sensitive configuration --- .github/workflows/sync.yml | 10 +++++-- README.md | 60 ++++++++++++++++++++++++++------------ 2 files changed, 49 insertions(+), 21 deletions(-) diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 9432193..2b61f09 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -26,7 +26,11 @@ jobs: - name: Run sync script env: TOKEN: ${{ secrets.TOKEN }} - PROFILE: ${{ secrets.PROFILE }} - PROFILE_0_FOLDERS: ${{ secrets.PROFILE_0_FOLDERS }} - PROFILE_1_FOLDERS: ${{ secrets.PROFILE_1_FOLDERS }} + # PROFILE and folder lists are non-sensitive configuration values. + # Store them as repository/organization variables and access via vars. + PROFILE: ${{ vars.PROFILE }} + PROFILE_0_FOLDERS: ${{ vars.PROFILE_0_FOLDERS }} + PROFILE_1_FOLDERS: ${{ vars.PROFILE_1_FOLDERS }} + PROFILE_2_FOLDERS: ${{ vars.PROFILE_2_FOLDERS }} + PROFILE_3_FOLDERS: ${{ vars.PROFILE_3_FOLDERS }} run: uv run python main.py diff --git a/README.md b/README.md index 4d03237..5b4b903 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ A tiny Python script that keeps your Control D Folders in sync with a set of remote block-lists. ## What it does + 1. Downloads the current JSON block-lists. 2. Deletes any existing folders with the same names. 3. Re-creates the folders and pushes all rules in batches. @@ -22,6 +23,7 @@ remote block-lists. 1. Log in to your Control D account. 2. Open the Profile you want to sync. 3. Copy the profile ID from the URL. + ``` https://controld.com/dashboard/profiles/741861frakbm/filters ^^^^^^^^^^^^ @@ -30,28 +32,43 @@ https://controld.com/dashboard/profiles/741861frakbm/filters ### Configure the script 1. **Clone & install** + ```bash git clone https://github.com/your-username/ctrld-sync.git cd ctrld-sync uv sync ``` -2. **Configure secrets** - Create a `.env` file (or set GitHub secrets) with: - ```py - TOKEN=your_control_d_api_token - PROFILE=your_profile_id # or comma-separated list of profile ids (e.g. your_id_1,your_id_2) - ``` +2. **Configure secrets & variables** + + Create a `.env` file for local runs or configure GitHub for CI. Use Secrets for sensitive values and Repository/Organization Variables for non-sensitive configuration: + - Sensitive (store as GitHub Secrets): + + ```text + TOKEN=your_control_d_api_token + ``` + + - Non-sensitive (store as GitHub Repository/Organization Variables or in `.env` for local runs): + ```text + PROFILE=your_profile_id # or comma-separated list of profile ids (e.g. your_id_1,your_id_2) + PROFILE_0_FOLDERS=https://example.com/folder1.json,https://example.com/folder2.json + PROFILE_1_FOLDERS=https://example.com/folder3.json,https://example.com/folder4.json + ``` + + Notes: + - The workflow reads `${{ secrets.TOKEN }}` for the API token and `${{ vars.PROFILE }}`, `${{ vars.PROFILE_0_FOLDERS }}`, etc. for non-sensitive variables. + - If you prefer all-local configuration, a `.env` file with the same names will work for local runs (but do NOT commit `.env` to the repo). + +3. **Configure Folders** -3. **Configure Folders** - **Option 1: Use default folders for all profiles** No additional configuration needed. All profiles will use the default folder URLs defined in `main.py`. - + **Option 2: Configure different folders for each profile** Add profile-specific environment variables to your `.env` file using the profile's index (starting from 0). For the first profile ID in your `PROFILE` list, use `PROFILE_0_FOLDERS`, for the second use `PROFILE_1_FOLDERS`, and so on. - + Example: + ```py # .env PROFILE=first_profile_id,second_profile_id @@ -61,7 +78,7 @@ https://controld.com/dashboard/profiles/741861frakbm/filters # Corresponds to second_profile_id PROFILE_1_FOLDERS=https://example.com/folder3.json,https://example.com/folder4.json ``` - + **Option 3: Edit default folders** Edit the `DEFAULT_FOLDER_URLS` list in `main.py` to change the default folders used when no profile-specific configuration is provided. @@ -70,6 +87,7 @@ https://controld.com/dashboard/profiles/741861frakbm/filters > Either "Block" or "Allow" actions are supported. 4. **Run locally** + ```bash uv run python main.py ``` @@ -80,12 +98,18 @@ https://controld.com/dashboard/profiles/741861frakbm/filters ### Configure GitHub Actions 1. Fork this repo. -2. Go to the "Actions" Tab and enable actions. -3. Go to the Repo Settings. -4. Under "Secrets and variables > Actions" create the following secrets like above, under "Repository secrets": - - `TOKEN`: your Control D API token - - `PROFILE`: your Control D profile ID(s) +2. Go to the "Actions" tab and enable actions. +3. Go to the repository Settings → "Secrets and variables" → "Actions". +4. Under the "Secrets" tab, add: + - `TOKEN` (your Control D API token) + +5. Under the "Variables" tab, add non-sensitive configuration values: + - `PROFILE` (one or more profile IDs, comma-separated) + - `PROFILE_0_FOLDERS`, `PROFILE_1_FOLDERS`, ... (comma-separated folder URLs for each profile index) + +6. The workflow will read `TOKEN` from secrets and the others from variables. This prevents accidental leakage of tokens while keeping configuration editable. ## Requirements -- Python 3.12+ -- `uv` (for dependency management) \ No newline at end of file + +- Python 3.12+ +- `uv` (for dependency management)