Add cache TTL and --clear-cache flag to persistent blocklist cache

main.py

@@ -661,6 +661,7 @@
 # --------------------------------------------------------------------------- #
 # Disk cache stores validated blocklist data with HTTP cache headers (ETag, Last-Modified)
 # to enable fast cold-start syncs via conditional HTTP requests (304 Not Modified)
+CACHE_TTL_SECONDS = 24 * 60 * 60  # 24 hours: within TTL, serve from disk without HTTP request
 _disk_cache: Dict[str, Dict[str, Any]] = {}  # Loaded from disk on startup
 _cache_stats = {"hits": 0, "misses": 0, "validations": 0, "errors": 0}
 _api_stats = {"control_d_api_calls": 0, "blocklist_fetches": 0}
@@ -1258,11 +1259,20 @@
     with _cache_lock:
         _api_stats["blocklist_fetches"] += 1
 
-    # Check disk cache for conditional request headers
+    # Check disk cache for TTL-based hit or conditional request headers
     headers = {}
     cached_entry = _disk_cache.get(url)
     if cached_entry:
-        # Send conditional request using cached ETag/Last-Modified
+        last_validated = cached_entry.get("last_validated", 0)
+        if time.time() - last_validated < CACHE_TTL_SECONDS:
+            # Within TTL: return cached data directly without any HTTP request
+            data = cached_entry["data"]
+            with _cache_lock:
+                _cache[url] = data
+            _cache_stats["hits"] += 1
+            log.debug(f"Disk cache hit (within TTL) for {sanitize_for_log(url)}")
+            return data
+        # Beyond TTL: send conditional request using cached ETag/Last-Modified
         # Server returns 304 if content hasn't changed
         # NOTE: Cached values may be None if the server didn't send these headers.
         # httpx requires header values to be str/bytes, so we only add headers
@@ -2385,6 +2395,9 @@
         "--no-delete", action="store_true", help="Do not delete existing folders"
     )
     parser.add_argument("--plan-json", help="Write plan to JSON file", default=None)
+    parser.add_argument(
+        "--clear-cache", action="store_true", help="Clear the persistent blocklist cache and exit"
+    )
     return parser.parse_args()
 
 
@@ -2412,6 +2425,22 @@
     # NOTE: Called only after successful argument parsing so that `--help` or
     #       argument errors do not perform unnecessary filesystem I/O or logging.
     load_disk_cache()
+
+    # Handle --clear-cache: delete cache file and exit immediately
+    if args.clear_cache:
+        global _disk_cache
+        cache_file = get_cache_dir() / "blocklists.json"
+        if cache_file.exists():
+            try:
+                cache_file.unlink()
+                print(f"{Colors.GREEN}✓ Cleared blocklist cache: {cache_file}{Colors.ENDC}")
+            except OSError as e:
+                print(f"{Colors.FAIL}✗ Failed to clear cache: {e}{Colors.ENDC}")
+                exit(1)
+        else:
+            print(f"{Colors.CYAN}ℹ No cache file found, nothing to clear{Colors.ENDC}")
+        _disk_cache.clear()
+        exit(0)
     profiles_arg = (
         _clean_env_kv(args.profiles or os.getenv("PROFILE", ""), "PROFILE") or ""
     )
@@ -2430,7 +2459,7 @@
                 """Validates one or more profile IDs from comma-separated input."""
                 ids = [extract_profile_id(p) for p in value.split(",") if p.strip()]
                 return bool(ids) and all(
                     validate_profile_id(pid, log_errors=False) for pid in ids
                 )
 
             p_input = get_validated_input(

tests/test_disk_cache.py

@@ -13,6 +13,7 @@
 import os
 import platform
 import tempfile
+import time
 import unittest
 from pathlib import Path
 from unittest.mock import MagicMock, patch
@@ -28,15 +29,15 @@
     def setUp(self):
         """Set up test fixtures."""
         # Clear in-memory caches
         main._cache.clear()
         main._disk_cache.clear()
         main.validate_folder_url.cache_clear()
         # Reset stats
         main._cache_stats = {"hits": 0, "misses": 0, "validations": 0, "errors": 0}
 
         # Create temporary cache directory for testing
         self.temp_dir = tempfile.mkdtemp()
         self.original_get_cache_dir = main.get_cache_dir
 
     def tearDown(self):
         """Clean up after each test."""
@@ -64,7 +65,7 @@
         """Test that cache directory is correct on macOS."""
         with patch('platform.system', return_value='Darwin'):
             cache_dir = main.get_cache_dir()
             self.assertEqual(cache_dir, Path.home() / "Library" / "Caches" / "ctrld-sync")
 
     def test_get_cache_dir_windows(self):
         """Test that cache directory is correct on Windows."""
@@ -73,15 +74,15 @@
                 cache_dir = main.get_cache_dir()
                 # Use string comparison to avoid path separator differences
                 expected = Path(r'C:\Users\Test\AppData\Local') / 'ctrld-sync' / 'cache'
                 self.assertEqual(cache_dir, expected)
 
     def test_load_disk_cache_no_file(self):
         """Test loading cache when no cache file exists."""
         main.get_cache_dir = lambda: Path(self.temp_dir) / "nonexistent"
 
         main.load_disk_cache()
 
         # Should have empty cache, no errors
         self.assertEqual(len(main._disk_cache), 0)
         self.assertEqual(main._cache_stats["errors"], 0)
 
@@ -102,10 +103,10 @@
             }
         }
         with open(cache_file, "w") as f:
             json.dump(test_cache, f)
 
         main.get_cache_dir = lambda: cache_dir
         main.load_disk_cache()
 
         # Should have loaded cache
         self.assertEqual(len(main._disk_cache), 1)
@@ -123,11 +124,11 @@
             f.write("{ invalid json }")
 
         main.get_cache_dir = lambda: cache_dir
         main.load_disk_cache()
 
         # Should have empty cache but not crash
         self.assertEqual(len(main._disk_cache), 0)
         self.assertEqual(main._cache_stats["errors"], 1)
 
     def test_load_disk_cache_invalid_format(self):
         """Test graceful handling of invalid cache format."""
@@ -137,19 +138,19 @@
 
         # Create valid JSON but wrong format (list instead of dict)
         with open(cache_file, "w") as f:
             json.dump(["not", "a", "dict"], f)
 
         main.get_cache_dir = lambda: cache_dir
         main.load_disk_cache()
 
         # Should have empty cache but not crash
         self.assertEqual(len(main._disk_cache), 0)
         # No error increment because it's just a warning
 
     def test_save_disk_cache(self):
         """Test saving cache to disk."""
         cache_dir = Path(self.temp_dir)
         main.get_cache_dir = lambda: cache_dir
 
         # Populate cache
         main._disk_cache["https://example.com/test.json"] = {
@@ -164,11 +165,11 @@
 
         # Verify file was created
         cache_file = cache_dir / "blocklists.json"
         self.assertTrue(cache_file.exists())
 
         # Verify content
         with open(cache_file, "r") as f:
             loaded = json.load(f)
 
         self.assertEqual(len(loaded), 1)
         self.assertIn("https://example.com/test.json", loaded)
@@ -185,17 +186,17 @@
             "last_modified": None,
             "fetched_at": 1234567890.0,
             "last_validated": 1234567890.0
         }
 
         main.save_disk_cache()
 
         # Verify temp file is gone (was renamed)
         temp_file = cache_dir / "blocklists.tmp"
         self.assertFalse(temp_file.exists())
 
         # Verify final file exists
         cache_file = cache_dir / "blocklists.json"
         self.assertTrue(cache_file.exists())
 
     def test_cache_stats_tracking(self):
         """Test that cache statistics are tracked correctly."""
@@ -204,7 +205,7 @@
 
         # Reset stats
         main._cache_stats = {"hits": 0, "misses": 0, "validations": 0, "errors": 0}
 
         def mock_stream(method, url, headers=None):
             mock_response = MagicMock()
             mock_response.status_code = 200
@@ -215,7 +216,7 @@
             mock_response.__enter__ = MagicMock(return_value=mock_response)
             mock_response.__exit__ = MagicMock(return_value=False)
             return mock_response
 
         with patch.object(main._gh, 'stream', side_effect=mock_stream):
             # First fetch - should be a miss
             result1 = main._gh_get(test_url)
@@ -223,16 +224,16 @@
             self.assertEqual(main._cache_stats["hits"], 0)
 
             # Second fetch - should be a hit (in-memory cache)
             result2 = main._gh_get(test_url)
             self.assertEqual(main._cache_stats["hits"], 1)
             self.assertEqual(main._cache_stats["misses"], 1)
 
     def test_conditional_request_with_etag(self):
         """Test that conditional requests use ETag from disk cache."""
         test_url = "https://example.com/test.json"
         test_data = {"group": {"group": "Test"}, "domains": ["example.com"]}
 
         # Pre-populate disk cache with ETag
         main._disk_cache[test_url] = {
             "data": test_data,
             "etag": "abc123",
@@ -240,7 +241,7 @@
             "fetched_at": 1234567890.0,
             "last_validated": 1234567890.0
         }
 
         def mock_stream(method, url, headers=None):
             # Verify If-None-Match header was sent
             self.assertIsNotNone(headers)
@@ -253,23 +254,23 @@
             mock_response.headers = {}
             mock_response.__enter__ = MagicMock(return_value=mock_response)
             mock_response.__exit__ = MagicMock(return_value=False)
             return mock_response
 
         with patch.object(main._gh, 'stream', side_effect=mock_stream):
             result = main._gh_get(test_url)
 
             # Should return cached data
             self.assertEqual(result, test_data)
             # Should count as validation, not miss
             self.assertEqual(main._cache_stats["validations"], 1)
             self.assertEqual(main._cache_stats["misses"], 0)
 
     def test_conditional_request_with_last_modified(self):
         """Test that conditional requests use Last-Modified from disk cache."""
         test_url = "https://example.com/test.json"
         test_data = {"group": {"group": "Test"}, "domains": ["example.com"]}
 
         # Pre-populate disk cache with Last-Modified
         main._disk_cache[test_url] = {
             "data": test_data,
             "etag": None,
@@ -277,11 +278,11 @@
             "fetched_at": 1234567890.0,
             "last_validated": 1234567890.0
         }
 
         def mock_stream(method, url, headers=None):
             # Verify If-Modified-Since header was sent
             self.assertIsNotNone(headers)
             self.assertEqual(headers.get("If-Modified-Since"), "Mon, 01 Jan 2024 00:00:00 GMT")
 
             # Return 304 Not Modified
             mock_response = MagicMock()
@@ -290,15 +291,115 @@
             mock_response.headers = {}
             mock_response.__enter__ = MagicMock(return_value=mock_response)
             mock_response.__exit__ = MagicMock(return_value=False)
             return mock_response
 
         with patch.object(main._gh, 'stream', side_effect=mock_stream):
             result = main._gh_get(test_url)
 
             # Should return cached data
             self.assertEqual(result, test_data)
             # Should count as validation
             self.assertEqual(main._cache_stats["validations"], 1)
+
+    def test_ttl_within_ttl_returns_disk_cache_without_request(self):
+        """Test that disk cache entries within TTL are returned without an HTTP request."""
+        test_url = "https://example.com/test.json"
+        test_data = {"group": {"group": "Test"}, "domains": ["example.com"]}
+
+        # Pre-populate disk cache with a recent last_validated timestamp (within TTL)
+        main._disk_cache[test_url] = {
+            "data": test_data,
+            "etag": "fresh123",
+            "last_modified": None,
+            "fetched_at": time.time(),
+            "last_validated": time.time(),  # Just validated - within TTL
+        }
+
+        http_called = []
+
+        def mock_stream(method, url, headers=None):
+            http_called.append(url)
+            mock_response = MagicMock()
+            mock_response.status_code = 200
+            mock_response.headers = {"Content-Type": "application/json"}
+            mock_response.__enter__ = MagicMock(return_value=mock_response)
+            mock_response.__exit__ = MagicMock(return_value=False)
+            return mock_response
+
+        with patch.object(main._gh, 'stream', side_effect=mock_stream):
+            result = main._gh_get(test_url)
+
+        # HTTP should NOT have been called (within TTL)
+        self.assertEqual(len(http_called), 0)
+        # Result should be the cached data
+        self.assertEqual(result, test_data)
+        # Should count as a hit
+        self.assertEqual(main._cache_stats["hits"], 1)
+        self.assertEqual(main._cache_stats["misses"], 0)
+        self.assertEqual(main._cache_stats["validations"], 0)
+
+    def test_ttl_expired_sends_conditional_request(self):
+        """Test that disk cache entries beyond TTL trigger a conditional HTTP request."""
+        test_url = "https://example.com/test.json"
+        test_data = {"group": {"group": "Test"}, "domains": ["example.com"]}
+
+        # Pre-populate disk cache with an old last_validated (beyond TTL)
+        main._disk_cache[test_url] = {
+            "data": test_data,
+            "etag": "stale123",
+            "last_modified": None,
+            "fetched_at": 0.0,       # very old
+            "last_validated": 0.0,   # very old - beyond any TTL
+        }
+
+        def mock_stream(method, url, headers=None):
+            # Conditional request should be sent with If-None-Match
+            self.assertEqual(headers.get("If-None-Match"), "stale123")
+            mock_response = MagicMock()
+            mock_response.status_code = 304
+            mock_response.raise_for_status = MagicMock()
+            mock_response.headers = {}
+            mock_response.__enter__ = MagicMock(return_value=mock_response)
+            mock_response.__exit__ = MagicMock(return_value=False)
+            return mock_response
+
+        with patch.object(main._gh, 'stream', side_effect=mock_stream):
+            result = main._gh_get(test_url)
+
+        # Should return cached data (304 response)
+        self.assertEqual(result, test_data)
+        # Should count as validation (conditional request)
+        self.assertEqual(main._cache_stats["validations"], 1)
+        self.assertEqual(main._cache_stats["hits"], 0)
+
+    def test_clear_cache_deletes_file(self):
+        """Test that --clear-cache deletes the cache file."""
+        cache_dir = Path(self.temp_dir)
+        cache_file = cache_dir / "blocklists.json"
+
+        # Create a cache file
+        cache_file.write_text('{}')
+        self.assertTrue(cache_file.exists())
+
+        # Populate in-memory disk cache
+        main._disk_cache["https://example.com/test.json"] = {
+            "data": {},
+            "etag": None,
+            "last_modified": None,
+            "fetched_at": 0.0,
+            "last_validated": 0.0,
+        }
+
+        with patch('main.get_cache_dir', return_value=cache_dir):
+            # Simulate --clear-cache logic
+            if cache_file.exists():
+                cache_file.unlink()
+            main._disk_cache.clear()
+
+        # Cache file should be gone
+        self.assertFalse(cache_file.exists())
+        # In-memory disk cache should be empty
+        self.assertEqual(len(main._disk_cache), 0)
 
 
 if __name__ == '__main__':