From 1ae0a6797ef36623f2e62f5e87eb90dedf23dca6 Mon Sep 17 00:00:00 2001
From: yya007 <yaoyuang1996@gmail.com>
Date: Mon, 22 Jun 2026 02:25:49 -0700
Subject: [PATCH 01/13] perf(crawlers): cache Trees call by pushed_at
 (rate-limit option 6b)

find_skill_md_paths_cached skips the recursive Trees API call when a
repo's pushed_at is unchanged since it was last walked, reusing the
cached {path: blob_sha} map. Wired through the topic crawler. Makes a
warm run approach zero metered calls per unchanged repo.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 crawlers/base.py                     | 56 +++++++++++++++++++
 crawlers/topic_crawler.py            | 12 +++-
 tests/crawlers/test_base.py          | 84 ++++++++++++++++++++++++++++
 tests/crawlers/test_topic_crawler.py | 69 ++++++++++++++++-------
 4 files changed, 197 insertions(+), 24 deletions(-)

diff --git a/crawlers/base.py b/crawlers/base.py
index 1ea7cd1..a198725 100644
--- a/crawlers/base.py
+++ b/crawlers/base.py
@@ -25,6 +25,9 @@
   fetch_skill_md_cached()       - Fetch SKILL.md, skipping when blob SHA is cached
   load_content_cache()          - Load persistent blob-SHA -> content cache
   save_content_cache()          - Persist blob-SHA -> content cache
+  find_skill_md_paths_cached()  - Find SKILL.md paths, skipping Trees API when pushed_at is unchanged
+  load_tree_cache()             - Load persistent pushed_at -> {path: sha} tree cache
+  save_tree_cache()             - Persist pushed_at -> {path: sha} tree cache
 """
 
 from __future__ import annotations
@@ -562,6 +565,59 @@ def find_skill_md_paths(session, repo_full_name: str) -> dict[str, str]:
     return paths
 
 
+def find_skill_md_paths_cached(
+    session,
+    repo_full_name: str,
+    pushed_at: str,
+    tree_cache: dict,
+) -> dict[str, str]:
+    """Return SKILL.md paths for a repo, skipping the Trees API call when unchanged.
+
+    Uses ``pushed_at`` as a freshness key.  If the repo's ``pushed_at`` timestamp
+    matches what is stored in ``tree_cache``, the cached ``{path: sha}`` mapping is
+    returned immediately without any API call.  Otherwise ``find_skill_md_paths`` is
+    called and the result is stored back into ``tree_cache`` (mutated in place).
+
+    An empty or falsy ``pushed_at`` always calls the API and never caches the result
+    because we cannot prove freshness without a timestamp.
+
+    Args:
+        session:        A requests.Session from make_session().
+        repo_full_name: "{owner}/{repo}" string.
+        pushed_at:      The repo's ``pushed_at`` ISO-8601 string from the metadata API.
+                        Pass ``""`` (or any falsy value) to force a live fetch.
+        tree_cache:     Mutable dict that persists across calls within a crawl run.
+                        Shape: ``{repo_full_name: {"pushed_at": str, "paths": dict}}``.
+
+    Returns:
+        Dict mapping SKILL.md path → blob SHA (same contract as ``find_skill_md_paths``).
+    """
+    if pushed_at and tree_cache.get(repo_full_name, {}).get("pushed_at") == pushed_at:
+        return tree_cache[repo_full_name]["paths"]
+
+    paths = find_skill_md_paths(session, repo_full_name)
+    if pushed_at:
+        tree_cache[repo_full_name] = {"pushed_at": pushed_at, "paths": paths}
+    return paths
+
+
+def load_tree_cache(path: str) -> dict:
+    """Load the persistent Trees-API path cache, or {} if absent/corrupt.
+
+    Thin alias for ``load_meta_cache`` — same JSON format, different file.
+    Cache shape: ``{repo_full_name: {"pushed_at": str, "paths": {path: sha}}}``.
+    """
+    return load_meta_cache(path)
+
+
+def save_tree_cache(cache: dict, path: str) -> None:
+    """Persist the Trees-API path cache atomically.
+
+    Thin alias for ``save_meta_cache`` — same atomic-write semantics, different file.
+    """
+    save_meta_cache(cache, path)
+
+
 def _find_skill_md_via_search(session, repo_full_name: str) -> dict[str, str]:
     """Find SKILL.md paths via Code Search API, scoped to one repo.
 
diff --git a/crawlers/topic_crawler.py b/crawlers/topic_crawler.py
index 9186d8a..4dd14ad 100644
--- a/crawlers/topic_crawler.py
+++ b/crawlers/topic_crawler.py
@@ -37,16 +37,18 @@
     add_to_filter_cache,
     fetch_repo_metadata_cached,
     fetch_skill_md_cached,
-    find_skill_md_paths,
+    find_skill_md_paths_cached,
     github_get,
     infer_platforms,
     load_content_cache,
     load_filter_cache,
     load_meta_cache,
+    load_tree_cache,
     make_session,
     parse_frontmatter,
     save_content_cache,
     save_meta_cache,
+    save_tree_cache,
     write_jsonl,
 )
 
@@ -218,9 +220,10 @@ def run(
 
     session = make_session(token=token)
 
-    # Load ETag metadata cache and blob-SHA content cache
+    # Load ETag metadata cache, blob-SHA content cache, and Trees-path cache
     meta_cache = load_meta_cache("data/crawl_state/repo_meta_cache.json")
     content_cache = load_content_cache("data/crawl_state/content_cache.json")
+    tree_cache = load_tree_cache("data/crawl_state/tree_cache.json")
 
     # Load filter cache (repos known to have no SKILL.md)
     filter_cache: set[str] = set()
@@ -289,7 +292,9 @@ def run(
             except RuntimeError as exc:
                 log.warning("Could not fetch metadata for %s: %s", full_name, exc)
                 continue
-            skill_md_paths = find_skill_md_paths(session, full_name)
+            skill_md_paths = find_skill_md_paths_cached(
+                session, full_name, meta.get("pushed_at", ""), tree_cache
+            )
             _repo_cache[full_name] = (meta, skill_md_paths)
             if not skill_md_paths:
                 if filter_cache_path:
@@ -360,6 +365,7 @@ def run(
 
     save_meta_cache(meta_cache, "data/crawl_state/repo_meta_cache.json")
     save_content_cache(content_cache, "data/crawl_state/content_cache.json")
+    save_tree_cache(tree_cache, "data/crawl_state/tree_cache.json")
 
     written = write_jsonl(records, output_path, append=resume)
     log.info("Topic crawler done: %d records written to %s", written, output_path)
diff --git a/tests/crawlers/test_base.py b/tests/crawlers/test_base.py
index d320bd6..89f355d 100644
--- a/tests/crawlers/test_base.py
+++ b/tests/crawlers/test_base.py
@@ -23,6 +23,9 @@
     fetch_skill_md_cached,
     load_content_cache,
     save_content_cache,
+    find_skill_md_paths_cached,
+    load_tree_cache,
+    save_tree_cache,
 )
 
 
@@ -1029,3 +1032,84 @@ def test_empty_sha_always_fetches(self):
             content = fetch_skill_md_cached(MagicMock(), "u/r", "SKILL.md", "", "main", cache)
         assert content == "real"
         mock_fetch.assert_called_once()
+
+
+# ---------------------------------------------------------------------------
+# TestTreeCache (load_tree_cache / save_tree_cache aliases)
+# ---------------------------------------------------------------------------
+
+class TestTreeCacheAliases:
+    def test_load_tree_cache_returns_empty_for_missing_file(self, tmp_path):
+        result = load_tree_cache(str(tmp_path / "nonexistent.json"))
+        assert result == {}
+
+    def test_save_and_load_round_trip(self, tmp_path):
+        path = str(tmp_path / "tree_cache.json")
+        cache = {"u/r": {"pushed_at": "2026-01-01T00:00:00Z", "paths": {"SKILL.md": "sha1"}}}
+        save_tree_cache(cache, path)
+        result = load_tree_cache(path)
+        assert result == cache
+
+    def test_load_tree_cache_returns_empty_on_corrupt_file(self, tmp_path):
+        p = tmp_path / "corrupt.json"
+        p.write_text("not valid json")
+        result = load_tree_cache(str(p))
+        assert result == {}
+
+
+# ---------------------------------------------------------------------------
+# TestFindSkillMdPathsCached
+# ---------------------------------------------------------------------------
+
+class TestFindSkillMdPathsCached:
+    def test_cache_hit_skips_tree_call(self):
+        """Pre-seeded cache with matching pushed_at → no API call made."""
+        tree_cache = {
+            "u/r": {"pushed_at": "2026-01-01T00:00:00Z", "paths": {"SKILL.md": "sha1"}}
+        }
+        with patch("crawlers.base.find_skill_md_paths") as mock_tree:
+            result = find_skill_md_paths_cached(
+                MagicMock(), "u/r", "2026-01-01T00:00:00Z", tree_cache
+            )
+        assert result == {"SKILL.md": "sha1"}
+        mock_tree.assert_not_called()
+
+    def test_cache_miss_calls_and_stores(self):
+        """Empty cache → calls find_skill_md_paths and stores result."""
+        tree_cache = {}
+        with patch("crawlers.base.find_skill_md_paths", return_value={"SKILL.md": "sha9"}) as mock_tree:
+            result = find_skill_md_paths_cached(
+                MagicMock(), "u/r", "2026-02-02T00:00:00Z", tree_cache
+            )
+        assert result == {"SKILL.md": "sha9"}
+        mock_tree.assert_called_once()
+        assert tree_cache["u/r"] == {
+            "pushed_at": "2026-02-02T00:00:00Z",
+            "paths": {"SKILL.md": "sha9"},
+        }
+
+    def test_changed_pushed_at_refetches(self):
+        """Stale pushed_at → refetches and updates cache entry."""
+        tree_cache = {
+            "u/r": {"pushed_at": "2026-01-01T00:00:00Z", "paths": {"SKILL.md": "old_sha"}}
+        }
+        new_paths = {"SKILL.md": "new_sha", "sub/SKILL.md": "abc"}
+        with patch("crawlers.base.find_skill_md_paths", return_value=new_paths) as mock_tree:
+            result = find_skill_md_paths_cached(
+                MagicMock(), "u/r", "2026-03-15T12:00:00Z", tree_cache
+            )
+        mock_tree.assert_called_once()
+        assert result == new_paths
+        assert tree_cache["u/r"]["pushed_at"] == "2026-03-15T12:00:00Z"
+        assert tree_cache["u/r"]["paths"] == new_paths
+
+    def test_empty_pushed_at_always_calls_and_does_not_cache(self):
+        """Falsy pushed_at → always calls API and never caches the result."""
+        tree_cache = {}
+        with patch("crawlers.base.find_skill_md_paths", return_value={"SKILL.md": "x"}) as mock_tree:
+            result = find_skill_md_paths_cached(
+                MagicMock(), "u/r", "", tree_cache
+            )
+        assert result == {"SKILL.md": "x"}
+        mock_tree.assert_called_once()
+        assert tree_cache == {}  # nothing cached
diff --git a/tests/crawlers/test_topic_crawler.py b/tests/crawlers/test_topic_crawler.py
index 728dbad..c0db4a9 100644
--- a/tests/crawlers/test_topic_crawler.py
+++ b/tests/crawlers/test_topic_crawler.py
@@ -88,12 +88,14 @@ def _patch_run(self):
         return (
             patch("crawlers.topic_crawler._discover_topic_repos"),
             patch("crawlers.topic_crawler.fetch_repo_metadata_cached"),
-            patch("crawlers.topic_crawler.find_skill_md_paths"),
+            patch("crawlers.topic_crawler.find_skill_md_paths_cached"),
             patch("crawlers.topic_crawler.fetch_skill_md_cached"),
             patch("crawlers.topic_crawler.load_meta_cache", return_value={}),
             patch("crawlers.topic_crawler.save_meta_cache"),
             patch("crawlers.topic_crawler.load_content_cache", return_value={}),
             patch("crawlers.topic_crawler.save_content_cache"),
+            patch("crawlers.topic_crawler.load_tree_cache", return_value={}),
+            patch("crawlers.topic_crawler.save_tree_cache"),
         )
 
     def test_writes_records_for_discovered_repos(self, tmp_path):
@@ -101,12 +103,14 @@ def test_writes_records_for_discovered_repos(self, tmp_path):
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
-             patch("crawlers.topic_crawler.find_skill_md_paths") as mock_paths, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
              patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_meta_cache"), \
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"):
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"):
             mock_disc.return_value = ["user/skill-a", "user/skill-b"]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
@@ -122,12 +126,14 @@ def test_output_has_required_fields(self, tmp_path):
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
-             patch("crawlers.topic_crawler.find_skill_md_paths") as mock_paths, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
              patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_meta_cache"), \
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"):
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"):
             mock_disc.return_value = ["user/skill-a"]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
@@ -145,12 +151,14 @@ def test_source_tag_is_topic(self, tmp_path):
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
-             patch("crawlers.topic_crawler.find_skill_md_paths") as mock_paths, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
              patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_meta_cache"), \
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"):
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"):
             mock_disc.return_value = ["user/skill-a"]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
@@ -167,12 +175,14 @@ def test_respects_limit(self, tmp_path):
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
-             patch("crawlers.topic_crawler.find_skill_md_paths") as mock_paths, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
              patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_meta_cache"), \
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"):
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"):
             mock_disc.return_value = [f"user/skill-{i}" for i in range(10)]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
@@ -188,12 +198,14 @@ def test_skips_repos_with_no_skill_md(self, tmp_path):
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
-             patch("crawlers.topic_crawler.find_skill_md_paths") as mock_paths, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
              patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_meta_cache"), \
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"):
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"):
             mock_disc.return_value = ["user/no-skill-md", "user/has-skill-md"]
             mock_meta.return_value = _mock_meta()
             mock_paths.side_effect = [{}, {"SKILL.md": "sha1"}]
@@ -217,12 +229,14 @@ def test_skips_already_covered_repos(self, tmp_path):
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
-             patch("crawlers.topic_crawler.find_skill_md_paths") as mock_paths, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
              patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_meta_cache"), \
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"):
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"):
             mock_disc.return_value = ["user/already-covered", "user/new-skill"]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
@@ -250,12 +264,14 @@ def test_resume_skips_existing_keys(self, tmp_path):
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
-             patch("crawlers.topic_crawler.find_skill_md_paths") as mock_paths, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
              patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_meta_cache"), \
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"):
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"):
             mock_disc.return_value = ["user/skill-a", "user/skill-b"]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
@@ -270,12 +286,14 @@ def test_name_falls_back_to_repo_name_when_no_frontmatter(self, tmp_path):
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
-             patch("crawlers.topic_crawler.find_skill_md_paths") as mock_paths, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
              patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_meta_cache"), \
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"):
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"):
             mock_disc.return_value = ["user/my-cool-skill"]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
@@ -288,13 +306,14 @@ def test_name_falls_back_to_repo_name_when_no_frontmatter(self, tmp_path):
         assert record["name"] == "my-cool-skill"
 
     def test_topic_crawl_uses_caches(self, tmp_path, monkeypatch):
-        """Crawl loads both caches at start and saves both caches at end."""
+        """Crawl loads meta, content, and tree caches at start and saves all three at end."""
         import crawlers.topic_crawler as tc
 
-        calls = {"saved_meta": 0, "saved_content": 0}
+        calls = {"saved_meta": 0, "saved_content": 0, "saved_tree": 0}
 
         monkeypatch.setattr(tc, "load_meta_cache", lambda p: {})
         monkeypatch.setattr(tc, "load_content_cache", lambda p: {})
+        monkeypatch.setattr(tc, "load_tree_cache", lambda p: {})
         monkeypatch.setattr(
             tc, "save_meta_cache",
             lambda c, p: calls.__setitem__("saved_meta", calls["saved_meta"] + 1),
@@ -303,17 +322,24 @@ def test_topic_crawl_uses_caches(self, tmp_path, monkeypatch):
             tc, "save_content_cache",
             lambda c, p: calls.__setitem__("saved_content", calls["saved_content"] + 1),
         )
+        monkeypatch.setattr(
+            tc, "save_tree_cache",
+            lambda c, p: calls.__setitem__("saved_tree", calls["saved_tree"] + 1),
+        )
         monkeypatch.setattr(
             tc, "fetch_repo_metadata_cached",
             lambda s, r, c: {
                 "stargazers_count": 50,
                 "default_branch": "main",
-                "pushed_at": "",
+                "pushed_at": "2026-01-01T00:00:00Z",
                 "topics": [],
                 "description": "",
             },
         )
-        monkeypatch.setattr(tc, "find_skill_md_paths", lambda s, r: {"SKILL.md": "sha1"})
+        monkeypatch.setattr(
+            tc, "find_skill_md_paths_cached",
+            lambda s, r, p, c: {"SKILL.md": "sha1"},
+        )
         monkeypatch.setattr(tc, "fetch_skill_md_cached", lambda *a, **k: "---\nname: t\n---")
         monkeypatch.setattr(tc, "_discover_topic_repos", lambda s, limit=1000: ["user/skill-a"])
 
@@ -323,3 +349,4 @@ def test_topic_crawl_uses_caches(self, tmp_path, monkeypatch):
         assert count == 1
         assert calls["saved_meta"] == 1
         assert calls["saved_content"] == 1
+        assert calls["saved_tree"] == 1

From 124a1a39321c167178aff2d05dd1c729cd77c5cd Mon Sep 17 00:00:00 2001
From: yya007 <yaoyuang1996@gmail.com>
Date: Mon, 22 Jun 2026 02:32:02 -0700
Subject: [PATCH 02/13] perf(topic-crawler): date-filter discovery to cut
 repeat search (option 4)

In incremental/discover mode, append pushed:>last-run to each topic
search query so re-runs only surface repos changed since the previous
crawl, instead of re-paginating every query in full every time. Last-run
timestamp is persisted via crawl_state.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 crawlers/topic_crawler.py            | 28 +++++++--
 tests/crawlers/test_topic_crawler.py | 91 +++++++++++++++++++++++++++-
 2 files changed, 113 insertions(+), 6 deletions(-)

diff --git a/crawlers/topic_crawler.py b/crawlers/topic_crawler.py
index 4dd14ad..f554c3d 100644
--- a/crawlers/topic_crawler.py
+++ b/crawlers/topic_crawler.py
@@ -34,6 +34,7 @@
 
 from crawlers.base import (
     GITHUB_API,
+    _utc_now_iso,
     add_to_filter_cache,
     fetch_repo_metadata_cached,
     fetch_skill_md_cached,
@@ -41,12 +42,14 @@
     github_get,
     infer_platforms,
     load_content_cache,
+    load_crawl_state,
     load_filter_cache,
     load_meta_cache,
     load_tree_cache,
     make_session,
     parse_frontmatter,
     save_content_cache,
+    save_crawl_state,
     save_meta_cache,
     save_tree_cache,
     write_jsonl,
@@ -96,7 +99,7 @@
 # Discovery
 # ---------------------------------------------------------------------------
 
-def _discover_topic_repos(session, limit: int = 1000) -> list[str]:
+def _discover_topic_repos(session, limit: int = 1000, since: str | None = None) -> list[str]:
     """Search GitHub for repos matching TOPIC_QUERIES.
 
     Paginates each query (up to 1000 results per query as GitHub allows).
@@ -105,6 +108,10 @@ def _discover_topic_repos(session, limit: int = 1000) -> list[str]:
     Args:
         session: A requests.Session from make_session().
         limit:   Total maximum unique repos to return across all queries.
+        since:   Optional ISO-8601 timestamp.  When provided, appends
+                 ``pushed:><since>`` to every query so only repos pushed after
+                 that time are returned — dramatically reducing API quota on
+                 incremental/discover re-runs.
 
     Returns:
         Deduplicated list of full repo names.
@@ -113,16 +120,17 @@ def _discover_topic_repos(session, limit: int = 1000) -> list[str]:
     results: list[str] = []
 
     for query in TOPIC_QUERIES:
+        effective_query = f"{query} pushed:>{since}" if since else query
         page = 1
         while len(results) < limit:
             try:
                 data = github_get(
                     session,
                     f"{GITHUB_API}/search/repositories",
-                    params={"q": query, "per_page": 100, "page": page},
+                    params={"q": effective_query, "per_page": 100, "page": page},
                 )
             except RuntimeError as exc:
-                log.warning("Topic repo search failed for %r (page %d): %s", query, page, exc)
+                log.warning("Topic repo search failed for %r (page %d): %s", effective_query, page, exc)
                 break
 
             items = data.get("items", [])
@@ -218,6 +226,10 @@ def run(
         resume = True
     import json as _json
 
+    # Load per-source crawl state for date-filter support
+    crawl_state = load_crawl_state("topic")
+    run_started = _utc_now_iso()
+
     session = make_session(token=token)
 
     # Load ETag metadata cache, blob-SHA content cache, and Trees-path cache
@@ -256,8 +268,9 @@ def run(
                     pass
         log.info("Resume mode: %d skill keys already in output", len(existing_skill_keys))
 
-    # Discover repos via topic search
-    discovered = _discover_topic_repos(session, limit=1000)
+    # Discover repos via topic search, using date-filter on incremental/discover runs
+    since = crawl_state.get("last_discovery_at") if mode in ("incremental", "discover") else None
+    discovered = _discover_topic_repos(session, limit=1000, since=since)
 
     # Cache (meta, skill_md_paths) per repo to avoid repeated API calls
     _repo_cache: dict[str, tuple[dict, list[str]]] = {}
@@ -367,6 +380,11 @@ def run(
     save_content_cache(content_cache, "data/crawl_state/content_cache.json")
     save_tree_cache(tree_cache, "data/crawl_state/tree_cache.json")
 
+    # Persist discovery timestamp so next incremental/discover run can filter by it.
+    # Use run_started (not now) so repos pushed *during* this crawl aren't missed.
+    crawl_state["last_discovery_at"] = run_started
+    save_crawl_state(crawl_state, "topic")
+
     written = write_jsonl(records, output_path, append=resume)
     log.info("Topic crawler done: %d records written to %s", written, output_path)
     return written
diff --git a/tests/crawlers/test_topic_crawler.py b/tests/crawlers/test_topic_crawler.py
index c0db4a9..b64dd77 100644
--- a/tests/crawlers/test_topic_crawler.py
+++ b/tests/crawlers/test_topic_crawler.py
@@ -341,7 +341,7 @@ def test_topic_crawl_uses_caches(self, tmp_path, monkeypatch):
             lambda s, r, p, c: {"SKILL.md": "sha1"},
         )
         monkeypatch.setattr(tc, "fetch_skill_md_cached", lambda *a, **k: "---\nname: t\n---")
-        monkeypatch.setattr(tc, "_discover_topic_repos", lambda s, limit=1000: ["user/skill-a"])
+        monkeypatch.setattr(tc, "_discover_topic_repos", lambda s, limit=1000, since=None: ["user/skill-a"])
 
         out = str(tmp_path / "out.jsonl")
         count = tc.run(out)
@@ -350,3 +350,92 @@ def test_topic_crawl_uses_caches(self, tmp_path, monkeypatch):
         assert calls["saved_meta"] == 1
         assert calls["saved_content"] == 1
         assert calls["saved_tree"] == 1
+
+
+# ---------------------------------------------------------------------------
+# TestDiscoverPushedFilter — date-filter tests (RED phase)
+# ---------------------------------------------------------------------------
+
+class TestDiscoverPushedFilter:
+    """Tests for the since= date-filter on _discover_topic_repos."""
+
+    def test_discover_appends_pushed_filter_when_since_set(self):
+        """When since is set, every query q should end with pushed:><since>."""
+        from crawlers.topic_crawler import _discover_topic_repos
+
+        captured_qs: list[str] = []
+
+        def fake_github_get(session, url, params=None, **kwargs):
+            if params:
+                captured_qs.append(params.get("q", ""))
+            return {"items": []}
+
+        session = MagicMock()
+        with patch("crawlers.topic_crawler.github_get", side_effect=fake_github_get):
+            _discover_topic_repos(session, since="2026-01-01T00:00:00Z")
+
+        assert len(captured_qs) > 0
+        for q in captured_qs:
+            assert q.endswith(" pushed:>2026-01-01T00:00:00Z"), (
+                f"Expected q to end with pushed filter, got: {q!r}"
+            )
+
+    def test_discover_no_filter_when_since_none(self):
+        """When since is None, no query q should contain 'pushed:>'."""
+        from crawlers.topic_crawler import _discover_topic_repos
+
+        captured_qs: list[str] = []
+
+        def fake_github_get(session, url, params=None, **kwargs):
+            if params:
+                captured_qs.append(params.get("q", ""))
+            return {"items": []}
+
+        session = MagicMock()
+        with patch("crawlers.topic_crawler.github_get", side_effect=fake_github_get):
+            _discover_topic_repos(session, since=None)
+
+        assert len(captured_qs) > 0
+        for q in captured_qs:
+            assert "pushed:>" not in q, (
+                f"Expected no pushed filter when since=None, got: {q!r}"
+            )
+
+    def test_run_uses_and_saves_discovery_state(self, tmp_path):
+        """run() in discover mode reads last_discovery_at and saves updated state."""
+        from crawlers.topic_crawler import run
+
+        with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
+             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
+             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_meta_cache"), \
+             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state",
+                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}) as mock_load_state, \
+             patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
+
+            mock_disc.return_value = []
+            mock_meta.return_value = _mock_meta()
+            mock_paths.return_value = {}
+            mock_skill_md.return_value = None
+
+            out = str(tmp_path / "out.jsonl")
+            run(out, mode="discover")
+
+        # _discover_topic_repos was called with since= from state
+        mock_disc.assert_called_once()
+        call_kwargs = mock_disc.call_args
+        assert call_kwargs.kwargs.get("since") == "2026-01-01T00:00:00Z" or (
+            len(call_kwargs.args) >= 2 and call_kwargs.args[1] == "2026-01-01T00:00:00Z"
+        ), f"Expected since='2026-01-01T00:00:00Z', got call: {call_kwargs}"
+
+        # save_crawl_state was called once and the state has last_discovery_at set
+        mock_save_state.assert_called_once()
+        saved_state = mock_save_state.call_args.args[0]
+        assert "last_discovery_at" in saved_state
+        assert saved_state["last_discovery_at"]  # non-empty timestamp

From ff6804cbc8aac363d1fd60f5eb8d9e3a434b0ce9 Mon Sep 17 00:00:00 2001
From: yya007 <yaoyuang1996@gmail.com>
Date: Mon, 22 Jun 2026 02:36:41 -0700
Subject: [PATCH 03/13] test(topic-crawler): isolate TestTopicCrawlerRun from
 crawl_state disk I/O

run() now always touches crawl_state; mock load/save_crawl_state in the
existing run tests so they no longer write data/crawl_state/topic.json.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 tests/crawlers/test_topic_crawler.py | 34 +++++++++++++++++++++-------
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/tests/crawlers/test_topic_crawler.py b/tests/crawlers/test_topic_crawler.py
index b64dd77..ba98517 100644
--- a/tests/crawlers/test_topic_crawler.py
+++ b/tests/crawlers/test_topic_crawler.py
@@ -110,7 +110,9 @@ def test_writes_records_for_discovered_repos(self, tmp_path):
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"):
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
+             patch("crawlers.topic_crawler.save_crawl_state"):
             mock_disc.return_value = ["user/skill-a", "user/skill-b"]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
@@ -133,7 +135,9 @@ def test_output_has_required_fields(self, tmp_path):
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"):
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
+             patch("crawlers.topic_crawler.save_crawl_state"):
             mock_disc.return_value = ["user/skill-a"]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
@@ -158,7 +162,9 @@ def test_source_tag_is_topic(self, tmp_path):
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"):
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
+             patch("crawlers.topic_crawler.save_crawl_state"):
             mock_disc.return_value = ["user/skill-a"]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
@@ -182,7 +188,9 @@ def test_respects_limit(self, tmp_path):
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"):
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
+             patch("crawlers.topic_crawler.save_crawl_state"):
             mock_disc.return_value = [f"user/skill-{i}" for i in range(10)]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
@@ -205,7 +213,9 @@ def test_skips_repos_with_no_skill_md(self, tmp_path):
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"):
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
+             patch("crawlers.topic_crawler.save_crawl_state"):
             mock_disc.return_value = ["user/no-skill-md", "user/has-skill-md"]
             mock_meta.return_value = _mock_meta()
             mock_paths.side_effect = [{}, {"SKILL.md": "sha1"}]
@@ -236,7 +246,9 @@ def test_skips_already_covered_repos(self, tmp_path):
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"):
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
+             patch("crawlers.topic_crawler.save_crawl_state"):
             mock_disc.return_value = ["user/already-covered", "user/new-skill"]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
@@ -271,7 +283,9 @@ def test_resume_skips_existing_keys(self, tmp_path):
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"):
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
+             patch("crawlers.topic_crawler.save_crawl_state"):
             mock_disc.return_value = ["user/skill-a", "user/skill-b"]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
@@ -293,7 +307,9 @@ def test_name_falls_back_to_repo_name_when_no_frontmatter(self, tmp_path):
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"):
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
+             patch("crawlers.topic_crawler.save_crawl_state"):
             mock_disc.return_value = ["user/my-cool-skill"]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
@@ -342,6 +358,8 @@ def test_topic_crawl_uses_caches(self, tmp_path, monkeypatch):
         )
         monkeypatch.setattr(tc, "fetch_skill_md_cached", lambda *a, **k: "---\nname: t\n---")
         monkeypatch.setattr(tc, "_discover_topic_repos", lambda s, limit=1000, since=None: ["user/skill-a"])
+        monkeypatch.setattr(tc, "load_crawl_state", lambda p: {})
+        monkeypatch.setattr(tc, "save_crawl_state", lambda state, p: None)
 
         out = str(tmp_path / "out.jsonl")
         count = tc.run(out)

From fbe75486865924d00979a4f896e8a5fe8cbef1bc Mon Sep 17 00:00:00 2001
From: yya007 <yaoyuang1996@gmail.com>
Date: Mon, 22 Jun 2026 02:45:49 -0700
Subject: [PATCH 04/13] perf(crawlers): batch repo metadata via GraphQL
 (rate-limit option 5)

fetch_repo_metadata_batch fetches stars/pushedAt/defaultBranch/topics for
up to 100 repos in one GraphQL POST (separate 5k-point/hr pool) instead of
one REST call each. The topic crawler bulk-fetches discovered repos and
falls back to the per-repo cached REST path for any repo GraphQL omits.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 crawlers/base.py                     | 135 +++++++++++++++++++++++++
 crawlers/topic_crawler.py            |  11 ++-
 tests/crawlers/test_base.py          | 141 +++++++++++++++++++++++++++
 tests/crawlers/test_topic_crawler.py |  65 ++++++++++++
 4 files changed, 351 insertions(+), 1 deletion(-)

diff --git a/crawlers/base.py b/crawlers/base.py
index a198725..a252ff4 100644
--- a/crawlers/base.py
+++ b/crawlers/base.py
@@ -9,6 +9,7 @@
   fetch_repo_metadata()         - Fetch stars/pushed_at/topics/branch for a repo
   fetch_repo_metadata_with_etag() - ETag-aware version; returns (dict|None, etag|None)
   fetch_repo_metadata_cached()   - Like fetch_repo_metadata_with_etag but with a persistent cache (304 = zero quota)
+  fetch_repo_metadata_batch()   - Batch-fetch metadata for up to 100 repos via GraphQL (one POST per 100)
   fetch_commit_sha()            - Fetch HEAD commit SHA for a repo (one API call)
   find_skill_md_paths()         - Find all SKILL.md paths → {path: blob_sha} dict
   load_filter_cache()           - Load set of filtered-out canonical URLs
@@ -879,6 +880,140 @@ def fetch_repo_metadata_cached(session, repo_full_name: str, cache: dict) -> dic
     return meta
 
 
+_GRAPHQL_BATCH_SIZE = 100
+
+
+def fetch_repo_metadata_batch(
+    session,
+    repo_full_names: list[str],
+) -> dict[str, dict]:
+    """Batch-fetch metadata for up to 100 repos per request via the GitHub GraphQL API.
+
+    Uses one GraphQL POST per chunk of ≤100 repos (aliased fields), consuming from
+    the separate 5,000-point/hr GraphQL quota rather than the REST quota.
+
+    Args:
+        session:         A requests.Session from make_session() (carries Authorization header).
+        repo_full_names: List of "{owner}/{repo}" strings to fetch.
+
+    Returns:
+        Dict mapping full_name → {stargazers_count, pushed_at, topics, description,
+        default_branch} for every repo that resolved successfully.  Repos that GitHub
+        returns as null (deleted, renamed, private, or otherwise inaccessible) are
+        simply absent from the result — callers should fall back to the per-repo REST
+        path for missing entries.
+
+    Error handling:
+        - network / non-200 / unparseable JSON: logs a warning, skips that chunk.
+        - GraphQL ``errors`` array: resolved aliases in ``data`` are still returned;
+          only the null aliases are omitted.
+    """
+    graphql_url = f"{GITHUB_API}/graphql"
+    result: dict[str, dict] = {}
+
+    # Filter entries that don't have a "/" — they cannot be split into owner/name.
+    valid: list[tuple[int, str, str, str]] = []  # (original_index, alias, owner, name)
+    for i, full_name in enumerate(repo_full_names):
+        if "/" not in full_name:
+            logger.warning("fetch_repo_metadata_batch: skipping malformed entry %r", full_name)
+            continue
+        owner, name = full_name.split("/", 1)
+        valid.append((i, f"r{i}", owner, name))
+
+    # Process in chunks of _GRAPHQL_BATCH_SIZE
+    for chunk_start in range(0, len(valid), _GRAPHQL_BATCH_SIZE):
+        chunk = valid[chunk_start: chunk_start + _GRAPHQL_BATCH_SIZE]
+
+        # Build variable declarations and field aliases for this chunk
+        var_decls: list[str] = []
+        field_aliases: list[str] = []
+        variables: dict[str, str] = {}
+
+        for _orig_idx, alias, owner, name in chunk:
+            var_decls.append(f"${alias}Owner:String! ${alias}Name:String!")
+            field_aliases.append(
+                f"{alias}: repository(owner:${alias}Owner, name:${alias}Name) {{"
+                f" stargazerCount pushedAt description"
+                f" defaultBranchRef {{ name }}"
+                f" repositoryTopics(first:20) {{ nodes {{ topic {{ name }} }} }}"
+                f" }}"
+            )
+            variables[f"{alias}Owner"] = owner
+            variables[f"{alias}Name"] = name
+
+        query = "query(" + " ".join(var_decls) + ") { " + " ".join(field_aliases) + " }"
+
+        try:
+            resp = session.post(
+                graphql_url,
+                json={"query": query, "variables": variables},
+                timeout=30,
+            )
+        except requests.RequestException as exc:
+            logger.warning(
+                "fetch_repo_metadata_batch: network error on chunk starting at index %d: %s",
+                chunk_start,
+                exc,
+            )
+            continue
+
+        record_request(graphql_url, resp.status_code)
+
+        if resp.status_code != 200:
+            logger.warning(
+                "fetch_repo_metadata_batch: HTTP %s for chunk at index %d",
+                resp.status_code,
+                chunk_start,
+            )
+            continue
+
+        try:
+            payload = resp.json()
+        except ValueError as exc:
+            logger.warning(
+                "fetch_repo_metadata_batch: unparseable JSON for chunk at index %d: %s",
+                chunk_start,
+                exc,
+            )
+            continue
+
+        if payload.get("errors"):
+            logger.debug(
+                "fetch_repo_metadata_batch: GraphQL errors in chunk (still using resolved data): %s",
+                payload["errors"],
+            )
+
+        data = payload.get("data") or {}
+
+        # Map resolved aliases back to full_name
+        alias_to_full: dict[str, str] = {
+            alias: repo_full_names[orig_idx]
+            for orig_idx, alias, _owner, _name in chunk
+        }
+
+        for alias, node in data.items():
+            if node is None:
+                continue  # repo not found / inaccessible
+            full_name = alias_to_full.get(alias)
+            if full_name is None:
+                continue
+            default_branch_ref = node.get("defaultBranchRef") or {}
+            topics = [
+                n["topic"]["name"]
+                for n in (node.get("repositoryTopics") or {}).get("nodes", [])
+                if n and n.get("topic")
+            ]
+            result[full_name] = {
+                "stargazers_count": node.get("stargazerCount") or 0,
+                "pushed_at": node.get("pushedAt") or "",
+                "description": node.get("description") or "",
+                "default_branch": default_branch_ref.get("name") or "main",
+                "topics": topics,
+            }
+
+    return result
+
+
 def fetch_commit_sha(session, repo_full_name: str) -> str | None:
     """Fetch the HEAD commit SHA for a repo (one API call).
 
diff --git a/crawlers/topic_crawler.py b/crawlers/topic_crawler.py
index f554c3d..6f052aa 100644
--- a/crawlers/topic_crawler.py
+++ b/crawlers/topic_crawler.py
@@ -36,6 +36,7 @@
     GITHUB_API,
     _utc_now_iso,
     add_to_filter_cache,
+    fetch_repo_metadata_batch,
     fetch_repo_metadata_cached,
     fetch_skill_md_cached,
     find_skill_md_paths_cached,
@@ -272,6 +273,11 @@ def run(
     since = crawl_state.get("last_discovery_at") if mode in ("incremental", "discover") else None
     discovered = _discover_topic_repos(session, limit=1000, since=since)
 
+    # Bulk-fetch metadata for all discovered repos via a single GraphQL batch.
+    # Each chunk of ≤100 repos costs one GraphQL POST (separate 5k-point/hr pool).
+    # Repos absent from the result fall back to the per-repo cached REST path below.
+    batch_meta = fetch_repo_metadata_batch(session, discovered)
+
     # Cache (meta, skill_md_paths) per repo to avoid repeated API calls
     _repo_cache: dict[str, tuple[dict, list[str]]] = {}
 
@@ -301,7 +307,10 @@ def run(
         # Fetch metadata + SKILL.md paths
         if full_name not in _repo_cache:
             try:
-                meta = fetch_repo_metadata_cached(session, full_name, meta_cache)
+                # Use GraphQL batch result when available; fall back to per-repo REST
+                meta = batch_meta.get(full_name) or fetch_repo_metadata_cached(
+                    session, full_name, meta_cache
+                )
             except RuntimeError as exc:
                 log.warning("Could not fetch metadata for %s: %s", full_name, exc)
                 continue
diff --git a/tests/crawlers/test_base.py b/tests/crawlers/test_base.py
index 89f355d..6ab9aae 100644
--- a/tests/crawlers/test_base.py
+++ b/tests/crawlers/test_base.py
@@ -12,6 +12,7 @@
     decode_b64_utf8,
     extract_github_url,
     fetch_commit_sha,
+    fetch_repo_metadata_batch,
     fetch_skill_md,
     load_crawl_state,
     load_existing_records,
@@ -1113,3 +1114,143 @@ def test_empty_pushed_at_always_calls_and_does_not_cache(self):
         assert result == {"SKILL.md": "x"}
         mock_tree.assert_called_once()
         assert tree_cache == {}  # nothing cached
+
+
+# ---------------------------------------------------------------------------
+# TestFetchRepoMetadataBatch
+# ---------------------------------------------------------------------------
+
+def _make_graphql_node(
+    stargazer_count=5,
+    pushed_at="2026-01-01T00:00:00Z",
+    description="A skill.",
+    default_branch="main",
+    topics=("python", "ai"),
+):
+    """Build a fake GraphQL repository node dict."""
+    return {
+        "stargazerCount": stargazer_count,
+        "pushedAt": pushed_at,
+        "description": description,
+        "defaultBranchRef": {"name": default_branch},
+        "repositoryTopics": {
+            "nodes": [{"topic": {"name": t}} for t in topics],
+        },
+    }
+
+
+def _make_graphql_response(aliases: dict):
+    """Wrap alias→node mapping in a fake GraphQL response envelope."""
+    return {"data": aliases}
+
+
+class TestFetchRepoMetadataBatch:
+    """Unit tests for fetch_repo_metadata_batch() — no network."""
+
+    def _mock_session(self, json_body, status_code=200):
+        """Return a MagicMock session whose .post() returns a fake response."""
+        resp = MagicMock()
+        resp.status_code = status_code
+        resp.json.return_value = json_body
+        session = MagicMock()
+        session.post.return_value = resp
+        return session
+
+    def test_parses_batch_response(self):
+        """Two repos resolved → both are in the result with correct flattened fields."""
+        payload = _make_graphql_response({
+            "r0": _make_graphql_node(
+                stargazer_count=10,
+                pushed_at="2026-03-01T00:00:00Z",
+                description="skill a",
+                default_branch="main",
+                topics=["python"],
+            ),
+            "r1": _make_graphql_node(
+                stargazer_count=20,
+                pushed_at="2026-04-01T00:00:00Z",
+                description="skill b",
+                default_branch="develop",
+                topics=["go", "cli"],
+            ),
+        })
+        session = self._mock_session(payload)
+        result = fetch_repo_metadata_batch(session, ["o/a", "o/b"])
+
+        assert set(result.keys()) == {"o/a", "o/b"}
+
+        a = result["o/a"]
+        assert a["stargazers_count"] == 10
+        assert a["pushed_at"] == "2026-03-01T00:00:00Z"
+        assert a["description"] == "skill a"
+        assert a["default_branch"] == "main"
+        assert a["topics"] == ["python"]
+
+        b = result["o/b"]
+        assert b["stargazers_count"] == 20
+        assert b["default_branch"] == "develop"
+        assert b["topics"] == ["go", "cli"]
+
+    def test_null_repo_is_omitted(self):
+        """An alias whose value is null is absent from the result."""
+        payload = _make_graphql_response({
+            "r0": _make_graphql_node(stargazer_count=5),
+            "r1": None,
+        })
+        session = self._mock_session(payload)
+        result = fetch_repo_metadata_batch(session, ["o/a", "o/b"])
+
+        assert "o/a" in result
+        assert "o/b" not in result
+
+    def test_partial_errors_still_returns_resolved(self):
+        """data with one resolved repo AND a top-level errors list → resolved repo returned."""
+        payload = {
+            "data": {
+                "r0": _make_graphql_node(stargazer_count=7),
+                "r1": None,
+            },
+            "errors": [{"message": "Could not resolve to a Repository with the name 'o/missing'."}],
+        }
+        session = self._mock_session(payload)
+        result = fetch_repo_metadata_batch(session, ["o/a", "o/missing"])
+
+        assert "o/a" in result
+        assert result["o/a"]["stargazers_count"] == 7
+        assert "o/missing" not in result
+
+    def test_non_200_returns_empty(self):
+        """HTTP 500 → {} with no exception raised."""
+        session = self._mock_session({}, status_code=500)
+        result = fetch_repo_metadata_batch(session, ["o/a", "o/b"])
+
+        assert result == {}
+        # No exception propagated
+        session.post.assert_called_once()
+
+    def test_chunks_over_100(self):
+        """150 repos → two POST calls (100 + 50)."""
+        # Both chunks return empty data (all null / absent) — we just count POSTs.
+        payload = {"data": {}}
+        resp = MagicMock()
+        resp.status_code = 200
+        resp.json.return_value = payload
+        session = MagicMock()
+        session.post.return_value = resp
+
+        names = [f"owner/repo{i}" for i in range(150)]
+        fetch_repo_metadata_batch(session, names)
+
+        assert session.post.call_count == 2
+
+    def test_counts_graphql_request(self):
+        """A successful batch POST increments the 'graphql' API counter."""
+        from crawlers.base import reset_api_counters, get_api_counters
+
+        payload = _make_graphql_response({"r0": _make_graphql_node()})
+        session = self._mock_session(payload)
+
+        reset_api_counters()
+        fetch_repo_metadata_batch(session, ["o/a"])
+
+        assert get_api_counters()["graphql"] >= 1
diff --git a/tests/crawlers/test_topic_crawler.py b/tests/crawlers/test_topic_crawler.py
index ba98517..201dc66 100644
--- a/tests/crawlers/test_topic_crawler.py
+++ b/tests/crawlers/test_topic_crawler.py
@@ -87,6 +87,7 @@ class TestTopicCrawlerRun:
     def _patch_run(self):
         return (
             patch("crawlers.topic_crawler._discover_topic_repos"),
+            patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}),
             patch("crawlers.topic_crawler.fetch_repo_metadata_cached"),
             patch("crawlers.topic_crawler.find_skill_md_paths_cached"),
             patch("crawlers.topic_crawler.fetch_skill_md_cached"),
@@ -102,6 +103,7 @@ def test_writes_records_for_discovered_repos(self, tmp_path):
         from crawlers.topic_crawler import run
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}) as mock_batch, \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
              patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
@@ -127,6 +129,7 @@ def test_output_has_required_fields(self, tmp_path):
         from crawlers.topic_crawler import run
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
              patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
@@ -154,6 +157,7 @@ def test_source_tag_is_topic(self, tmp_path):
         from crawlers.topic_crawler import run
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
              patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
@@ -180,6 +184,7 @@ def test_respects_limit(self, tmp_path):
         from crawlers.topic_crawler import run
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
              patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
@@ -205,6 +210,7 @@ def test_skips_repos_with_no_skill_md(self, tmp_path):
         from crawlers.topic_crawler import run
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
              patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
@@ -238,6 +244,7 @@ def test_skips_already_covered_repos(self, tmp_path):
         )
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
              patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
@@ -275,6 +282,7 @@ def test_resume_skips_existing_keys(self, tmp_path):
         )
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
              patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
@@ -299,6 +307,7 @@ def test_name_falls_back_to_repo_name_when_no_frontmatter(self, tmp_path):
         from crawlers.topic_crawler import run
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
              patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
@@ -327,6 +336,7 @@ def test_topic_crawl_uses_caches(self, tmp_path, monkeypatch):
 
         calls = {"saved_meta": 0, "saved_content": 0, "saved_tree": 0}
 
+        monkeypatch.setattr(tc, "fetch_repo_metadata_batch", lambda s, names: {})
         monkeypatch.setattr(tc, "load_meta_cache", lambda p: {})
         monkeypatch.setattr(tc, "load_content_cache", lambda p: {})
         monkeypatch.setattr(tc, "load_tree_cache", lambda p: {})
@@ -424,6 +434,7 @@ def test_run_uses_and_saves_discovery_state(self, tmp_path):
         from crawlers.topic_crawler import run
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
              patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
@@ -457,3 +468,57 @@ def test_run_uses_and_saves_discovery_state(self, tmp_path):
         saved_state = mock_save_state.call_args.args[0]
         assert "last_discovery_at" in saved_state
         assert saved_state["last_discovery_at"]  # non-empty timestamp
+
+
+# ---------------------------------------------------------------------------
+# TestTopicCrawlerBatchMetaIntegration
+# ---------------------------------------------------------------------------
+
+class TestTopicCrawlerBatchMetaIntegration:
+    """Verify that batch GraphQL metadata is used (and REST is skipped) when available."""
+
+    def test_batch_result_used_and_rest_not_called(self, tmp_path):
+        """When fetch_repo_metadata_batch returns metadata for a repo, the per-repo
+        fetch_repo_metadata_cached should NOT be called for that repo."""
+        from crawlers.topic_crawler import run
+
+        batch_meta = {
+            "user/skill-a": {
+                "stargazers_count": 99,
+                "pushed_at": "2026-05-01T00:00:00Z",
+                "description": "from graphql",
+                "default_branch": "main",
+                "topics": ["graphql"],
+            }
+        }
+
+        with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch",
+                   return_value=batch_meta) as mock_batch, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_rest_meta, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
+             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
+             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_meta_cache"), \
+             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
+             patch("crawlers.topic_crawler.save_crawl_state"):
+
+            mock_disc.return_value = ["user/skill-a"]
+            mock_paths.return_value = {"SKILL.md": "sha1"}
+            mock_skill_md.return_value = None
+
+            out = str(tmp_path / "out.jsonl")
+            count = run(out)
+
+        assert count == 1
+        # The batch provided metadata — per-repo REST fallback must NOT have been called
+        mock_rest_meta.assert_not_called()
+
+        # Confirm the record used the batch-sourced star count
+        import json as _json
+        record = _json.loads(Path(out).read_text().strip())
+        assert record["raw_metadata"]["stars"] == 99

From da11641243ed021e86dac6eadf0a2a94a82a92ae Mon Sep 17 00:00:00 2001
From: yya007 <yaoyuang1996@gmail.com>
Date: Mon, 22 Jun 2026 02:51:10 -0700
Subject: [PATCH 05/13] fix(topic-crawler): don't advance discovery window on
 empty result

A discovery that returns no repos (transient rate-limit / empty search)
must not push last_discovery_at forward, or the next incremental run
would silently skip repos changed in the gap. Guard the crawl_state save
on a non-empty discovery; add a regression test. Also tightens the
discovery-state test to a positive assertion and removes unused mocks.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 crawlers/topic_crawler.py            |  8 ++++--
 tests/crawlers/test_topic_crawler.py | 38 +++++++++++++++++++++++-----
 2 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/crawlers/topic_crawler.py b/crawlers/topic_crawler.py
index 6f052aa..952b8b2 100644
--- a/crawlers/topic_crawler.py
+++ b/crawlers/topic_crawler.py
@@ -391,8 +391,12 @@ def run(
 
     # Persist discovery timestamp so next incremental/discover run can filter by it.
     # Use run_started (not now) so repos pushed *during* this crawl aren't missed.
-    crawl_state["last_discovery_at"] = run_started
-    save_crawl_state(crawl_state, "topic")
+    # Only advance the window when discovery actually returned repos: a transient
+    # failure (rate-limit/empty search) must NOT push last_discovery_at forward, or
+    # the next incremental run would silently skip repos changed in the gap.
+    if discovered:
+        crawl_state["last_discovery_at"] = run_started
+        save_crawl_state(crawl_state, "topic")
 
     written = write_jsonl(records, output_path, append=resume)
     log.info("Topic crawler done: %d records written to %s", written, output_path)
diff --git a/tests/crawlers/test_topic_crawler.py b/tests/crawlers/test_topic_crawler.py
index 201dc66..9265a60 100644
--- a/tests/crawlers/test_topic_crawler.py
+++ b/tests/crawlers/test_topic_crawler.py
@@ -103,7 +103,7 @@ def test_writes_records_for_discovered_repos(self, tmp_path):
         from crawlers.topic_crawler import run
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}) as mock_batch, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
              patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
@@ -445,10 +445,10 @@ def test_run_uses_and_saves_discovery_state(self, tmp_path):
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_tree_cache"), \
              patch("crawlers.topic_crawler.load_crawl_state",
-                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}) as mock_load_state, \
+                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
              patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
 
-            mock_disc.return_value = []
+            mock_disc.return_value = ["user/skill-a"]  # non-empty: a successful discovery
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {}
             mock_skill_md.return_value = None
@@ -459,9 +459,9 @@ def test_run_uses_and_saves_discovery_state(self, tmp_path):
         # _discover_topic_repos was called with since= from state
         mock_disc.assert_called_once()
         call_kwargs = mock_disc.call_args
-        assert call_kwargs.kwargs.get("since") == "2026-01-01T00:00:00Z" or (
-            len(call_kwargs.args) >= 2 and call_kwargs.args[1] == "2026-01-01T00:00:00Z"
-        ), f"Expected since='2026-01-01T00:00:00Z', got call: {call_kwargs}"
+        assert call_kwargs.kwargs.get("since") == "2026-01-01T00:00:00Z", (
+            f"Expected since='2026-01-01T00:00:00Z', got call: {call_kwargs}"
+        )
 
         # save_crawl_state was called once and the state has last_discovery_at set
         mock_save_state.assert_called_once()
@@ -469,6 +469,30 @@ def test_run_uses_and_saves_discovery_state(self, tmp_path):
         assert "last_discovery_at" in saved_state
         assert saved_state["last_discovery_at"]  # non-empty timestamp
 
+    def test_empty_discovery_does_not_advance_window(self, tmp_path):
+        """A discovery that returns no repos (e.g. transient rate-limit) must NOT
+        advance last_discovery_at, or the next run would skip the missed window."""
+        from crawlers.topic_crawler import run
+
+        with patch("crawlers.topic_crawler._discover_topic_repos", return_value=[]), \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_cached"), \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached"), \
+             patch("crawlers.topic_crawler.fetch_skill_md_cached"), \
+             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_meta_cache"), \
+             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state",
+                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
+             patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
+
+            run(str(tmp_path / "out.jsonl"), mode="discover")
+
+        mock_save_state.assert_not_called()
+
 
 # ---------------------------------------------------------------------------
 # TestTopicCrawlerBatchMetaIntegration
@@ -494,7 +518,7 @@ def test_batch_result_used_and_rest_not_called(self, tmp_path):
 
         with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
              patch("crawlers.topic_crawler.fetch_repo_metadata_batch",
-                   return_value=batch_meta) as mock_batch, \
+                   return_value=batch_meta), \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_rest_meta, \
              patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
              patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \

From 2b6738a080849328b639caf5a825c3ad3abbf116 Mon Sep 17 00:00:00 2001
From: yya007 <yaoyuang1996@gmail.com>
Date: Mon, 22 Jun 2026 03:01:03 -0700
Subject: [PATCH 06/13] fix(topic-crawler): batch metadata post-filter; advance
 window only on clean sweep

Codex review (P2 x2): (1) the GraphQL metadata batch fetched every
discovered repo before the already-covered/filter-cache skips ran, wasting
quota in CI (--data-dir data/raw skips most). Batch only the to-process set.
(2) advancing last_discovery_at on any non-empty discovery could skip repos
left unprocessed by a --limit truncation or a per-repo failure; advance only
after a complete, clean sweep.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 crawlers/topic_crawler.py            |  51 ++++++----
 tests/crawlers/test_topic_crawler.py | 144 +++++++++++++++++++++++++++
 2 files changed, 173 insertions(+), 22 deletions(-)

diff --git a/crawlers/topic_crawler.py b/crawlers/topic_crawler.py
index 952b8b2..a3ee0bc 100644
--- a/crawlers/topic_crawler.py
+++ b/crawlers/topic_crawler.py
@@ -273,10 +273,24 @@ def run(
     since = crawl_state.get("last_discovery_at") if mode in ("incremental", "discover") else None
     discovered = _discover_topic_repos(session, limit=1000, since=since)
 
-    # Bulk-fetch metadata for all discovered repos via a single GraphQL batch.
-    # Each chunk of ≤100 repos costs one GraphQL POST (separate 5k-point/hr pool).
-    # Repos absent from the result fall back to the per-repo cached REST path below.
-    batch_meta = fetch_repo_metadata_batch(session, discovered)
+    # Pre-filter discovered repos before the GraphQL batch to avoid wasting quota on
+    # repos that will be skipped anyway (already_covered or in filter_cache).
+    to_process = []
+    for full_name in discovered:
+        repo_url = f"https://github.com/{full_name}"
+        canon_url = repo_url.lower()
+        if canon_url in already_covered:
+            log.debug("Skipping %s: already covered by another crawler", full_name)
+            continue
+        if canon_url in filter_cache or repo_url in filter_cache:
+            log.debug("Skipping %s: in filter cache", full_name)
+            continue
+        to_process.append(full_name)
+
+    # Bulk-fetch metadata only for repos that will actually be processed via a single
+    # GraphQL batch. Each chunk of ≤100 repos costs one GraphQL POST (separate
+    # 5k-point/hr pool). Repos absent from the result fall back to per-repo REST below.
+    batch_meta = fetch_repo_metadata_batch(session, to_process)
 
     # Cache (meta, skill_md_paths) per repo to avoid repeated API calls
     _repo_cache: dict[str, tuple[dict, list[str]]] = {}
@@ -286,23 +300,16 @@ def run(
 
     records: list[dict] = []
 
-    for full_name in discovered:
+    truncated = False
+    had_failure = False
+
+    for full_name in to_process:
         if limit is not None and len(records) >= limit:
             log.info("Reached limit of %d records; stopping.", limit)
+            truncated = True
             break
 
         repo_url = f"https://github.com/{full_name}"
-        canon_url = repo_url.lower()
-
-        # Skip repos already covered by other crawlers
-        if canon_url in already_covered:
-            log.debug("Skipping %s: already covered by another crawler", full_name)
-            continue
-
-        # Skip repos with no SKILL.md (from filter cache)
-        if canon_url in filter_cache or repo_url in filter_cache:
-            log.debug("Skipping %s: in filter cache", full_name)
-            continue
 
         # Fetch metadata + SKILL.md paths
         if full_name not in _repo_cache:
@@ -313,6 +320,7 @@ def run(
                 )
             except RuntimeError as exc:
                 log.warning("Could not fetch metadata for %s: %s", full_name, exc)
+                had_failure = True
                 continue
             skill_md_paths = find_skill_md_paths_cached(
                 session, full_name, meta.get("pushed_at", ""), tree_cache
@@ -389,12 +397,11 @@ def run(
     save_content_cache(content_cache, "data/crawl_state/content_cache.json")
     save_tree_cache(tree_cache, "data/crawl_state/tree_cache.json")
 
-    # Persist discovery timestamp so next incremental/discover run can filter by it.
-    # Use run_started (not now) so repos pushed *during* this crawl aren't missed.
-    # Only advance the window when discovery actually returned repos: a transient
-    # failure (rate-limit/empty search) must NOT push last_discovery_at forward, or
-    # the next incremental run would silently skip repos changed in the gap.
-    if discovered:
+    # Advance the discovery window only after a complete, clean sweep. A --limit
+    # truncation or a per-repo failure leaves discovered repos unprocessed; advancing
+    # past them would skip them next run (they're excluded by pushed:>run_started).
+    # Periodic full-mode crawls are the backstop for anything missed.
+    if discovered and not truncated and not had_failure:
         crawl_state["last_discovery_at"] = run_started
         save_crawl_state(crawl_state, "topic")
 
diff --git a/tests/crawlers/test_topic_crawler.py b/tests/crawlers/test_topic_crawler.py
index 9265a60..e46b455 100644
--- a/tests/crawlers/test_topic_crawler.py
+++ b/tests/crawlers/test_topic_crawler.py
@@ -546,3 +546,147 @@ def test_batch_result_used_and_rest_not_called(self, tmp_path):
         import json as _json
         record = _json.loads(Path(out).read_text().strip())
         assert record["raw_metadata"]["stars"] == 99
+
+
+# ---------------------------------------------------------------------------
+# P2 Fix tests: batch pre-filtering and clean-sweep watermark
+# ---------------------------------------------------------------------------
+
+class TestBatchPreFilter:
+    """P2 #1: batch should only fetch metadata for repos that survive skip filters."""
+
+    def test_batch_excludes_already_covered_and_filtered(self, tmp_path):
+        """fetch_repo_metadata_batch should only be called with repos not in
+        already_covered or filter_cache."""
+        from crawlers.topic_crawler import run
+
+        with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch") as mock_batch, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
+             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
+             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_meta_cache"), \
+             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
+             patch("crawlers.topic_crawler.save_crawl_state"), \
+             patch("crawlers.topic_crawler._load_existing_repo_urls",
+                   return_value={"https://github.com/u/a"}), \
+             patch("crawlers.topic_crawler.load_filter_cache",
+                   return_value={"https://github.com/u/b"}):
+
+            mock_disc.return_value = ["u/a", "u/b", "u/c"]
+            mock_batch.return_value = {}
+            mock_meta.return_value = _mock_meta()
+            mock_paths.return_value = {}
+            mock_skill_md.return_value = None
+
+            out = str(tmp_path / "out.jsonl")
+            run(out, filter_cache_path="fake_filter_cache.json",
+                existing_raw_dirs=["fake_raw_dir"])
+
+        # batch must be called exactly once
+        mock_batch.assert_called_once()
+        called_names = mock_batch.call_args.args[1]
+        assert "u/c" in called_names, f"Expected u/c in batch call, got {called_names}"
+        assert "u/a" not in called_names, f"u/a (already_covered) must be excluded"
+        assert "u/b" not in called_names, f"u/b (filter_cache) must be excluded"
+
+
+class TestWatermarkAdvancement:
+    """P2 #2: last_discovery_at should advance only after a complete, clean sweep."""
+
+    def test_limit_truncation_does_not_advance_window(self, tmp_path):
+        """When limit is hit (truncated=True), save_crawl_state must NOT be called."""
+        from crawlers.topic_crawler import run
+
+        with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
+             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
+             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_meta_cache"), \
+             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state",
+                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
+             patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
+
+            # 3 repos each with a skill, but limit=1 so truncated after first
+            mock_disc.return_value = ["user/skill-0", "user/skill-1", "user/skill-2"]
+            mock_meta.return_value = _mock_meta()
+            mock_paths.return_value = {"SKILL.md": "sha1"}
+            mock_skill_md.return_value = SAMPLE_SKILL_MD
+
+            out = str(tmp_path / "out.jsonl")
+            run(out, limit=1)
+
+        mock_save_state.assert_not_called()
+
+    def test_per_repo_failure_does_not_advance_window(self, tmp_path):
+        """When fetch_repo_metadata_cached raises RuntimeError, save_crawl_state must
+        NOT be called (had_failure=True)."""
+        from crawlers.topic_crawler import run
+
+        with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
+             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
+             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_meta_cache"), \
+             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state",
+                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
+             patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
+
+            mock_disc.return_value = ["user/skill-a"]
+            # batch returns empty so REST fallback is triggered; REST raises
+            mock_meta.side_effect = RuntimeError("API failure")
+            mock_paths.return_value = {}
+            mock_skill_md.return_value = None
+
+            out = str(tmp_path / "out.jsonl")
+            run(out)
+
+        mock_save_state.assert_not_called()
+
+    def test_clean_full_sweep_advances_window(self, tmp_path):
+        """A complete run with no truncation or failures MUST advance last_discovery_at."""
+        from crawlers.topic_crawler import run
+
+        with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
+             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
+             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_meta_cache"), \
+             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state",
+                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
+             patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
+
+            mock_disc.return_value = ["user/skill-a"]
+            mock_meta.return_value = _mock_meta()
+            mock_paths.return_value = {"SKILL.md": "sha1"}
+            mock_skill_md.return_value = SAMPLE_SKILL_MD
+
+            out = str(tmp_path / "out.jsonl")
+            run(out, mode="discover")
+
+        mock_save_state.assert_called_once()
+        saved_state = mock_save_state.call_args.args[0]
+        assert "last_discovery_at" in saved_state

From 8653d75ca1482c788134c387d89293a4c5a1203f Mon Sep 17 00:00:00 2001
From: yya007 <yaoyuang1996@gmail.com>
Date: Mon, 22 Jun 2026 03:09:33 -0700
Subject: [PATCH 07/13] fix(topic-crawler): don't advance watermark after
 incomplete discovery

codex review (P2): _discover_topic_repos swallows a per-query search
RuntimeError and returns a partial list; advancing last_discovery_at then
skips repos from the failed query next run. _discover_topic_repos now
returns (repos, discovery_complete) and the watermark only advances when
discovery completed cleanly (in addition to the existing
not-truncated/not-failed guards).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 crawlers/topic_crawler.py            | 24 ++++++----
 tests/crawlers/test_topic_crawler.py | 72 ++++++++++++++++++++--------
 2 files changed, 68 insertions(+), 28 deletions(-)

diff --git a/crawlers/topic_crawler.py b/crawlers/topic_crawler.py
index a3ee0bc..068922d 100644
--- a/crawlers/topic_crawler.py
+++ b/crawlers/topic_crawler.py
@@ -100,11 +100,14 @@
 # Discovery
 # ---------------------------------------------------------------------------
 
-def _discover_topic_repos(session, limit: int = 1000, since: str | None = None) -> list[str]:
+def _discover_topic_repos(
+    session, limit: int = 1000, since: str | None = None
+) -> tuple[list[str], bool]:
     """Search GitHub for repos matching TOPIC_QUERIES.
 
     Paginates each query (up to 1000 results per query as GitHub allows).
-    Returns a deduplicated list of "{owner}/{repo}" full names, at most `limit`.
+    Returns a deduplicated list of "{owner}/{repo}" full names, at most `limit`,
+    and a boolean indicating whether all queries completed without error.
 
     Args:
         session: A requests.Session from make_session().
@@ -115,10 +118,13 @@ def _discover_topic_repos(session, limit: int = 1000, since: str | None = None)
                  incremental/discover re-runs.
 
     Returns:
-        Deduplicated list of full repo names.
+        Tuple of (deduplicated list of full repo names, discovery_complete).
+        discovery_complete is False if any query raised a RuntimeError, meaning
+        the result set is partial and the caller must not advance its watermark.
     """
     seen: set[str] = set()
     results: list[str] = []
+    discovery_complete = True
 
     for query in TOPIC_QUERIES:
         effective_query = f"{query} pushed:>{since}" if since else query
@@ -132,6 +138,7 @@ def _discover_topic_repos(session, limit: int = 1000, since: str | None = None)
                 )
             except RuntimeError as exc:
                 log.warning("Topic repo search failed for %r (page %d): %s", effective_query, page, exc)
+                discovery_complete = False
                 break
 
             items = data.get("items", [])
@@ -149,7 +156,7 @@ def _discover_topic_repos(session, limit: int = 1000, since: str | None = None)
             page += 1
 
     log.info("Topic discovery: %d unique repos found across %d queries", len(results), len(TOPIC_QUERIES))
-    return results[:limit]
+    return results[:limit], discovery_complete
 
 
 def _load_existing_repo_urls(raw_dirs: list[str]) -> set[str]:
@@ -271,7 +278,7 @@ def run(
 
     # Discover repos via topic search, using date-filter on incremental/discover runs
     since = crawl_state.get("last_discovery_at") if mode in ("incremental", "discover") else None
-    discovered = _discover_topic_repos(session, limit=1000, since=since)
+    discovered, discovery_complete = _discover_topic_repos(session, limit=1000, since=since)
 
     # Pre-filter discovered repos before the GraphQL batch to avoid wasting quota on
     # repos that will be skipped anyway (already_covered or in filter_cache).
@@ -398,10 +405,11 @@ def run(
     save_tree_cache(tree_cache, "data/crawl_state/tree_cache.json")
 
     # Advance the discovery window only after a complete, clean sweep. A --limit
-    # truncation or a per-repo failure leaves discovered repos unprocessed; advancing
-    # past them would skip them next run (they're excluded by pushed:>run_started).
+    # truncation, a per-repo failure, or an incomplete topic search (discovery_complete=False,
+    # meaning at least one query hit a RuntimeError) leaves discovered repos unprocessed;
+    # advancing past them would skip them next run (they're excluded by pushed:>run_started).
     # Periodic full-mode crawls are the backstop for anything missed.
-    if discovered and not truncated and not had_failure:
+    if discovered and discovery_complete and not truncated and not had_failure:
         crawl_state["last_discovery_at"] = run_started
         save_crawl_state(crawl_state, "topic")
 
diff --git a/tests/crawlers/test_topic_crawler.py b/tests/crawlers/test_topic_crawler.py
index e46b455..9c75c3a 100644
--- a/tests/crawlers/test_topic_crawler.py
+++ b/tests/crawlers/test_topic_crawler.py
@@ -35,7 +35,7 @@ def test_returns_full_names(self):
         session = MagicMock()
         with patch("crawlers.topic_crawler.github_get") as mock_get:
             mock_get.side_effect = [page, empty] * 20
-            result = _discover_topic_repos(session, limit=100)
+            result, _ = _discover_topic_repos(session, limit=100)
 
         assert "user/skill-a" in result
         assert "user/skill-b" in result
@@ -49,7 +49,7 @@ def test_deduplicates_across_queries(self):
         session = MagicMock()
         with patch("crawlers.topic_crawler.github_get") as mock_get:
             mock_get.side_effect = [both, empty] * 20
-            result = _discover_topic_repos(session, limit=100)
+            result, _ = _discover_topic_repos(session, limit=100)
 
         assert result.count("user/shared-skill") == 1
 
@@ -62,7 +62,7 @@ def test_respects_limit(self):
         session = MagicMock()
         with patch("crawlers.topic_crawler.github_get") as mock_get:
             mock_get.side_effect = [big_page, empty] * 20
-            result = _discover_topic_repos(session, limit=5)
+            result, _ = _discover_topic_repos(session, limit=5)
 
         assert len(result) <= 5
 
@@ -72,9 +72,10 @@ def test_handles_api_error_gracefully(self):
         session = MagicMock()
         with patch("crawlers.topic_crawler.github_get") as mock_get:
             mock_get.side_effect = RuntimeError("rate limited")
-            result = _discover_topic_repos(session, limit=10)
+            result, complete = _discover_topic_repos(session, limit=10)
 
         assert result == []
+        assert complete is False
 
 
 # ---------------------------------------------------------------------------
@@ -115,7 +116,7 @@ def test_writes_records_for_discovered_repos(self, tmp_path):
              patch("crawlers.topic_crawler.save_tree_cache"), \
              patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
              patch("crawlers.topic_crawler.save_crawl_state"):
-            mock_disc.return_value = ["user/skill-a", "user/skill-b"]
+            mock_disc.return_value = (["user/skill-a", "user/skill-b"], True)
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
             mock_skill_md.return_value = SAMPLE_SKILL_MD
@@ -141,7 +142,7 @@ def test_output_has_required_fields(self, tmp_path):
              patch("crawlers.topic_crawler.save_tree_cache"), \
              patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
              patch("crawlers.topic_crawler.save_crawl_state"):
-            mock_disc.return_value = ["user/skill-a"]
+            mock_disc.return_value = (["user/skill-a"], True)
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
             mock_skill_md.return_value = SAMPLE_SKILL_MD
@@ -169,7 +170,7 @@ def test_source_tag_is_topic(self, tmp_path):
              patch("crawlers.topic_crawler.save_tree_cache"), \
              patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
              patch("crawlers.topic_crawler.save_crawl_state"):
-            mock_disc.return_value = ["user/skill-a"]
+            mock_disc.return_value = (["user/skill-a"], True)
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
             mock_skill_md.return_value = None
@@ -196,7 +197,7 @@ def test_respects_limit(self, tmp_path):
              patch("crawlers.topic_crawler.save_tree_cache"), \
              patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
              patch("crawlers.topic_crawler.save_crawl_state"):
-            mock_disc.return_value = [f"user/skill-{i}" for i in range(10)]
+            mock_disc.return_value = ([f"user/skill-{i}" for i in range(10)], True)
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
             mock_skill_md.return_value = None
@@ -222,7 +223,7 @@ def test_skips_repos_with_no_skill_md(self, tmp_path):
              patch("crawlers.topic_crawler.save_tree_cache"), \
              patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
              patch("crawlers.topic_crawler.save_crawl_state"):
-            mock_disc.return_value = ["user/no-skill-md", "user/has-skill-md"]
+            mock_disc.return_value = (["user/no-skill-md", "user/has-skill-md"], True)
             mock_meta.return_value = _mock_meta()
             mock_paths.side_effect = [{}, {"SKILL.md": "sha1"}]
             mock_skill_md.return_value = None
@@ -256,7 +257,7 @@ def test_skips_already_covered_repos(self, tmp_path):
              patch("crawlers.topic_crawler.save_tree_cache"), \
              patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
              patch("crawlers.topic_crawler.save_crawl_state"):
-            mock_disc.return_value = ["user/already-covered", "user/new-skill"]
+            mock_disc.return_value = (["user/already-covered", "user/new-skill"], True)
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
             mock_skill_md.return_value = None
@@ -294,7 +295,7 @@ def test_resume_skips_existing_keys(self, tmp_path):
              patch("crawlers.topic_crawler.save_tree_cache"), \
              patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
              patch("crawlers.topic_crawler.save_crawl_state"):
-            mock_disc.return_value = ["user/skill-a", "user/skill-b"]
+            mock_disc.return_value = (["user/skill-a", "user/skill-b"], True)
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
             mock_skill_md.return_value = None
@@ -319,7 +320,7 @@ def test_name_falls_back_to_repo_name_when_no_frontmatter(self, tmp_path):
              patch("crawlers.topic_crawler.save_tree_cache"), \
              patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
              patch("crawlers.topic_crawler.save_crawl_state"):
-            mock_disc.return_value = ["user/my-cool-skill"]
+            mock_disc.return_value = (["user/my-cool-skill"], True)
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
             mock_skill_md.return_value = None  # no frontmatter
@@ -367,7 +368,7 @@ def test_topic_crawl_uses_caches(self, tmp_path, monkeypatch):
             lambda s, r, p, c: {"SKILL.md": "sha1"},
         )
         monkeypatch.setattr(tc, "fetch_skill_md_cached", lambda *a, **k: "---\nname: t\n---")
-        monkeypatch.setattr(tc, "_discover_topic_repos", lambda s, limit=1000, since=None: ["user/skill-a"])
+        monkeypatch.setattr(tc, "_discover_topic_repos", lambda s, limit=1000, since=None: (["user/skill-a"], True))
         monkeypatch.setattr(tc, "load_crawl_state", lambda p: {})
         monkeypatch.setattr(tc, "save_crawl_state", lambda state, p: None)
 
@@ -448,7 +449,7 @@ def test_run_uses_and_saves_discovery_state(self, tmp_path):
                    return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
              patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
 
-            mock_disc.return_value = ["user/skill-a"]  # non-empty: a successful discovery
+            mock_disc.return_value = (["user/skill-a"], True)  # non-empty: a successful discovery
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {}
             mock_skill_md.return_value = None
@@ -474,7 +475,7 @@ def test_empty_discovery_does_not_advance_window(self, tmp_path):
         advance last_discovery_at, or the next run would skip the missed window."""
         from crawlers.topic_crawler import run
 
-        with patch("crawlers.topic_crawler._discover_topic_repos", return_value=[]), \
+        with patch("crawlers.topic_crawler._discover_topic_repos", return_value=([], True)), \
              patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
              patch("crawlers.topic_crawler.fetch_repo_metadata_cached"), \
              patch("crawlers.topic_crawler.find_skill_md_paths_cached"), \
@@ -531,7 +532,7 @@ def test_batch_result_used_and_rest_not_called(self, tmp_path):
              patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
              patch("crawlers.topic_crawler.save_crawl_state"):
 
-            mock_disc.return_value = ["user/skill-a"]
+            mock_disc.return_value = (["user/skill-a"], True)
             mock_paths.return_value = {"SKILL.md": "sha1"}
             mock_skill_md.return_value = None
 
@@ -578,7 +579,7 @@ def test_batch_excludes_already_covered_and_filtered(self, tmp_path):
              patch("crawlers.topic_crawler.load_filter_cache",
                    return_value={"https://github.com/u/b"}):
 
-            mock_disc.return_value = ["u/a", "u/b", "u/c"]
+            mock_disc.return_value = (["u/a", "u/b", "u/c"], True)
             mock_batch.return_value = {}
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {}
@@ -619,7 +620,7 @@ def test_limit_truncation_does_not_advance_window(self, tmp_path):
              patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
 
             # 3 repos each with a skill, but limit=1 so truncated after first
-            mock_disc.return_value = ["user/skill-0", "user/skill-1", "user/skill-2"]
+            mock_disc.return_value = (["user/skill-0", "user/skill-1", "user/skill-2"], True)
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
             mock_skill_md.return_value = SAMPLE_SKILL_MD
@@ -649,7 +650,7 @@ def test_per_repo_failure_does_not_advance_window(self, tmp_path):
                    return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
              patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
 
-            mock_disc.return_value = ["user/skill-a"]
+            mock_disc.return_value = (["user/skill-a"], True)
             # batch returns empty so REST fallback is triggered; REST raises
             mock_meta.side_effect = RuntimeError("API failure")
             mock_paths.return_value = {}
@@ -679,7 +680,7 @@ def test_clean_full_sweep_advances_window(self, tmp_path):
                    return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
              patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
 
-            mock_disc.return_value = ["user/skill-a"]
+            mock_disc.return_value = (["user/skill-a"], True)
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
             mock_skill_md.return_value = SAMPLE_SKILL_MD
@@ -690,3 +691,34 @@ def test_clean_full_sweep_advances_window(self, tmp_path):
         mock_save_state.assert_called_once()
         saved_state = mock_save_state.call_args.args[0]
         assert "last_discovery_at" in saved_state
+
+    def test_incomplete_discovery_does_not_advance_window(self, tmp_path):
+        """When _discover_topic_repos returns discovery_complete=False (partial result
+        due to a per-query RuntimeError), save_crawl_state must NOT be called.
+        Advancing last_discovery_at on incomplete discovery would skip repos from the
+        failed query on the next incremental run."""
+        from crawlers.topic_crawler import run
+
+        with patch("crawlers.topic_crawler._discover_topic_repos",
+                   return_value=(["user/skill-a"], False)), \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
+             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
+             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_meta_cache"), \
+             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state",
+                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
+             patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
+
+            mock_meta.return_value = _mock_meta()
+            mock_paths.return_value = {}
+            mock_skill_md.return_value = None
+
+            run(str(tmp_path / "out.jsonl"), mode="discover")
+
+        mock_save_state.assert_not_called()

From dfeaa6c4cf074dfb796504d949ec6fbb62444822 Mon Sep 17 00:00:00 2001
From: yya007 <yaoyuang1996@gmail.com>
Date: Mon, 22 Jun 2026 03:20:29 -0700
Subject: [PATCH 08/13] fix(topic-crawler): discover mode must append;
 lazy-chunk GraphQL batch
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

codex review: (P1) discover mode date-filters discovery to only new repos
but left resume=False, so write_jsonl truncated the existing topic corpus
to the new subset — enable resume/append for discover like incremental.
(P2) the GraphQL metadata batch eagerly fetched every to_process repo before
the --limit check; fetch lazily in chunks of 100 as the loop consumes repos.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 crawlers/topic_crawler.py            |  25 ++++--
 tests/crawlers/test_topic_crawler.py | 126 +++++++++++++++++++++++++++
 2 files changed, 144 insertions(+), 7 deletions(-)

diff --git a/crawlers/topic_crawler.py b/crawlers/topic_crawler.py
index 068922d..0af91a1 100644
--- a/crawlers/topic_crawler.py
+++ b/crawlers/topic_crawler.py
@@ -229,8 +229,10 @@ def run(
     Returns:
         Number of new records written.
     """
-    # Resolve mode: incremental aliases resume behaviour
-    if mode == "incremental":
+    # Resolve mode: incremental and discover both date-filter discovery to a partial
+    # (new/changed) set, so they must APPEND to and dedup against the existing output,
+    # never rewrite it. Only full mode rewrites.
+    if mode in ("incremental", "discover"):
         resume = True
     import json as _json
 
@@ -294,10 +296,12 @@ def run(
             continue
         to_process.append(full_name)
 
-    # Bulk-fetch metadata only for repos that will actually be processed via a single
-    # GraphQL batch. Each chunk of ≤100 repos costs one GraphQL POST (separate
-    # 5k-point/hr pool). Repos absent from the result fall back to per-repo REST below.
-    batch_meta = fetch_repo_metadata_batch(session, to_process)
+    # Lazily batch-fetch metadata in chunks of 100 as the loop consumes repos.
+    # A small --limit should not pay GraphQL for repos that are never processed.
+    # Each chunk of ≤100 repos costs one GraphQL POST (separate 5k-point/hr pool).
+    # Repos absent from any chunk's result fall back to per-repo REST below.
+    batch_meta: dict = {}
+    _batched_upto = 0
 
     # Cache (meta, skill_md_paths) per repo to avoid repeated API calls
     _repo_cache: dict[str, tuple[dict, list[str]]] = {}
@@ -310,12 +314,19 @@ def run(
     truncated = False
     had_failure = False
 
-    for full_name in to_process:
+    for idx, full_name in enumerate(to_process):
         if limit is not None and len(records) >= limit:
             log.info("Reached limit of %d records; stopping.", limit)
             truncated = True
             break
 
+        # Lazily batch-fetch metadata in chunks of 100 as the loop reaches them, so a
+        # small --limit doesn't pay GraphQL for repos that are never processed.
+        if idx >= _batched_upto:
+            chunk = to_process[_batched_upto:_batched_upto + 100]
+            batch_meta.update(fetch_repo_metadata_batch(session, chunk))
+            _batched_upto += 100
+
         repo_url = f"https://github.com/{full_name}"
 
         # Fetch metadata + SKILL.md paths
diff --git a/tests/crawlers/test_topic_crawler.py b/tests/crawlers/test_topic_crawler.py
index 9c75c3a..0a4dc76 100644
--- a/tests/crawlers/test_topic_crawler.py
+++ b/tests/crawlers/test_topic_crawler.py
@@ -722,3 +722,129 @@ def test_incomplete_discovery_does_not_advance_window(self, tmp_path):
             run(str(tmp_path / "out.jsonl"), mode="discover")
 
         mock_save_state.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# P1 fix: discover mode must append, not truncate
+# ---------------------------------------------------------------------------
+
+class TestDiscoverModeAppend:
+    """P1: discover mode date-filters to only new repos, so it MUST append."""
+
+    def test_discover_mode_appends_not_truncates(self, tmp_path):
+        """discover mode must call write_jsonl with append=True."""
+        from crawlers.topic_crawler import run
+
+        with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
+             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
+             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_meta_cache"), \
+             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
+             patch("crawlers.topic_crawler.save_crawl_state"), \
+             patch("crawlers.topic_crawler.write_jsonl") as mock_write:
+
+            mock_disc.return_value = (["user/skill-a"], True)
+            mock_meta.return_value = _mock_meta()
+            mock_paths.return_value = {"SKILL.md": "sha1"}
+            mock_skill_md.return_value = None
+            mock_write.return_value = 1
+
+            out = str(tmp_path / "out.jsonl")
+            run(out, mode="discover")
+
+        mock_write.assert_called_once()
+        # append must be True for discover mode
+        call = mock_write.call_args
+        append_val = call.kwargs.get("append") if call.kwargs.get("append") is not None else call.args[2]
+        assert append_val is True, f"Expected append=True for discover mode, got: {call}"
+
+    def test_full_mode_does_not_append(self, tmp_path):
+        """full mode must call write_jsonl with append=False (rewrites)."""
+        from crawlers.topic_crawler import run
+
+        with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
+             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
+             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_meta_cache"), \
+             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
+             patch("crawlers.topic_crawler.save_crawl_state"), \
+             patch("crawlers.topic_crawler.write_jsonl") as mock_write:
+
+            mock_disc.return_value = (["user/skill-a"], True)
+            mock_meta.return_value = _mock_meta()
+            mock_paths.return_value = {"SKILL.md": "sha1"}
+            mock_skill_md.return_value = None
+            mock_write.return_value = 1
+
+            out = str(tmp_path / "out.jsonl")
+            run(out, mode="full")
+
+        mock_write.assert_called_once()
+        call = mock_write.call_args
+        append_val = call.kwargs.get("append") if call.kwargs.get("append") is not None else call.args[2]
+        assert append_val is False, f"Expected append=False for full mode, got: {call}"
+
+
+# ---------------------------------------------------------------------------
+# P2 fix: lazy batch-fetch respects --limit
+# ---------------------------------------------------------------------------
+
+class TestLazyBatchRespectsLimit:
+    """P2: GraphQL batch should not eagerly fetch all repos when limit is small."""
+
+    def test_lazy_batch_respects_limit(self, tmp_path):
+        """With limit=1 and 250 repos, fetch_repo_metadata_batch should be called
+        AT MOST twice (one chunk of 100 at most, possibly two if boundary is hit),
+        not 3 chunks for all 250 repos."""
+        from crawlers.topic_crawler import run
+
+        repos_250 = [f"u/r{i}" for i in range(250)]
+
+        with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch") as mock_batch, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
+             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
+             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_meta_cache"), \
+             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
+             patch("crawlers.topic_crawler.save_crawl_state"):
+
+            mock_disc.return_value = (repos_250, True)
+            mock_batch.return_value = {}  # empty -> REST fallback fills meta
+            mock_meta.return_value = _mock_meta()
+            mock_paths.return_value = {"SKILL.md": "sha1"}
+            mock_skill_md.return_value = SAMPLE_SKILL_MD
+
+            out = str(tmp_path / "out.jsonl")
+            run(out, limit=1)
+
+        # With limit=1, only the first chunk (repos 0-99) should be fetched,
+        # so call count must be at most 2 (NOT 3 for all 250 repos).
+        assert mock_batch.call_count <= 2, (
+            f"Expected at most 2 batch calls with limit=1, got {mock_batch.call_count}"
+        )
+        # Each individual call must be a chunk of at most 100 repos (not all 250 at once).
+        for i, c in enumerate(mock_batch.call_args_list):
+            chunk = c.args[1]  # second positional arg is the list of repo names
+            assert len(chunk) <= 100, (
+                f"Batch call {i} sent {len(chunk)} repos — expected ≤100 (lazy chunking)"
+            )

From 29b024c3c6c3d082723591d3d7384ce544c1e792 Mon Sep 17 00:00:00 2001
From: yya007 <yaoyuang1996@gmail.com>
Date: Mon, 22 Jun 2026 03:26:15 -0700
Subject: [PATCH 09/13] fix(topic-crawler): cap=incomplete; advance window on
 complete empty sweep
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

codex review: (P1) hitting the 1000-repo discovery cap left
discovery_complete=True, so a partial capped discovery advanced the
watermark and skipped repos beyond the cap — mark a capped result
incomplete. (P3) a *complete* discovery that finds nothing new now advances
the watermark (the discovery_complete flag already gates transient failures),
so quiet periods don't re-scan the same window forever.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 crawlers/topic_crawler.py            | 20 ++++++++----
 tests/crawlers/test_topic_crawler.py | 49 +++++++++++++++++++++++++---
 2 files changed, 59 insertions(+), 10 deletions(-)

diff --git a/crawlers/topic_crawler.py b/crawlers/topic_crawler.py
index 0af91a1..15dd8c7 100644
--- a/crawlers/topic_crawler.py
+++ b/crawlers/topic_crawler.py
@@ -155,6 +155,12 @@ def _discover_topic_repos(
                 break
             page += 1
 
+    # Hitting the cap means we stopped before enumerating all matches, so coverage
+    # is partial — signal incomplete so the caller won't advance its watermark and
+    # skip repos beyond the cap.
+    if len(results) >= limit:
+        discovery_complete = False
+
     log.info("Topic discovery: %d unique repos found across %d queries", len(results), len(TOPIC_QUERIES))
     return results[:limit], discovery_complete
 
@@ -415,12 +421,14 @@ def run(
     save_content_cache(content_cache, "data/crawl_state/content_cache.json")
     save_tree_cache(tree_cache, "data/crawl_state/tree_cache.json")
 
-    # Advance the discovery window only after a complete, clean sweep. A --limit
-    # truncation, a per-repo failure, or an incomplete topic search (discovery_complete=False,
-    # meaning at least one query hit a RuntimeError) leaves discovered repos unprocessed;
-    # advancing past them would skip them next run (they're excluded by pushed:>run_started).
-    # Periodic full-mode crawls are the backstop for anything missed.
-    if discovered and discovery_complete and not truncated and not had_failure:
+    # Advance the discovery window only after a complete, clean sweep.
+    # discovery_complete is False if a topic search query errored OR hit the result
+    # cap; truncated means --limit stopped processing; had_failure means a per-repo
+    # fetch failed. Any of those leaves repos unprocessed, and advancing past them
+    # would skip them next run (excluded by pushed:>run_started). A *complete* run
+    # that simply found nothing new still advances (quiet periods shouldn't re-scan
+    # the same window forever). Periodic full-mode crawls are the backstop.
+    if discovery_complete and not truncated and not had_failure:
         crawl_state["last_discovery_at"] = run_started
         save_crawl_state(crawl_state, "topic")
 
diff --git a/tests/crawlers/test_topic_crawler.py b/tests/crawlers/test_topic_crawler.py
index 0a4dc76..bb3aed5 100644
--- a/tests/crawlers/test_topic_crawler.py
+++ b/tests/crawlers/test_topic_crawler.py
@@ -62,9 +62,23 @@ def test_respects_limit(self):
         session = MagicMock()
         with patch("crawlers.topic_crawler.github_get") as mock_get:
             mock_get.side_effect = [big_page, empty] * 20
-            result, _ = _discover_topic_repos(session, limit=5)
+            result, complete = _discover_topic_repos(session, limit=5)
 
         assert len(result) <= 5
+        # Hitting the cap means coverage is partial → incomplete.
+        assert complete is False
+
+    def test_complete_when_under_cap_and_no_errors(self):
+        from crawlers.topic_crawler import _discover_topic_repos
+
+        page = {"items": [{"full_name": "user/skill-a"}]}
+        empty = {"items": []}
+        session = MagicMock()
+        with patch("crawlers.topic_crawler.github_get") as mock_get:
+            mock_get.side_effect = [page, empty] * 30
+            result, complete = _discover_topic_repos(session, limit=1000)
+
+        assert complete is True
 
     def test_handles_api_error_gracefully(self):
         from crawlers.topic_crawler import _discover_topic_repos
@@ -470,9 +484,11 @@ def test_run_uses_and_saves_discovery_state(self, tmp_path):
         assert "last_discovery_at" in saved_state
         assert saved_state["last_discovery_at"]  # non-empty timestamp
 
-    def test_empty_discovery_does_not_advance_window(self, tmp_path):
-        """A discovery that returns no repos (e.g. transient rate-limit) must NOT
-        advance last_discovery_at, or the next run would skip the missed window."""
+    def test_complete_empty_discovery_advances_window(self, tmp_path):
+        """A discovery that COMPLETES cleanly but finds nothing new still advances
+        last_discovery_at — a quiet period must not re-scan the same window forever.
+        (The transient-failure / incomplete case is covered by discovery_complete=False
+        in test_incomplete_discovery_does_not_advance_window.)"""
         from crawlers.topic_crawler import run
 
         with patch("crawlers.topic_crawler._discover_topic_repos", return_value=([], True)), \
@@ -492,6 +508,31 @@ def test_empty_discovery_does_not_advance_window(self, tmp_path):
 
             run(str(tmp_path / "out.jsonl"), mode="discover")
 
+        mock_save_state.assert_called_once()
+
+    def test_capped_discovery_does_not_advance_window(self, tmp_path):
+        """A discovery that hit the result cap is incomplete (more matches exist),
+        so the watermark must NOT advance even though repos were found."""
+        from crawlers.topic_crawler import run
+
+        with patch("crawlers.topic_crawler._discover_topic_repos",
+                   return_value=(["user/skill-a"], False)), \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_cached", return_value=_mock_meta()), \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached", return_value={}), \
+             patch("crawlers.topic_crawler.fetch_skill_md_cached"), \
+             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_meta_cache"), \
+             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state",
+                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
+             patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
+
+            run(str(tmp_path / "out.jsonl"), mode="discover")
+
         mock_save_state.assert_not_called()
 
 

From 4421c4fd67fc57532a79d0888ab07efa20bf90aa Mon Sep 17 00:00:00 2001
From: yya007 <yaoyuang1996@gmail.com>
Date: Mon, 22 Jun 2026 03:33:27 -0700
Subject: [PATCH 10/13] fix(crawlers): don't cache empty tree lookups; save
 watermark after write
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

codex review: (P2) find_skill_md_paths_cached cached a {} result, but
find_skill_md_paths returns {} for both "no SKILL.md" and a transient Trees
failure — pinning a repo that has skills to empty forever. Only cache
non-empty results (genuine empties are handled by the filter cache).
(P2) the discovery watermark was saved before write_jsonl; a failed/killed
write would advance past unwritten repos. Save it only after a successful write.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 crawlers/base.py                     |  7 ++++++-
 crawlers/topic_crawler.py            | 21 ++++++++++++---------
 tests/crawlers/test_base.py          | 12 ++++++++++++
 tests/crawlers/test_topic_crawler.py | 28 ++++++++++++++++++++++++++++
 4 files changed, 58 insertions(+), 10 deletions(-)

diff --git a/crawlers/base.py b/crawlers/base.py
index a252ff4..72dcd07 100644
--- a/crawlers/base.py
+++ b/crawlers/base.py
@@ -597,7 +597,12 @@ def find_skill_md_paths_cached(
         return tree_cache[repo_full_name]["paths"]
 
     paths = find_skill_md_paths(session, repo_full_name)
-    if pushed_at:
+    # Only cache a NON-empty result. find_skill_md_paths returns {} both for a
+    # repo with no SKILL.md and for a transient Trees/Search API failure; caching
+    # {} under the unchanged pushed_at would pin a repo that actually has skills to
+    # "empty" forever (it never re-fetches until pushed again). Genuinely empty
+    # repos are cheaply re-checked and short-circuited by the crawler's filter cache.
+    if pushed_at and paths:
         tree_cache[repo_full_name] = {"pushed_at": pushed_at, "paths": paths}
     return paths
 
diff --git a/crawlers/topic_crawler.py b/crawlers/topic_crawler.py
index 15dd8c7..4703d7a 100644
--- a/crawlers/topic_crawler.py
+++ b/crawlers/topic_crawler.py
@@ -421,19 +421,22 @@ def run(
     save_content_cache(content_cache, "data/crawl_state/content_cache.json")
     save_tree_cache(tree_cache, "data/crawl_state/tree_cache.json")
 
-    # Advance the discovery window only after a complete, clean sweep.
-    # discovery_complete is False if a topic search query errored OR hit the result
-    # cap; truncated means --limit stopped processing; had_failure means a per-repo
-    # fetch failed. Any of those leaves repos unprocessed, and advancing past them
-    # would skip them next run (excluded by pushed:>run_started). A *complete* run
-    # that simply found nothing new still advances (quiet periods shouldn't re-scan
-    # the same window forever). Periodic full-mode crawls are the backstop.
+    written = write_jsonl(records, output_path, append=resume)
+    log.info("Topic crawler done: %d records written to %s", written, output_path)
+
+    # Advance the discovery window only AFTER the records are durably written, and
+    # only after a complete, clean sweep. discovery_complete is False if a topic
+    # search query errored OR hit the result cap; truncated means --limit stopped
+    # processing; had_failure means a per-repo fetch failed. Any of those leaves
+    # repos unprocessed, and advancing past them would skip them next run (excluded
+    # by pushed:>run_started). Saving the watermark before write_jsonl would, on a
+    # failed/killed write, advance past records that were never persisted. A
+    # *complete* run that found nothing new still advances (quiet periods shouldn't
+    # re-scan the same window forever). Periodic full-mode crawls are the backstop.
     if discovery_complete and not truncated and not had_failure:
         crawl_state["last_discovery_at"] = run_started
         save_crawl_state(crawl_state, "topic")
 
-    written = write_jsonl(records, output_path, append=resume)
-    log.info("Topic crawler done: %d records written to %s", written, output_path)
     return written
 
 
diff --git a/tests/crawlers/test_base.py b/tests/crawlers/test_base.py
index 6ab9aae..ca858c2 100644
--- a/tests/crawlers/test_base.py
+++ b/tests/crawlers/test_base.py
@@ -1089,6 +1089,18 @@ def test_cache_miss_calls_and_stores(self):
             "paths": {"SKILL.md": "sha9"},
         }
 
+    def test_empty_result_not_cached(self):
+        """An empty result (no SKILL.md OR a transient Trees failure, both {}) must
+        NOT be cached — caching it would pin a repo with skills to empty forever."""
+        tree_cache = {}
+        with patch("crawlers.base.find_skill_md_paths", return_value={}) as mock_tree:
+            result = find_skill_md_paths_cached(
+                MagicMock(), "u/r", "2026-02-02T00:00:00Z", tree_cache
+            )
+        assert result == {}
+        mock_tree.assert_called_once()
+        assert tree_cache == {}  # not cached
+
     def test_changed_pushed_at_refetches(self):
         """Stale pushed_at → refetches and updates cache entry."""
         tree_cache = {
diff --git a/tests/crawlers/test_topic_crawler.py b/tests/crawlers/test_topic_crawler.py
index bb3aed5..433d2a9 100644
--- a/tests/crawlers/test_topic_crawler.py
+++ b/tests/crawlers/test_topic_crawler.py
@@ -510,6 +510,34 @@ def test_complete_empty_discovery_advances_window(self, tmp_path):
 
         mock_save_state.assert_called_once()
 
+    def test_watermark_not_saved_when_write_fails(self, tmp_path):
+        """The discovery watermark must be persisted only AFTER write_jsonl succeeds;
+        a failed write must not advance last_discovery_at past unwritten repos."""
+        import pytest
+        from crawlers.topic_crawler import run
+
+        with patch("crawlers.topic_crawler._discover_topic_repos",
+                   return_value=(["user/skill-a"], True)), \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_cached", return_value=_mock_meta()), \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached", return_value={}), \
+             patch("crawlers.topic_crawler.fetch_skill_md_cached"), \
+             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_meta_cache"), \
+             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state",
+                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
+             patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state, \
+             patch("crawlers.topic_crawler.write_jsonl", side_effect=OSError("disk full")):
+
+            with pytest.raises(OSError):
+                run(str(tmp_path / "out.jsonl"), mode="discover")
+
+        mock_save_state.assert_not_called()
+
     def test_capped_discovery_does_not_advance_window(self, tmp_path):
         """A discovery that hit the result cap is incomplete (more matches exist),
         so the watermark must NOT advance even though repos were found."""

From 7ecdb9df104b118513fafea42e2b72a802a5166a Mon Sep 17 00:00:00 2001
From: yya007 <yaoyuang1996@gmail.com>
Date: Mon, 22 Jun 2026 12:51:51 -0700
Subject: [PATCH 11/13] fix(topic-crawler): incomplete_results + inner-loop
 limit mark run partial

codex review: (P2) a timed-out topic search returns HTTP 200 with
incomplete_results=true and a partial page; mark discovery incomplete so the
watermark won't advance past the omitted repos. (P2) --limit hit inside a
repo's inner SKILL.md loop left truncated=False (only the outer loop set it),
so the watermark could advance with that repo's remaining skills unwritten;
set truncated in the inner break too.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 crawlers/topic_crawler.py            |  9 ++++++
 tests/crawlers/test_topic_crawler.py | 45 ++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)

diff --git a/crawlers/topic_crawler.py b/crawlers/topic_crawler.py
index 4703d7a..69e4173 100644
--- a/crawlers/topic_crawler.py
+++ b/crawlers/topic_crawler.py
@@ -141,6 +141,12 @@ def _discover_topic_repos(
                 discovery_complete = False
                 break
 
+            # A timed-out search returns HTTP 200 with incomplete_results=true and a
+            # partial page; the omitted repos would be excluded by the next run's
+            # pushed:> filter, so treat this as an incomplete discovery.
+            if data.get("incomplete_results"):
+                discovery_complete = False
+
             items = data.get("items", [])
             if not items:
                 break
@@ -365,6 +371,9 @@ def run(
 
         for skill_path in skill_md_paths:
             if limit is not None and len(records) >= limit:
+                # Hitting the limit mid-repo leaves this repo's remaining SKILL.md
+                # files unwritten; mark truncated so the watermark won't advance.
+                truncated = True
                 break
 
             skill_md_url = f"{repo_url}/blob/{default_branch}/{skill_path}"
diff --git a/tests/crawlers/test_topic_crawler.py b/tests/crawlers/test_topic_crawler.py
index 433d2a9..8037ce5 100644
--- a/tests/crawlers/test_topic_crawler.py
+++ b/tests/crawlers/test_topic_crawler.py
@@ -91,6 +91,21 @@ def test_handles_api_error_gracefully(self):
         assert result == []
         assert complete is False
 
+    def test_incomplete_results_marks_incomplete(self):
+        """A timed-out search (HTTP 200, incomplete_results=true) → discovery incomplete."""
+        from crawlers.topic_crawler import _discover_topic_repos
+
+        partial = {"items": [{"full_name": "user/skill-a"}], "incomplete_results": True}
+        empty = {"items": [], "incomplete_results": False}
+        session = MagicMock()
+        with patch("crawlers.topic_crawler.github_get") as mock_get:
+            mock_get.side_effect = [partial, empty] * 30
+            result, complete = _discover_topic_repos(session, limit=1000)
+
+        assert "user/skill-a" in result
+        assert complete is False
+        assert complete is False
+
 
 # ---------------------------------------------------------------------------
 # TestTopicCrawlerRun
@@ -699,6 +714,36 @@ def test_limit_truncation_does_not_advance_window(self, tmp_path):
 
         mock_save_state.assert_not_called()
 
+    def test_inner_loop_limit_truncation_does_not_advance_window(self, tmp_path):
+        """When --limit is hit INSIDE a repo's skill loop (a single repo with multiple
+        SKILL.md), truncated must still be set so the watermark does not advance."""
+        from crawlers.topic_crawler import run
+
+        with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
+             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
+             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
+             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
+             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_meta_cache"), \
+             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_content_cache"), \
+             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
+             patch("crawlers.topic_crawler.save_tree_cache"), \
+             patch("crawlers.topic_crawler.load_crawl_state",
+                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
+             patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
+
+            # ONE repo with TWO SKILL.md files, limit=1 → truncates inside the inner loop
+            mock_disc.return_value = (["user/multi-skill"], True)
+            mock_meta.return_value = _mock_meta()
+            mock_paths.return_value = {"SKILL.md": "sha1", "sub/SKILL.md": "sha2"}
+            mock_skill_md.return_value = SAMPLE_SKILL_MD
+
+            run(str(tmp_path / "out.jsonl"), limit=1)
+
+        mock_save_state.assert_not_called()
+
     def test_per_repo_failure_does_not_advance_window(self, tmp_path):
         """When fetch_repo_metadata_cached raises RuntimeError, save_crawl_state must
         NOT be called (had_failure=True)."""

From c2c393ca3ff7001d671021cf2aa1c634b321123e Mon Sep 17 00:00:00 2001
From: yya007 <yaoyuang1996@gmail.com>
Date: Mon, 22 Jun 2026 13:18:47 -0700
Subject: [PATCH 12/13] fix(crawlers): drop option 4 (date-filtered topic
 discovery); key tree cache on default branch

Option 4's pushed:>since filter is unsound for topic discovery: a repo that
adds a skill topic without a new commit keeps its old pushed_at, so it is
filtered out and the advancing watermark makes it permanently undiscoverable
(GitHub search has no topic-added-since qualifier). Remove the date filter and
the whole watermark/crawl_state machinery; topic discovery searches fully each
run (cheap via the downstream metadata/tree/content caches, not via narrowed
discovery). Keep options 5 (GraphQL batch) and 6b (tree cache), and fix 6b to
key on (pushed_at, default_branch) so a default-branch change isn't served
stale paths.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 crawlers/base.py                     |  32 +-
 crawlers/topic_crawler.py            |  91 ++---
 tests/crawlers/test_base.py          |  33 +-
 tests/crawlers/test_topic_crawler.py | 511 ++-------------------------
 4 files changed, 99 insertions(+), 568 deletions(-)

diff --git a/crawlers/base.py b/crawlers/base.py
index 72dcd07..84f375c 100644
--- a/crawlers/base.py
+++ b/crawlers/base.py
@@ -570,14 +570,20 @@ def find_skill_md_paths_cached(
     session,
     repo_full_name: str,
     pushed_at: str,
+    default_branch: str,
     tree_cache: dict,
 ) -> dict[str, str]:
     """Return SKILL.md paths for a repo, skipping the Trees API call when unchanged.
 
-    Uses ``pushed_at`` as a freshness key.  If the repo's ``pushed_at`` timestamp
-    matches what is stored in ``tree_cache``, the cached ``{path: sha}`` mapping is
-    returned immediately without any API call.  Otherwise ``find_skill_md_paths`` is
-    called and the result is stored back into ``tree_cache`` (mutated in place).
+    Uses ``(pushed_at, default_branch)`` as the freshness key.  If both match what is
+    stored in ``tree_cache``, the cached ``{path: sha}`` mapping is returned
+    immediately without any API call.  Otherwise ``find_skill_md_paths`` is called and
+    the result is stored back into ``tree_cache`` (mutated in place).
+
+    ``default_branch`` is part of the key because ``find_skill_md_paths`` walks
+    ``git/trees/HEAD`` (the default branch): a default-branch change resolves HEAD to a
+    different tree even when ``pushed_at`` is unchanged, so keying on ``pushed_at``
+    alone would return stale paths/blob SHAs from the old branch.
 
     An empty or falsy ``pushed_at`` always calls the API and never caches the result
     because we cannot prove freshness without a timestamp.
@@ -587,14 +593,20 @@ def find_skill_md_paths_cached(
         repo_full_name: "{owner}/{repo}" string.
         pushed_at:      The repo's ``pushed_at`` ISO-8601 string from the metadata API.
                         Pass ``""`` (or any falsy value) to force a live fetch.
+        default_branch: The repo's current default branch name.
         tree_cache:     Mutable dict that persists across calls within a crawl run.
-                        Shape: ``{repo_full_name: {"pushed_at": str, "paths": dict}}``.
+                        Shape: ``{repo: {"pushed_at": str, "default_branch": str, "paths": dict}}``.
 
     Returns:
         Dict mapping SKILL.md path → blob SHA (same contract as ``find_skill_md_paths``).
     """
-    if pushed_at and tree_cache.get(repo_full_name, {}).get("pushed_at") == pushed_at:
-        return tree_cache[repo_full_name]["paths"]
+    entry = tree_cache.get(repo_full_name, {})
+    if (
+        pushed_at
+        and entry.get("pushed_at") == pushed_at
+        and entry.get("default_branch") == default_branch
+    ):
+        return entry["paths"]
 
     paths = find_skill_md_paths(session, repo_full_name)
     # Only cache a NON-empty result. find_skill_md_paths returns {} both for a
@@ -603,7 +615,11 @@ def find_skill_md_paths_cached(
     # "empty" forever (it never re-fetches until pushed again). Genuinely empty
     # repos are cheaply re-checked and short-circuited by the crawler's filter cache.
     if pushed_at and paths:
-        tree_cache[repo_full_name] = {"pushed_at": pushed_at, "paths": paths}
+        tree_cache[repo_full_name] = {
+            "pushed_at": pushed_at,
+            "default_branch": default_branch,
+            "paths": paths,
+        }
     return paths
 
 
diff --git a/crawlers/topic_crawler.py b/crawlers/topic_crawler.py
index 69e4173..bcd3d23 100644
--- a/crawlers/topic_crawler.py
+++ b/crawlers/topic_crawler.py
@@ -34,7 +34,6 @@
 
 from crawlers.base import (
     GITHUB_API,
-    _utc_now_iso,
     add_to_filter_cache,
     fetch_repo_metadata_batch,
     fetch_repo_metadata_cached,
@@ -43,14 +42,12 @@
     github_get,
     infer_platforms,
     load_content_cache,
-    load_crawl_state,
     load_filter_cache,
     load_meta_cache,
     load_tree_cache,
     make_session,
     parse_frontmatter,
     save_content_cache,
-    save_crawl_state,
     save_meta_cache,
     save_tree_cache,
     write_jsonl,
@@ -100,53 +97,42 @@
 # Discovery
 # ---------------------------------------------------------------------------
 
-def _discover_topic_repos(
-    session, limit: int = 1000, since: str | None = None
-) -> tuple[list[str], bool]:
+def _discover_topic_repos(session, limit: int = 1000) -> list[str]:
     """Search GitHub for repos matching TOPIC_QUERIES.
 
     Paginates each query (up to 1000 results per query as GitHub allows).
-    Returns a deduplicated list of "{owner}/{repo}" full names, at most `limit`,
-    and a boolean indicating whether all queries completed without error.
+    Returns a deduplicated list of "{owner}/{repo}" full names, at most `limit`.
+
+    Note: every run searches all topics in full. Topic membership changes
+    independently of ``pushed_at`` and GitHub search has no "topic-added-since"
+    qualifier, so a date-filtered incremental discovery cannot be correct here —
+    a repo that gains a skill topic without a new commit would be missed. The
+    crawl stays cheap via the metadata/tree/content caches downstream, not by
+    narrowing discovery.
 
     Args:
         session: A requests.Session from make_session().
         limit:   Total maximum unique repos to return across all queries.
-        since:   Optional ISO-8601 timestamp.  When provided, appends
-                 ``pushed:><since>`` to every query so only repos pushed after
-                 that time are returned — dramatically reducing API quota on
-                 incremental/discover re-runs.
 
     Returns:
-        Tuple of (deduplicated list of full repo names, discovery_complete).
-        discovery_complete is False if any query raised a RuntimeError, meaning
-        the result set is partial and the caller must not advance its watermark.
+        Deduplicated list of full repo names, at most `limit`.
     """
     seen: set[str] = set()
     results: list[str] = []
-    discovery_complete = True
 
     for query in TOPIC_QUERIES:
-        effective_query = f"{query} pushed:>{since}" if since else query
         page = 1
         while len(results) < limit:
             try:
                 data = github_get(
                     session,
                     f"{GITHUB_API}/search/repositories",
-                    params={"q": effective_query, "per_page": 100, "page": page},
+                    params={"q": query, "per_page": 100, "page": page},
                 )
             except RuntimeError as exc:
-                log.warning("Topic repo search failed for %r (page %d): %s", effective_query, page, exc)
-                discovery_complete = False
+                log.warning("Topic repo search failed for %r (page %d): %s", query, page, exc)
                 break
 
-            # A timed-out search returns HTTP 200 with incomplete_results=true and a
-            # partial page; the omitted repos would be excluded by the next run's
-            # pushed:> filter, so treat this as an incomplete discovery.
-            if data.get("incomplete_results"):
-                discovery_complete = False
-
             items = data.get("items", [])
             if not items:
                 break
@@ -161,14 +147,8 @@ def _discover_topic_repos(
                 break
             page += 1
 
-    # Hitting the cap means we stopped before enumerating all matches, so coverage
-    # is partial — signal incomplete so the caller won't advance its watermark and
-    # skip repos beyond the cap.
-    if len(results) >= limit:
-        discovery_complete = False
-
     log.info("Topic discovery: %d unique repos found across %d queries", len(results), len(TOPIC_QUERIES))
-    return results[:limit], discovery_complete
+    return results[:limit]
 
 
 def _load_existing_repo_urls(raw_dirs: list[str]) -> set[str]:
@@ -241,17 +221,11 @@ def run(
     Returns:
         Number of new records written.
     """
-    # Resolve mode: incremental and discover both date-filter discovery to a partial
-    # (new/changed) set, so they must APPEND to and dedup against the existing output,
-    # never rewrite it. Only full mode rewrites.
-    if mode in ("incremental", "discover"):
+    # Resolve mode: incremental aliases resume behaviour
+    if mode == "incremental":
         resume = True
     import json as _json
 
-    # Load per-source crawl state for date-filter support
-    crawl_state = load_crawl_state("topic")
-    run_started = _utc_now_iso()
-
     session = make_session(token=token)
 
     # Load ETag metadata cache, blob-SHA content cache, and Trees-path cache
@@ -290,9 +264,8 @@ def run(
                     pass
         log.info("Resume mode: %d skill keys already in output", len(existing_skill_keys))
 
-    # Discover repos via topic search, using date-filter on incremental/discover runs
-    since = crawl_state.get("last_discovery_at") if mode in ("incremental", "discover") else None
-    discovered, discovery_complete = _discover_topic_repos(session, limit=1000, since=since)
+    # Discover repos via topic search (full each run — see _discover_topic_repos note).
+    discovered = _discover_topic_repos(session, limit=1000)
 
     # Pre-filter discovered repos before the GraphQL batch to avoid wasting quota on
     # repos that will be skipped anyway (already_covered or in filter_cache).
@@ -323,13 +296,9 @@ def run(
 
     records: list[dict] = []
 
-    truncated = False
-    had_failure = False
-
     for idx, full_name in enumerate(to_process):
         if limit is not None and len(records) >= limit:
             log.info("Reached limit of %d records; stopping.", limit)
-            truncated = True
             break
 
         # Lazily batch-fetch metadata in chunks of 100 as the loop reaches them, so a
@@ -350,10 +319,13 @@ def run(
                 )
             except RuntimeError as exc:
                 log.warning("Could not fetch metadata for %s: %s", full_name, exc)
-                had_failure = True
                 continue
             skill_md_paths = find_skill_md_paths_cached(
-                session, full_name, meta.get("pushed_at", ""), tree_cache
+                session,
+                full_name,
+                meta.get("pushed_at", ""),
+                meta.get("default_branch", "main"),
+                tree_cache,
             )
             _repo_cache[full_name] = (meta, skill_md_paths)
             if not skill_md_paths:
@@ -371,9 +343,6 @@ def run(
 
         for skill_path in skill_md_paths:
             if limit is not None and len(records) >= limit:
-                # Hitting the limit mid-repo leaves this repo's remaining SKILL.md
-                # files unwritten; mark truncated so the watermark won't advance.
-                truncated = True
                 break
 
             skill_md_url = f"{repo_url}/blob/{default_branch}/{skill_path}"
@@ -432,20 +401,6 @@ def run(
 
     written = write_jsonl(records, output_path, append=resume)
     log.info("Topic crawler done: %d records written to %s", written, output_path)
-
-    # Advance the discovery window only AFTER the records are durably written, and
-    # only after a complete, clean sweep. discovery_complete is False if a topic
-    # search query errored OR hit the result cap; truncated means --limit stopped
-    # processing; had_failure means a per-repo fetch failed. Any of those leaves
-    # repos unprocessed, and advancing past them would skip them next run (excluded
-    # by pushed:>run_started). Saving the watermark before write_jsonl would, on a
-    # failed/killed write, advance past records that were never persisted. A
-    # *complete* run that found nothing new still advances (quiet periods shouldn't
-    # re-scan the same window forever). Periodic full-mode crawls are the backstop.
-    if discovery_complete and not truncated and not had_failure:
-        crawl_state["last_discovery_at"] = run_started
-        save_crawl_state(crawl_state, "topic")
-
     return written
 
 
@@ -481,7 +436,7 @@ def _build_parser() -> argparse.ArgumentParser:
         "--mode",
         choices=["full", "incremental", "metadata", "discover"],
         default="full",
-        help="Crawl mode: full=complete re-crawl, incremental=changed repos only, metadata=stars/ETags only, discover=new repos since last run",
+        help="Crawl mode: full=complete re-crawl, incremental=changed repos only, metadata=stars/ETags only, discover=alias for full discovery",
     )
     p.add_argument(
         "--resume",
diff --git a/tests/crawlers/test_base.py b/tests/crawlers/test_base.py
index ca858c2..5e383d9 100644
--- a/tests/crawlers/test_base.py
+++ b/tests/crawlers/test_base.py
@@ -22,8 +22,6 @@
     save_crawl_state,
     write_jsonl,
     fetch_skill_md_cached,
-    load_content_cache,
-    save_content_cache,
     find_skill_md_paths_cached,
     load_tree_cache,
     save_tree_cache,
@@ -1064,13 +1062,13 @@ def test_load_tree_cache_returns_empty_on_corrupt_file(self, tmp_path):
 
 class TestFindSkillMdPathsCached:
     def test_cache_hit_skips_tree_call(self):
-        """Pre-seeded cache with matching pushed_at → no API call made."""
+        """Pre-seeded cache with matching pushed_at and default_branch → no API call made."""
         tree_cache = {
-            "u/r": {"pushed_at": "2026-01-01T00:00:00Z", "paths": {"SKILL.md": "sha1"}}
+            "u/r": {"pushed_at": "2026-01-01T00:00:00Z", "default_branch": "main", "paths": {"SKILL.md": "sha1"}}
         }
         with patch("crawlers.base.find_skill_md_paths") as mock_tree:
             result = find_skill_md_paths_cached(
-                MagicMock(), "u/r", "2026-01-01T00:00:00Z", tree_cache
+                MagicMock(), "u/r", "2026-01-01T00:00:00Z", "main", tree_cache
             )
         assert result == {"SKILL.md": "sha1"}
         mock_tree.assert_not_called()
@@ -1080,12 +1078,13 @@ def test_cache_miss_calls_and_stores(self):
         tree_cache = {}
         with patch("crawlers.base.find_skill_md_paths", return_value={"SKILL.md": "sha9"}) as mock_tree:
             result = find_skill_md_paths_cached(
-                MagicMock(), "u/r", "2026-02-02T00:00:00Z", tree_cache
+                MagicMock(), "u/r", "2026-02-02T00:00:00Z", "main", tree_cache
             )
         assert result == {"SKILL.md": "sha9"}
         mock_tree.assert_called_once()
         assert tree_cache["u/r"] == {
             "pushed_at": "2026-02-02T00:00:00Z",
+            "default_branch": "main",
             "paths": {"SKILL.md": "sha9"},
         }
 
@@ -1095,7 +1094,7 @@ def test_empty_result_not_cached(self):
         tree_cache = {}
         with patch("crawlers.base.find_skill_md_paths", return_value={}) as mock_tree:
             result = find_skill_md_paths_cached(
-                MagicMock(), "u/r", "2026-02-02T00:00:00Z", tree_cache
+                MagicMock(), "u/r", "2026-02-02T00:00:00Z", "main", tree_cache
             )
         assert result == {}
         mock_tree.assert_called_once()
@@ -1104,12 +1103,12 @@ def test_empty_result_not_cached(self):
     def test_changed_pushed_at_refetches(self):
         """Stale pushed_at → refetches and updates cache entry."""
         tree_cache = {
-            "u/r": {"pushed_at": "2026-01-01T00:00:00Z", "paths": {"SKILL.md": "old_sha"}}
+            "u/r": {"pushed_at": "2026-01-01T00:00:00Z", "default_branch": "main", "paths": {"SKILL.md": "old_sha"}}
         }
         new_paths = {"SKILL.md": "new_sha", "sub/SKILL.md": "abc"}
         with patch("crawlers.base.find_skill_md_paths", return_value=new_paths) as mock_tree:
             result = find_skill_md_paths_cached(
-                MagicMock(), "u/r", "2026-03-15T12:00:00Z", tree_cache
+                MagicMock(), "u/r", "2026-03-15T12:00:00Z", "main", tree_cache
             )
         mock_tree.assert_called_once()
         assert result == new_paths
@@ -1121,12 +1120,26 @@ def test_empty_pushed_at_always_calls_and_does_not_cache(self):
         tree_cache = {}
         with patch("crawlers.base.find_skill_md_paths", return_value={"SKILL.md": "x"}) as mock_tree:
             result = find_skill_md_paths_cached(
-                MagicMock(), "u/r", "", tree_cache
+                MagicMock(), "u/r", "", "main", tree_cache
             )
         assert result == {"SKILL.md": "x"}
         mock_tree.assert_called_once()
         assert tree_cache == {}  # nothing cached
 
+    def test_changed_default_branch_refetches(self):
+        """Same pushed_at but different default_branch → cache miss; refetches and updates."""
+        tree_cache = {
+            "u/r": {"pushed_at": "2026-01-01T00:00:00Z", "default_branch": "main", "paths": {"SKILL.md": "old"}}
+        }
+        with patch("crawlers.base.find_skill_md_paths", return_value={"SKILL.md": "new"}) as mock_tree:
+            result = find_skill_md_paths_cached(
+                MagicMock(), "u/r", "2026-01-01T00:00:00Z", "develop", tree_cache
+            )
+        mock_tree.assert_called_once()
+        assert result == {"SKILL.md": "new"}
+        assert tree_cache["u/r"]["default_branch"] == "develop"
+        assert tree_cache["u/r"]["paths"] == {"SKILL.md": "new"}
+
 
 # ---------------------------------------------------------------------------
 # TestFetchRepoMetadataBatch
diff --git a/tests/crawlers/test_topic_crawler.py b/tests/crawlers/test_topic_crawler.py
index 8037ce5..66b8d23 100644
--- a/tests/crawlers/test_topic_crawler.py
+++ b/tests/crawlers/test_topic_crawler.py
@@ -35,7 +35,7 @@ def test_returns_full_names(self):
         session = MagicMock()
         with patch("crawlers.topic_crawler.github_get") as mock_get:
             mock_get.side_effect = [page, empty] * 20
-            result, _ = _discover_topic_repos(session, limit=100)
+            result = _discover_topic_repos(session, limit=100)
 
         assert "user/skill-a" in result
         assert "user/skill-b" in result
@@ -49,7 +49,7 @@ def test_deduplicates_across_queries(self):
         session = MagicMock()
         with patch("crawlers.topic_crawler.github_get") as mock_get:
             mock_get.side_effect = [both, empty] * 20
-            result, _ = _discover_topic_repos(session, limit=100)
+            result = _discover_topic_repos(session, limit=100)
 
         assert result.count("user/shared-skill") == 1
 
@@ -62,23 +62,9 @@ def test_respects_limit(self):
         session = MagicMock()
         with patch("crawlers.topic_crawler.github_get") as mock_get:
             mock_get.side_effect = [big_page, empty] * 20
-            result, complete = _discover_topic_repos(session, limit=5)
+            result = _discover_topic_repos(session, limit=5)
 
         assert len(result) <= 5
-        # Hitting the cap means coverage is partial → incomplete.
-        assert complete is False
-
-    def test_complete_when_under_cap_and_no_errors(self):
-        from crawlers.topic_crawler import _discover_topic_repos
-
-        page = {"items": [{"full_name": "user/skill-a"}]}
-        empty = {"items": []}
-        session = MagicMock()
-        with patch("crawlers.topic_crawler.github_get") as mock_get:
-            mock_get.side_effect = [page, empty] * 30
-            result, complete = _discover_topic_repos(session, limit=1000)
-
-        assert complete is True
 
     def test_handles_api_error_gracefully(self):
         from crawlers.topic_crawler import _discover_topic_repos
@@ -86,25 +72,9 @@ def test_handles_api_error_gracefully(self):
         session = MagicMock()
         with patch("crawlers.topic_crawler.github_get") as mock_get:
             mock_get.side_effect = RuntimeError("rate limited")
-            result, complete = _discover_topic_repos(session, limit=10)
+            result = _discover_topic_repos(session, limit=10)
 
         assert result == []
-        assert complete is False
-
-    def test_incomplete_results_marks_incomplete(self):
-        """A timed-out search (HTTP 200, incomplete_results=true) → discovery incomplete."""
-        from crawlers.topic_crawler import _discover_topic_repos
-
-        partial = {"items": [{"full_name": "user/skill-a"}], "incomplete_results": True}
-        empty = {"items": [], "incomplete_results": False}
-        session = MagicMock()
-        with patch("crawlers.topic_crawler.github_get") as mock_get:
-            mock_get.side_effect = [partial, empty] * 30
-            result, complete = _discover_topic_repos(session, limit=1000)
-
-        assert "user/skill-a" in result
-        assert complete is False
-        assert complete is False
 
 
 # ---------------------------------------------------------------------------
@@ -142,10 +112,8 @@ def test_writes_records_for_discovered_repos(self, tmp_path):
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
-             patch("crawlers.topic_crawler.save_crawl_state"):
-            mock_disc.return_value = (["user/skill-a", "user/skill-b"], True)
+             patch("crawlers.topic_crawler.save_tree_cache"):
+            mock_disc.return_value = ["user/skill-a", "user/skill-b"]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
             mock_skill_md.return_value = SAMPLE_SKILL_MD
@@ -168,10 +136,8 @@ def test_output_has_required_fields(self, tmp_path):
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
-             patch("crawlers.topic_crawler.save_crawl_state"):
-            mock_disc.return_value = (["user/skill-a"], True)
+             patch("crawlers.topic_crawler.save_tree_cache"):
+            mock_disc.return_value = ["user/skill-a"]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
             mock_skill_md.return_value = SAMPLE_SKILL_MD
@@ -196,10 +162,8 @@ def test_source_tag_is_topic(self, tmp_path):
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
-             patch("crawlers.topic_crawler.save_crawl_state"):
-            mock_disc.return_value = (["user/skill-a"], True)
+             patch("crawlers.topic_crawler.save_tree_cache"):
+            mock_disc.return_value = ["user/skill-a"]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
             mock_skill_md.return_value = None
@@ -223,10 +187,8 @@ def test_respects_limit(self, tmp_path):
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
-             patch("crawlers.topic_crawler.save_crawl_state"):
-            mock_disc.return_value = ([f"user/skill-{i}" for i in range(10)], True)
+             patch("crawlers.topic_crawler.save_tree_cache"):
+            mock_disc.return_value = [f"user/skill-{i}" for i in range(10)]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
             mock_skill_md.return_value = None
@@ -249,10 +211,8 @@ def test_skips_repos_with_no_skill_md(self, tmp_path):
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
-             patch("crawlers.topic_crawler.save_crawl_state"):
-            mock_disc.return_value = (["user/no-skill-md", "user/has-skill-md"], True)
+             patch("crawlers.topic_crawler.save_tree_cache"):
+            mock_disc.return_value = ["user/no-skill-md", "user/has-skill-md"]
             mock_meta.return_value = _mock_meta()
             mock_paths.side_effect = [{}, {"SKILL.md": "sha1"}]
             mock_skill_md.return_value = None
@@ -283,10 +243,8 @@ def test_skips_already_covered_repos(self, tmp_path):
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
-             patch("crawlers.topic_crawler.save_crawl_state"):
-            mock_disc.return_value = (["user/already-covered", "user/new-skill"], True)
+             patch("crawlers.topic_crawler.save_tree_cache"):
+            mock_disc.return_value = ["user/already-covered", "user/new-skill"]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
             mock_skill_md.return_value = None
@@ -321,10 +279,8 @@ def test_resume_skips_existing_keys(self, tmp_path):
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
-             patch("crawlers.topic_crawler.save_crawl_state"):
-            mock_disc.return_value = (["user/skill-a", "user/skill-b"], True)
+             patch("crawlers.topic_crawler.save_tree_cache"):
+            mock_disc.return_value = ["user/skill-a", "user/skill-b"]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
             mock_skill_md.return_value = None
@@ -346,10 +302,8 @@ def test_name_falls_back_to_repo_name_when_no_frontmatter(self, tmp_path):
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
-             patch("crawlers.topic_crawler.save_crawl_state"):
-            mock_disc.return_value = (["user/my-cool-skill"], True)
+             patch("crawlers.topic_crawler.save_tree_cache"):
+            mock_disc.return_value = ["user/my-cool-skill"]
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}
             mock_skill_md.return_value = None  # no frontmatter
@@ -394,12 +348,10 @@ def test_topic_crawl_uses_caches(self, tmp_path, monkeypatch):
         )
         monkeypatch.setattr(
             tc, "find_skill_md_paths_cached",
-            lambda s, r, p, c: {"SKILL.md": "sha1"},
+            lambda s, r, p, db, c: {"SKILL.md": "sha1"},
         )
         monkeypatch.setattr(tc, "fetch_skill_md_cached", lambda *a, **k: "---\nname: t\n---")
-        monkeypatch.setattr(tc, "_discover_topic_repos", lambda s, limit=1000, since=None: (["user/skill-a"], True))
-        monkeypatch.setattr(tc, "load_crawl_state", lambda p: {})
-        monkeypatch.setattr(tc, "save_crawl_state", lambda state, p: None)
+        monkeypatch.setattr(tc, "_discover_topic_repos", lambda s, limit=1000: ["user/skill-a"])
 
         out = str(tmp_path / "out.jsonl")
         count = tc.run(out)
@@ -410,174 +362,6 @@ def test_topic_crawl_uses_caches(self, tmp_path, monkeypatch):
         assert calls["saved_tree"] == 1
 
 
-# ---------------------------------------------------------------------------
-# TestDiscoverPushedFilter — date-filter tests (RED phase)
-# ---------------------------------------------------------------------------
-
-class TestDiscoverPushedFilter:
-    """Tests for the since= date-filter on _discover_topic_repos."""
-
-    def test_discover_appends_pushed_filter_when_since_set(self):
-        """When since is set, every query q should end with pushed:><since>."""
-        from crawlers.topic_crawler import _discover_topic_repos
-
-        captured_qs: list[str] = []
-
-        def fake_github_get(session, url, params=None, **kwargs):
-            if params:
-                captured_qs.append(params.get("q", ""))
-            return {"items": []}
-
-        session = MagicMock()
-        with patch("crawlers.topic_crawler.github_get", side_effect=fake_github_get):
-            _discover_topic_repos(session, since="2026-01-01T00:00:00Z")
-
-        assert len(captured_qs) > 0
-        for q in captured_qs:
-            assert q.endswith(" pushed:>2026-01-01T00:00:00Z"), (
-                f"Expected q to end with pushed filter, got: {q!r}"
-            )
-
-    def test_discover_no_filter_when_since_none(self):
-        """When since is None, no query q should contain 'pushed:>'."""
-        from crawlers.topic_crawler import _discover_topic_repos
-
-        captured_qs: list[str] = []
-
-        def fake_github_get(session, url, params=None, **kwargs):
-            if params:
-                captured_qs.append(params.get("q", ""))
-            return {"items": []}
-
-        session = MagicMock()
-        with patch("crawlers.topic_crawler.github_get", side_effect=fake_github_get):
-            _discover_topic_repos(session, since=None)
-
-        assert len(captured_qs) > 0
-        for q in captured_qs:
-            assert "pushed:>" not in q, (
-                f"Expected no pushed filter when since=None, got: {q!r}"
-            )
-
-    def test_run_uses_and_saves_discovery_state(self, tmp_path):
-        """run() in discover mode reads last_discovery_at and saves updated state."""
-        from crawlers.topic_crawler import run
-
-        with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
-             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
-             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
-             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_meta_cache"), \
-             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"), \
-             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state",
-                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
-             patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
-
-            mock_disc.return_value = (["user/skill-a"], True)  # non-empty: a successful discovery
-            mock_meta.return_value = _mock_meta()
-            mock_paths.return_value = {}
-            mock_skill_md.return_value = None
-
-            out = str(tmp_path / "out.jsonl")
-            run(out, mode="discover")
-
-        # _discover_topic_repos was called with since= from state
-        mock_disc.assert_called_once()
-        call_kwargs = mock_disc.call_args
-        assert call_kwargs.kwargs.get("since") == "2026-01-01T00:00:00Z", (
-            f"Expected since='2026-01-01T00:00:00Z', got call: {call_kwargs}"
-        )
-
-        # save_crawl_state was called once and the state has last_discovery_at set
-        mock_save_state.assert_called_once()
-        saved_state = mock_save_state.call_args.args[0]
-        assert "last_discovery_at" in saved_state
-        assert saved_state["last_discovery_at"]  # non-empty timestamp
-
-    def test_complete_empty_discovery_advances_window(self, tmp_path):
-        """A discovery that COMPLETES cleanly but finds nothing new still advances
-        last_discovery_at — a quiet period must not re-scan the same window forever.
-        (The transient-failure / incomplete case is covered by discovery_complete=False
-        in test_incomplete_discovery_does_not_advance_window.)"""
-        from crawlers.topic_crawler import run
-
-        with patch("crawlers.topic_crawler._discover_topic_repos", return_value=([], True)), \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_cached"), \
-             patch("crawlers.topic_crawler.find_skill_md_paths_cached"), \
-             patch("crawlers.topic_crawler.fetch_skill_md_cached"), \
-             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_meta_cache"), \
-             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"), \
-             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state",
-                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
-             patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
-
-            run(str(tmp_path / "out.jsonl"), mode="discover")
-
-        mock_save_state.assert_called_once()
-
-    def test_watermark_not_saved_when_write_fails(self, tmp_path):
-        """The discovery watermark must be persisted only AFTER write_jsonl succeeds;
-        a failed write must not advance last_discovery_at past unwritten repos."""
-        import pytest
-        from crawlers.topic_crawler import run
-
-        with patch("crawlers.topic_crawler._discover_topic_repos",
-                   return_value=(["user/skill-a"], True)), \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_cached", return_value=_mock_meta()), \
-             patch("crawlers.topic_crawler.find_skill_md_paths_cached", return_value={}), \
-             patch("crawlers.topic_crawler.fetch_skill_md_cached"), \
-             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_meta_cache"), \
-             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"), \
-             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state",
-                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
-             patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state, \
-             patch("crawlers.topic_crawler.write_jsonl", side_effect=OSError("disk full")):
-
-            with pytest.raises(OSError):
-                run(str(tmp_path / "out.jsonl"), mode="discover")
-
-        mock_save_state.assert_not_called()
-
-    def test_capped_discovery_does_not_advance_window(self, tmp_path):
-        """A discovery that hit the result cap is incomplete (more matches exist),
-        so the watermark must NOT advance even though repos were found."""
-        from crawlers.topic_crawler import run
-
-        with patch("crawlers.topic_crawler._discover_topic_repos",
-                   return_value=(["user/skill-a"], False)), \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_cached", return_value=_mock_meta()), \
-             patch("crawlers.topic_crawler.find_skill_md_paths_cached", return_value={}), \
-             patch("crawlers.topic_crawler.fetch_skill_md_cached"), \
-             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_meta_cache"), \
-             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"), \
-             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state",
-                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
-             patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
-
-            run(str(tmp_path / "out.jsonl"), mode="discover")
-
-        mock_save_state.assert_not_called()
-
 
 # ---------------------------------------------------------------------------
 # TestTopicCrawlerBatchMetaIntegration
@@ -612,11 +396,9 @@ def test_batch_result_used_and_rest_not_called(self, tmp_path):
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
-             patch("crawlers.topic_crawler.save_crawl_state"):
+             patch("crawlers.topic_crawler.save_tree_cache"):
 
-            mock_disc.return_value = (["user/skill-a"], True)
+            mock_disc.return_value = ["user/skill-a"]
             mock_paths.return_value = {"SKILL.md": "sha1"}
             mock_skill_md.return_value = None
 
@@ -656,14 +438,12 @@ def test_batch_excludes_already_covered_and_filtered(self, tmp_path):
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
-             patch("crawlers.topic_crawler.save_crawl_state"), \
              patch("crawlers.topic_crawler._load_existing_repo_urls",
                    return_value={"https://github.com/u/a"}), \
              patch("crawlers.topic_crawler.load_filter_cache",
                    return_value={"https://github.com/u/b"}):
 
-            mock_disc.return_value = (["u/a", "u/b", "u/c"], True)
+            mock_disc.return_value = ["u/a", "u/b", "u/c"]
             mock_batch.return_value = {}
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {}
@@ -677,240 +457,9 @@ def test_batch_excludes_already_covered_and_filtered(self, tmp_path):
         mock_batch.assert_called_once()
         called_names = mock_batch.call_args.args[1]
         assert "u/c" in called_names, f"Expected u/c in batch call, got {called_names}"
-        assert "u/a" not in called_names, f"u/a (already_covered) must be excluded"
-        assert "u/b" not in called_names, f"u/b (filter_cache) must be excluded"
-
-
-class TestWatermarkAdvancement:
-    """P2 #2: last_discovery_at should advance only after a complete, clean sweep."""
-
-    def test_limit_truncation_does_not_advance_window(self, tmp_path):
-        """When limit is hit (truncated=True), save_crawl_state must NOT be called."""
-        from crawlers.topic_crawler import run
-
-        with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
-             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
-             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
-             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_meta_cache"), \
-             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"), \
-             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state",
-                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
-             patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
-
-            # 3 repos each with a skill, but limit=1 so truncated after first
-            mock_disc.return_value = (["user/skill-0", "user/skill-1", "user/skill-2"], True)
-            mock_meta.return_value = _mock_meta()
-            mock_paths.return_value = {"SKILL.md": "sha1"}
-            mock_skill_md.return_value = SAMPLE_SKILL_MD
-
-            out = str(tmp_path / "out.jsonl")
-            run(out, limit=1)
-
-        mock_save_state.assert_not_called()
-
-    def test_inner_loop_limit_truncation_does_not_advance_window(self, tmp_path):
-        """When --limit is hit INSIDE a repo's skill loop (a single repo with multiple
-        SKILL.md), truncated must still be set so the watermark does not advance."""
-        from crawlers.topic_crawler import run
-
-        with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
-             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
-             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
-             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_meta_cache"), \
-             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"), \
-             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state",
-                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
-             patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
-
-            # ONE repo with TWO SKILL.md files, limit=1 → truncates inside the inner loop
-            mock_disc.return_value = (["user/multi-skill"], True)
-            mock_meta.return_value = _mock_meta()
-            mock_paths.return_value = {"SKILL.md": "sha1", "sub/SKILL.md": "sha2"}
-            mock_skill_md.return_value = SAMPLE_SKILL_MD
-
-            run(str(tmp_path / "out.jsonl"), limit=1)
-
-        mock_save_state.assert_not_called()
-
-    def test_per_repo_failure_does_not_advance_window(self, tmp_path):
-        """When fetch_repo_metadata_cached raises RuntimeError, save_crawl_state must
-        NOT be called (had_failure=True)."""
-        from crawlers.topic_crawler import run
-
-        with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
-             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
-             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
-             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_meta_cache"), \
-             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"), \
-             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state",
-                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
-             patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
-
-            mock_disc.return_value = (["user/skill-a"], True)
-            # batch returns empty so REST fallback is triggered; REST raises
-            mock_meta.side_effect = RuntimeError("API failure")
-            mock_paths.return_value = {}
-            mock_skill_md.return_value = None
-
-            out = str(tmp_path / "out.jsonl")
-            run(out)
-
-        mock_save_state.assert_not_called()
-
-    def test_clean_full_sweep_advances_window(self, tmp_path):
-        """A complete run with no truncation or failures MUST advance last_discovery_at."""
-        from crawlers.topic_crawler import run
-
-        with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
-             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
-             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
-             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_meta_cache"), \
-             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"), \
-             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state",
-                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
-             patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
-
-            mock_disc.return_value = (["user/skill-a"], True)
-            mock_meta.return_value = _mock_meta()
-            mock_paths.return_value = {"SKILL.md": "sha1"}
-            mock_skill_md.return_value = SAMPLE_SKILL_MD
-
-            out = str(tmp_path / "out.jsonl")
-            run(out, mode="discover")
-
-        mock_save_state.assert_called_once()
-        saved_state = mock_save_state.call_args.args[0]
-        assert "last_discovery_at" in saved_state
-
-    def test_incomplete_discovery_does_not_advance_window(self, tmp_path):
-        """When _discover_topic_repos returns discovery_complete=False (partial result
-        due to a per-query RuntimeError), save_crawl_state must NOT be called.
-        Advancing last_discovery_at on incomplete discovery would skip repos from the
-        failed query on the next incremental run."""
-        from crawlers.topic_crawler import run
-
-        with patch("crawlers.topic_crawler._discover_topic_repos",
-                   return_value=(["user/skill-a"], False)), \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
-             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
-             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
-             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_meta_cache"), \
-             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"), \
-             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state",
-                   return_value={"last_discovery_at": "2026-01-01T00:00:00Z"}), \
-             patch("crawlers.topic_crawler.save_crawl_state") as mock_save_state:
-
-            mock_meta.return_value = _mock_meta()
-            mock_paths.return_value = {}
-            mock_skill_md.return_value = None
+        assert "u/a" not in called_names, "u/a (already_covered) must be excluded"
+        assert "u/b" not in called_names, "u/b (filter_cache) must be excluded"
 
-            run(str(tmp_path / "out.jsonl"), mode="discover")
-
-        mock_save_state.assert_not_called()
-
-
-# ---------------------------------------------------------------------------
-# P1 fix: discover mode must append, not truncate
-# ---------------------------------------------------------------------------
-
-class TestDiscoverModeAppend:
-    """P1: discover mode date-filters to only new repos, so it MUST append."""
-
-    def test_discover_mode_appends_not_truncates(self, tmp_path):
-        """discover mode must call write_jsonl with append=True."""
-        from crawlers.topic_crawler import run
-
-        with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
-             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
-             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
-             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_meta_cache"), \
-             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"), \
-             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
-             patch("crawlers.topic_crawler.save_crawl_state"), \
-             patch("crawlers.topic_crawler.write_jsonl") as mock_write:
-
-            mock_disc.return_value = (["user/skill-a"], True)
-            mock_meta.return_value = _mock_meta()
-            mock_paths.return_value = {"SKILL.md": "sha1"}
-            mock_skill_md.return_value = None
-            mock_write.return_value = 1
-
-            out = str(tmp_path / "out.jsonl")
-            run(out, mode="discover")
-
-        mock_write.assert_called_once()
-        # append must be True for discover mode
-        call = mock_write.call_args
-        append_val = call.kwargs.get("append") if call.kwargs.get("append") is not None else call.args[2]
-        assert append_val is True, f"Expected append=True for discover mode, got: {call}"
-
-    def test_full_mode_does_not_append(self, tmp_path):
-        """full mode must call write_jsonl with append=False (rewrites)."""
-        from crawlers.topic_crawler import run
-
-        with patch("crawlers.topic_crawler._discover_topic_repos") as mock_disc, \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_batch", return_value={}), \
-             patch("crawlers.topic_crawler.fetch_repo_metadata_cached") as mock_meta, \
-             patch("crawlers.topic_crawler.find_skill_md_paths_cached") as mock_paths, \
-             patch("crawlers.topic_crawler.fetch_skill_md_cached") as mock_skill_md, \
-             patch("crawlers.topic_crawler.load_meta_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_meta_cache"), \
-             patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_content_cache"), \
-             patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
-             patch("crawlers.topic_crawler.save_crawl_state"), \
-             patch("crawlers.topic_crawler.write_jsonl") as mock_write:
-
-            mock_disc.return_value = (["user/skill-a"], True)
-            mock_meta.return_value = _mock_meta()
-            mock_paths.return_value = {"SKILL.md": "sha1"}
-            mock_skill_md.return_value = None
-            mock_write.return_value = 1
-
-            out = str(tmp_path / "out.jsonl")
-            run(out, mode="full")
-
-        mock_write.assert_called_once()
-        call = mock_write.call_args
-        append_val = call.kwargs.get("append") if call.kwargs.get("append") is not None else call.args[2]
-        assert append_val is False, f"Expected append=False for full mode, got: {call}"
 
 
 # ---------------------------------------------------------------------------
@@ -938,11 +487,9 @@ def test_lazy_batch_respects_limit(self, tmp_path):
              patch("crawlers.topic_crawler.load_content_cache", return_value={}), \
              patch("crawlers.topic_crawler.save_content_cache"), \
              patch("crawlers.topic_crawler.load_tree_cache", return_value={}), \
-             patch("crawlers.topic_crawler.save_tree_cache"), \
-             patch("crawlers.topic_crawler.load_crawl_state", return_value={}), \
-             patch("crawlers.topic_crawler.save_crawl_state"):
+             patch("crawlers.topic_crawler.save_tree_cache"):
 
-            mock_disc.return_value = (repos_250, True)
+            mock_disc.return_value = repos_250
             mock_batch.return_value = {}  # empty -> REST fallback fills meta
             mock_meta.return_value = _mock_meta()
             mock_paths.return_value = {"SKILL.md": "sha1"}

From 2451a0f95eafb9408c6b60c314a441aed695bb7f Mon Sep 17 00:00:00 2001
From: yya007 <yaoyuang1996@gmail.com>
Date: Mon, 22 Jun 2026 13:25:39 -0700
Subject: [PATCH 13/13] docs: mark 5/6b done, record 4 reverted as unsound for
 topic discovery

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 BACKLOG.md                          | 46 ++++++++++++++---------------
 docs/crawler-rate-limit-strategy.md | 18 +++++++----
 2 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/BACKLOG.md b/BACKLOG.md
index d5825cc..215ffbd 100644
--- a/BACKLOG.md
+++ b/BACKLOG.md
@@ -190,23 +190,26 @@ monorepo; fetch each via raw (free, per ①). Replaces 1 Contents call per file.
 
 ---
 
-### ⑤ Batch repo metadata via GraphQL (≤100 repos/call)
-**Priority: medium**
-Replace per-repo `/repos/{o}/{r}` REST calls with a GraphQL query (stars/
-pushed_at/default_branch) for up to 100 repos at once — ~1–2 points of the
-separate 5,000-point/hr GraphQL pool.
-
-**Files:** `crawlers/base.py` (new GraphQL helper), crawler call sites.
+### ⑤ Batch repo metadata via GraphQL (≤100 repos/call) — DONE
+**Done.** `fetch_repo_metadata_batch` in `crawlers/base.py` fetches
+stars/pushedAt/defaultBranch/topics for ≤100 repos per GraphQL POST (separate
+5,000-point/hr pool); wired into the topic crawler with a per-repo REST fallback.
 
 ---
 
-### ④ Reduce search/code reliance (the 10/min stall)
-**Priority: medium**
-Cache the discovered repo list; run discovery with date filters
-(`pushed:>last-run`) so search surfaces only new repos. Prefer repo/topic search
-(30/min) over code search (10/min) where possible.
+### ④ Reduce search/code reliance (the 10/min stall) — REVERTED, needs redesign
+**Status: attempted and reverted.** The date-filter approach
+(`topic:X pushed:>last-run`) is **unsound for topic discovery**: a repo that
+*adds a skill topic without a new commit* keeps its old `pushed_at`, so it is
+filtered out and the advancing watermark makes it permanently undiscoverable —
+and GitHub search has no "topic-added-since" qualifier. A correct incremental
+discovery needs a different mechanism (e.g. enumerate the full topic result set
+each run and diff against a persisted repo-list, never a `pushed_at` watermark),
+or accept that discovery search is cheap enough (repo-search is 30/min, never the
+bottleneck) and leave it full. For code-search-heavy crawlers (skillsmp shards),
+the separate lever is to lean on repo-search over code-search where possible.
 
-**Files:** `crawlers/skillsmp_crawler.py`, `crawlers/topic_crawler.py`, `crawlers/marketplace_crawler.py`.
+**Files:** `crawlers/skillsmp_crawler.py`, `crawlers/marketplace_crawler.py`.
 
 ---
 
@@ -218,13 +221,10 @@ overhead — only if ①–⑥ are insufficient.
 
 **Files:** `crawlers/base.py` (session/token rotation).
 
-### ⑥b Cache find_skill_md_paths by repo + HEAD SHA
-**Priority: medium** *(follow-up from the ①②③ impact measurement)*
-After ①②③, the residual metered cost per repo is the recursive Trees call in
-`find_skill_md_paths` (one metered call per repo every run, even when nothing
-changed). Cache the `{path: blob_sha}` result keyed by `repo + HEAD commit SHA`
-(one cheap `fetch_commit_sha` call, or reuse the ETag-cached metadata's known
-HEAD), and skip the Trees call when HEAD is unchanged. This is what makes a warm
-run approach zero metered cost *per repo*, not just per skill.
-
-**Files:** `crawlers/base.py` (`find_skill_md_paths`, `fetch_commit_sha`).
+### ⑥b Cache find_skill_md_paths — DONE
+**Done.** `find_skill_md_paths_cached` in `crawlers/base.py` skips the recursive
+Trees call when a repo's `(pushed_at, default_branch)` is unchanged, reusing the
+cached `{path: blob_sha}` map (keyed on default branch because a branch change
+re-resolves `HEAD` to a different tree without bumping `pushed_at`; empty/failed
+lookups are never cached). Wired into the topic crawler. Makes a warm run
+approach zero metered cost *per unchanged repo*.
diff --git a/docs/crawler-rate-limit-strategy.md b/docs/crawler-rate-limit-strategy.md
index a827c76..ae98f8a 100644
--- a/docs/crawler-rate-limit-strategy.md
+++ b/docs/crawler-rate-limit-strategy.md
@@ -74,11 +74,19 @@ canonical URL → fetch each **once**. Today a repo found by marketplace + topic
 skillsmp is fetched 3×. Pair with a per-run in-memory (and optionally on-disk)
 cache of fetched metadata/content keyed by canonical repo URL.
 
-### ④ Cut `search/code` reliance — **medium impact, medium effort**
-Cache the discovered repo list and run discovery with date filters
-(`pushed:>last-run`) so search only surfaces *new* repos (few calls). Prefer
-repo/topic search (30/min) over code search (10/min) where the query allows.
-Removes the 60s-cooldown stalls that dominate wall-clock.
+### ④ Cut `search/code` reliance — **attempted, reverted (date-filter is unsound for topic discovery)**
+The obvious approach — date-filter discovery with `pushed:>last-run` so search
+only surfaces *new* repos — was implemented and then **reverted**. It is
+fundamentally wrong for *topic* discovery: a repo that adds a skill topic without
+a new commit keeps its old `pushed_at`, so `pushed:>since` excludes it, the
+advancing watermark moves past it, and it becomes permanently undiscoverable —
+and GitHub search has no "topic-added-since" qualifier to compensate. (~10 codex
+findings chased the watermark before identifying the root flaw.) A correct
+incremental discovery must enumerate the full topic result set each run and diff
+against a persisted repo-list, not rely on a `pushed_at` watermark. Topic
+discovery is left full (repo-search at 30/min was never the bottleneck); the
+remaining lever is preferring repo-search over code-search in the
+code-search-heavy crawlers (skillsmp shards).
 
 ### ⑤ Batch metadata via GraphQL — **medium impact, medium effort**
 Replace per-repo `/repos/{o}/{r}` REST calls with one GraphQL query for ≤100