Fix pipeline robustness and MinerU content compatibility

citationclaw/app/task_executor.py

@@ -1157,7 +1157,7 @@ async def _enrich_unknown_affiliations(self, merged_file: Path, config,
             client = AsyncOpenAI(
                 api_key=config.openai_api_key,
                 base_url=(config.openai_base_url or "").rstrip("/") + "/",
-                http_client=make_async_client(timeout=60.0),
+                http_client=make_async_client(timeout=120.0),
             )
 
             affil_results = {}  # author_name_lower → institution
@@ -1178,7 +1178,7 @@ async def _search_one(name_lower, display_name, paper_title):
                                 temperature=0.0,
                                 extra_body={"web_search_options": {}},
                             ),
-                            timeout=45,
+                            timeout=120,
                         )
                     answer = (resp.choices[0].message.content or "").strip()
                     # Clean up: remove quotes, markdown, etc.

citationclaw/core/openalex_client.py

@@ -78,7 +78,8 @@ def _parse_work(self, work: dict) -> dict:
                 "name": author.get("display_name", ""),
                 "openalex_id": author.get("id", ""),
                 "affiliation": inst.get("display_name", ""),
-                "country": inst.get("country_code", ""),
+                # OpenAlex may include the key with a JSON null value.
+                "country": inst.get("country_code") or "",
             })
         oa_loc = work.get("best_oa_location") or {}
         venue = work.get("primary_location", {}).get("source", {}).get("display_name", "")

citationclaw/core/pdf_mineru_parser.py

@@ -510,7 +510,7 @@ def _extract_from_zip(self, zip_bytes: bytes, output_dir: Path) -> Optional[dict
                     md_text = zf.read(name).decode('utf-8')
                     (output_dir / "full.md").write_text(md_text, encoding="utf-8")
                 elif 'content_list' in name and name.endswith('.json'):
-                    content_list = json.loads(zf.read(name))
+                    content_list = self._normalize_content_list(json.loads(zf.read(name)))
                     (output_dir / "content_list.json").write_text(
                         json.dumps(content_list, ensure_ascii=False), encoding="utf-8"
                     )
@@ -523,10 +523,7 @@ def _extract_from_zip(self, zip_bytes: bytes, output_dir: Path) -> Optional[dict
         return {
             "content_list": content_list,
             "full_md": md_text,
-            "first_page_blocks": (
-                [b for b in content_list if b.get("page_idx", 99) == 0][:20]
-                if content_list else self._md_to_first_page(md_text)
-            ),
+            "first_page_blocks": self._first_page_blocks(content_list, md_text),
             "references_md": self._extract_references(md_text),
             "source": "mineru_cloud_precision",
             "parsed_at": parsed_at,
@@ -560,7 +557,7 @@ def _parse_local_mineru(self, pdf_path: Path, output_dir: Path) -> Optional[dict
             content_list = []
             for f in output_dir.rglob("*content_list.json"):
                 with open(f) as fh:
-                    content_list = json.load(fh)
+                    content_list = self._normalize_content_list(json.load(fh))
                 break
 
             md_text = ""
@@ -582,9 +579,7 @@ def _parse_local_mineru(self, pdf_path: Path, output_dir: Path) -> Optional[dict
             return {
                 "content_list": content_list,
                 "full_md": md_text,
-                "first_page_blocks": (
-                    [b for b in content_list if b.get("page_idx", 99) == 0][:20]
-                ),
+                "first_page_blocks": self._first_page_blocks(content_list, md_text),
                 "references_md": self._extract_references(md_text),
                 "source": "mineru_local",
                 "parsed_at": parsed_at,
@@ -685,7 +680,7 @@ def _load_cached(self, output_dir: Path) -> Optional[dict]:
             parsed_at = datetime.now(timezone.utc).isoformat()
             for f in output_dir.rglob("*content_list.json"):
                 with open(f) as fh:
-                    content_list = json.load(fh)
+                    content_list = self._normalize_content_list(json.load(fh))
                 break
             if meta_path.exists():
                 meta = json.loads(meta_path.read_text(encoding="utf-8"))
@@ -694,17 +689,83 @@ def _load_cached(self, output_dir: Path) -> Optional[dict]:
             return {
                 "content_list": content_list,
                 "full_md": md_text,
-                "first_page_blocks": (
-                    [b for b in content_list if b.get("page_idx", 99) == 0][:20]
-                    if content_list else self._md_to_first_page(md_text)
-                ),
+                "first_page_blocks": self._first_page_blocks(content_list, md_text),
                 "references_md": self._extract_references(md_text),
                 "source": source or ("mineru" if content_list else "pymupdf"),
                 "parsed_at": parsed_at,
             }
         except Exception:
             return None
 
+    @classmethod
+    def _normalize_content_list(cls, raw_content) -> list:
+        """Normalize MinerU's flat and page-grouped content-list formats.
+
+        Depending on the Precision pipeline response, ``content_list.json`` may
+        be either ``list[dict]`` with an explicit ``page_idx`` or
+        ``list[list[dict]]`` where each outer item represents one page.  Downstream
+        code expects the former shape and a top-level ``text`` field.
+        """
+        if not isinstance(raw_content, list):
+            return []
+
+        normalized = []
+
+        def add_block(block, page_idx: int):
+            if not isinstance(block, dict):
+                return
+            item = dict(block)
+            item.setdefault("page_idx", page_idx)
+            if not str(item.get("text") or "").strip():
+                text = cls._content_block_text(item.get("content"))
+                if text:
+                    item["text"] = text
+            normalized.append(item)
+
+        for index, entry in enumerate(raw_content):
+            if isinstance(entry, list):
+                for block in entry:
+                    add_block(block, index)
+            else:
+                # Flat format already carries page_idx. Default to page zero for
+                # older variants where the field is omitted.
+                add_block(entry, 0)
+
+        return normalized
+
+    @classmethod
+    def _content_block_text(cls, value) -> str:
+        """Extract readable text from the nested Precision content schema."""
+        if isinstance(value, str):
+            return value.strip()
+        if isinstance(value, list):
+            parts = [cls._content_block_text(item) for item in value]
+            return " ".join(part for part in parts if part).strip()
+        if not isinstance(value, dict):
+            return ""
+
+        # These fields are metadata or asset locations, not PDF text.
+        ignored = {"type", "level", "math_type", "image_source", "path"}
+        parts = []
+        for key, item in value.items():
+            if key in ignored:
+                continue
+            text = cls._content_block_text(item)
+            if text:
+                parts.append(text)
+        return " ".join(parts).strip()
+
+    @classmethod
+    def _first_page_blocks(cls, content_list: list, md_text: str) -> list:
+        """Return usable first-page blocks, falling back to markdown text."""
+        blocks = [
+            block for block in content_list
+            if isinstance(block, dict)
+            and block.get("page_idx", 99) == 0
+            and str(block.get("text") or "").strip()
+        ][:20]
+        return blocks or cls._md_to_first_page(md_text)
+
     @staticmethod
     def _md_to_first_page(text: str) -> list:
         """Convert first-page text to pseudo content blocks."""

citationclaw/core/pipeline_adapter.py

@@ -107,7 +107,11 @@ def to_legacy_record(
         if api_authors_snapshot:
             api_lines = []
             for a in api_authors_snapshot:
-                api_lines.append(f"{a.get('name','')} | {a.get('affiliation','') or '未知'} | {ScholarSearchAgent._normalize_country(a.get('country',''))}")
+                api_lines.append(
+                    f"{a.get('name', '')} | "
+                    f"{a.get('affiliation', '') or '未知'} | "
+                    f"{ScholarSearchAgent._normalize_country(a.get('country') or '')}"
+                )
             api_affil_str = "\n".join(api_lines)
 
         # Build PDF-only snapshot string

citationclaw/core/scholar_search_agent.py

@@ -80,7 +80,7 @@ def _ensure_client(self):
             self._client = AsyncOpenAI(
                 api_key=self._api_key,
                 base_url=base,
-                http_client=make_async_client(timeout=120.0),
+                http_client=make_async_client(timeout=300.0),
             )
 
     async def search_paper_authors(self, paper_title: str, authors: List[dict]) -> List[ScholarResult]:
@@ -127,14 +127,14 @@ async def search_paper_authors(self, paper_title: str, authors: List[dict]) -> L
                     model=self._model,
                     messages=[{"role": "user", "content": prompt}],
                     temperature=0.1,
-                    timeout=60.0,
+                    timeout=300.0,
                 ),
-                timeout=90.0,  # Hard asyncio timeout as safety net
+                timeout=300.0,  # Hard asyncio timeout as safety net
             )
             text = response.choices[0].message.content.strip()
             return self._parse_response(text)
-        except asyncio.TimeoutError:
-            self._log(f"    ⚠ 搜索LLM超时 (90s)")
+        except _aio.TimeoutError:
+            self._log("    ⚠ 搜索LLM超时 (300s)")
             return []
         except Exception as e:
             self._log(f"    ⚠ 搜索LLM调用失败: {e}")
@@ -206,9 +206,9 @@ def _clean_field(raw: str) -> str:
         return s.strip()
 
     @staticmethod
-    def _normalize_country(raw: str) -> str:
-        """Normalize country names to Chinese."""
-        s = raw.strip()
+    def _normalize_country(raw: Optional[str]) -> str:
+        """Normalize country names to Chinese, tolerating missing API values."""
+        s = str(raw or "").strip()
         # Remove parenthetical codes like "(CN)" "（US）"
         s = re.sub(r'[（(]\s*[A-Z]{2,3}\s*[）)]', '', s).strip()
         # Map common codes and English names to Chinese

citationclaw/skills/phase4_citation_extract.py

@@ -270,7 +270,7 @@ async def _llm_extract(
             client = AsyncOpenAI(
                 api_key=ctx.config.effective_light_api_key(),
                 base_url=(ctx.config.effective_light_base_url() or "").rstrip("/") + "/",
-                http_client=make_async_client(timeout=60.0),
+                http_client=make_async_client(timeout=600.0),
             )
 
             parsed_paragraphs = self._build_paragraphs(contexts)

citationclaw/templates/index.html

@@ -157,6 +157,7 @@ <h1 class="hero-title">论文被引<span>画像分析</span></h1>
       <div class="col-md-4" style="display:flex;align-items:center">
         <select id="idx-openai-model" class="form-select" style="font-size:12.5px">
           <option value="gemini-3-flash-preview-search">gemini-3-flash-preview-search (default)</option>
+          <option value="gemini-3.6-flash-search">gemini-3.6-flash-search</option>
           <option value="gemini-3.1-pro-preview-search">gemini-3.1-pro-preview-search</option>
           <option value="gpt-5-search-api">gpt-5-search-api</option>
           <option value="deepseek-r1-search">deepseek-r1-search</option>

test/test_openalex_client.py

@@ -21,7 +21,11 @@ def test_parse_work_response():
             {
                 "author": {"id": "A1", "display_name": "Ashish Vaswani"},
                 "institutions": [{"display_name": "Google Brain", "country_code": "US"}],
-            }
+            },
+            {
+                "author": {"id": "A2", "display_name": "Unknown Country"},
+                "institutions": [{"display_name": "Example University", "country_code": None}],
+            },
         ],
         "doi": "https://doi.org/10.xxxx",
     }
@@ -30,6 +34,7 @@ def test_parse_work_response():
     assert result["authors"][0]["name"] == "Ashish Vaswani"
     assert result["authors"][0]["affiliation"] == "Google Brain"
     assert result["authors"][0]["country"] == "US"
+    assert result["authors"][1]["country"] == ""
     assert result["source"] == "openalex"
 
 def test_parse_author_response():

test/test_pdf_mineru_parser.py

@@ -1,6 +1,10 @@
 import sys, os
 sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
 
+import io
+import json
+import zipfile
+
 import pytest
 from citationclaw.core.pdf_mineru_parser import MinerUParser
 
@@ -31,3 +35,97 @@ def test_paper_key():
     k3 = parser.paper_key({"doi": "10.5678/other"})
     assert k1 == k2  # same DOI -> same key
     assert k1 != k3  # different DOI -> different key
+
+
+def test_normalize_page_grouped_content_list():
+    raw = [
+        [
+            {
+                "type": "title",
+                "content": {
+                    "title_content": [
+                        {"type": "text", "content": "A Paper Title"}
+                    ],
+                    "level": 1,
+                },
+            },
+            {
+                "type": "paragraph",
+                "content": {
+                    "paragraph_content": [
+                        {"type": "text", "content": "Alice and Bob"}
+                    ]
+                },
+            },
+        ],
+        [
+            {
+                "type": "paragraph",
+                "content": {
+                    "paragraph_content": [
+                        {"type": "text", "content": "Second page"}
+                    ]
+                },
+            }
+        ],
+    ]
+
+    normalized = MinerUParser._normalize_content_list(raw)
+
+    assert [block["page_idx"] for block in normalized] == [0, 0, 1]
+    assert normalized[0]["text"] == "A Paper Title"
+    assert normalized[1]["text"] == "Alice and Bob"
+    assert normalized[2]["text"] == "Second page"
+
+
+def test_normalize_preserves_flat_content_list():
+    raw = [
+        {"type": "title", "text": "Flat Title", "page_idx": 0},
+        {"type": "text", "text": "Page Two", "page_idx": 1},
+    ]
+
+    assert MinerUParser._normalize_content_list(raw) == raw
+
+
+def test_extract_from_zip_accepts_page_grouped_content_list(tmp_path):
+    raw_content = [[{
+        "type": "title",
+        "content": {
+            "title_content": [{"type": "text", "content": "Paper Title"}],
+            "level": 1,
+        },
+    }]]
+    buffer = io.BytesIO()
+    with zipfile.ZipFile(buffer, "w") as archive:
+        archive.writestr("result/full.md", "# Paper Title\n\nAlice Example")
+        archive.writestr("result/content_list.json", json.dumps(raw_content))
+
+    result = MinerUParser()._extract_from_zip(buffer.getvalue(), tmp_path)
+
+    assert result is not None
+    assert result["source"] == "mineru_cloud_precision"
+    assert result["first_page_blocks"][0]["text"] == "Paper Title"
+    cached_content = json.loads((tmp_path / "content_list.json").read_text())
+    assert cached_content[0]["page_idx"] == 0
+
+
+def test_load_cached_accepts_existing_page_grouped_content_list(tmp_path):
+    (tmp_path / "full.md").write_text("# Cached Paper\n\nCached Author")
+    (tmp_path / "content_list.json").write_text(json.dumps([[{
+        "type": "paragraph",
+        "content": {
+            "paragraph_content": [
+                {"type": "text", "content": "Cached Author"}
+            ]
+        },
+    }]]))
+    (tmp_path / "meta.json").write_text(json.dumps({
+        "source": "mineru_cloud_precision",
+        "parsed_at": "2026-08-12T00:00:00+00:00",
+    }))
+
+    result = MinerUParser()._load_cached(tmp_path)
+
+    assert result is not None
+    assert result["source"] == "mineru_cloud_precision"
+    assert result["first_page_blocks"][0]["text"] == "Cached Author"