From d8414c2a003bc3459252889a62696ac2d00a8e27 Mon Sep 17 00:00:00 2001 From: Huiser WANG Date: Thu, 13 Aug 2026 00:01:00 +0800 Subject: [PATCH] fix pipeline robustness and MinerU compatibility --- citationclaw/app/task_executor.py | 4 +- citationclaw/core/openalex_client.py | 3 +- citationclaw/core/pdf_mineru_parser.py | 89 ++++++++++++++--- citationclaw/core/pipeline_adapter.py | 6 +- citationclaw/core/scholar_search_agent.py | 16 +-- .../skills/phase4_citation_extract.py | 2 +- citationclaw/templates/index.html | 1 + test/test_openalex_client.py | 7 +- test/test_pdf_mineru_parser.py | 98 +++++++++++++++++++ test/test_pipeline_adapter.py | 31 ++++++ test/test_scholar_search_validation.py | 1 + 11 files changed, 230 insertions(+), 28 deletions(-) diff --git a/citationclaw/app/task_executor.py b/citationclaw/app/task_executor.py index 2f72d0f..2a1f190 100644 --- a/citationclaw/app/task_executor.py +++ b/citationclaw/app/task_executor.py @@ -1157,7 +1157,7 @@ async def _enrich_unknown_affiliations(self, merged_file: Path, config, client = AsyncOpenAI( api_key=config.openai_api_key, base_url=(config.openai_base_url or "").rstrip("/") + "/", - http_client=make_async_client(timeout=60.0), + http_client=make_async_client(timeout=120.0), ) affil_results = {} # author_name_lower → institution @@ -1178,7 +1178,7 @@ async def _search_one(name_lower, display_name, paper_title): temperature=0.0, extra_body={"web_search_options": {}}, ), - timeout=45, + timeout=120, ) answer = (resp.choices[0].message.content or "").strip() # Clean up: remove quotes, markdown, etc. diff --git a/citationclaw/core/openalex_client.py b/citationclaw/core/openalex_client.py index 92fa499..7228e4f 100644 --- a/citationclaw/core/openalex_client.py +++ b/citationclaw/core/openalex_client.py @@ -78,7 +78,8 @@ def _parse_work(self, work: dict) -> dict: "name": author.get("display_name", ""), "openalex_id": author.get("id", ""), "affiliation": inst.get("display_name", ""), - "country": inst.get("country_code", ""), + # OpenAlex may include the key with a JSON null value. + "country": inst.get("country_code") or "", }) oa_loc = work.get("best_oa_location") or {} venue = work.get("primary_location", {}).get("source", {}).get("display_name", "") diff --git a/citationclaw/core/pdf_mineru_parser.py b/citationclaw/core/pdf_mineru_parser.py index d69914f..0851325 100644 --- a/citationclaw/core/pdf_mineru_parser.py +++ b/citationclaw/core/pdf_mineru_parser.py @@ -510,7 +510,7 @@ def _extract_from_zip(self, zip_bytes: bytes, output_dir: Path) -> Optional[dict md_text = zf.read(name).decode('utf-8') (output_dir / "full.md").write_text(md_text, encoding="utf-8") elif 'content_list' in name and name.endswith('.json'): - content_list = json.loads(zf.read(name)) + content_list = self._normalize_content_list(json.loads(zf.read(name))) (output_dir / "content_list.json").write_text( json.dumps(content_list, ensure_ascii=False), encoding="utf-8" ) @@ -523,10 +523,7 @@ def _extract_from_zip(self, zip_bytes: bytes, output_dir: Path) -> Optional[dict return { "content_list": content_list, "full_md": md_text, - "first_page_blocks": ( - [b for b in content_list if b.get("page_idx", 99) == 0][:20] - if content_list else self._md_to_first_page(md_text) - ), + "first_page_blocks": self._first_page_blocks(content_list, md_text), "references_md": self._extract_references(md_text), "source": "mineru_cloud_precision", "parsed_at": parsed_at, @@ -560,7 +557,7 @@ def _parse_local_mineru(self, pdf_path: Path, output_dir: Path) -> Optional[dict content_list = [] for f in output_dir.rglob("*content_list.json"): with open(f) as fh: - content_list = json.load(fh) + content_list = self._normalize_content_list(json.load(fh)) break md_text = "" @@ -582,9 +579,7 @@ def _parse_local_mineru(self, pdf_path: Path, output_dir: Path) -> Optional[dict return { "content_list": content_list, "full_md": md_text, - "first_page_blocks": ( - [b for b in content_list if b.get("page_idx", 99) == 0][:20] - ), + "first_page_blocks": self._first_page_blocks(content_list, md_text), "references_md": self._extract_references(md_text), "source": "mineru_local", "parsed_at": parsed_at, @@ -685,7 +680,7 @@ def _load_cached(self, output_dir: Path) -> Optional[dict]: parsed_at = datetime.now(timezone.utc).isoformat() for f in output_dir.rglob("*content_list.json"): with open(f) as fh: - content_list = json.load(fh) + content_list = self._normalize_content_list(json.load(fh)) break if meta_path.exists(): meta = json.loads(meta_path.read_text(encoding="utf-8")) @@ -694,10 +689,7 @@ def _load_cached(self, output_dir: Path) -> Optional[dict]: return { "content_list": content_list, "full_md": md_text, - "first_page_blocks": ( - [b for b in content_list if b.get("page_idx", 99) == 0][:20] - if content_list else self._md_to_first_page(md_text) - ), + "first_page_blocks": self._first_page_blocks(content_list, md_text), "references_md": self._extract_references(md_text), "source": source or ("mineru" if content_list else "pymupdf"), "parsed_at": parsed_at, @@ -705,6 +697,75 @@ def _load_cached(self, output_dir: Path) -> Optional[dict]: except Exception: return None + @classmethod + def _normalize_content_list(cls, raw_content) -> list: + """Normalize MinerU's flat and page-grouped content-list formats. + + Depending on the Precision pipeline response, ``content_list.json`` may + be either ``list[dict]`` with an explicit ``page_idx`` or + ``list[list[dict]]`` where each outer item represents one page. Downstream + code expects the former shape and a top-level ``text`` field. + """ + if not isinstance(raw_content, list): + return [] + + normalized = [] + + def add_block(block, page_idx: int): + if not isinstance(block, dict): + return + item = dict(block) + item.setdefault("page_idx", page_idx) + if not str(item.get("text") or "").strip(): + text = cls._content_block_text(item.get("content")) + if text: + item["text"] = text + normalized.append(item) + + for index, entry in enumerate(raw_content): + if isinstance(entry, list): + for block in entry: + add_block(block, index) + else: + # Flat format already carries page_idx. Default to page zero for + # older variants where the field is omitted. + add_block(entry, 0) + + return normalized + + @classmethod + def _content_block_text(cls, value) -> str: + """Extract readable text from the nested Precision content schema.""" + if isinstance(value, str): + return value.strip() + if isinstance(value, list): + parts = [cls._content_block_text(item) for item in value] + return " ".join(part for part in parts if part).strip() + if not isinstance(value, dict): + return "" + + # These fields are metadata or asset locations, not PDF text. + ignored = {"type", "level", "math_type", "image_source", "path"} + parts = [] + for key, item in value.items(): + if key in ignored: + continue + text = cls._content_block_text(item) + if text: + parts.append(text) + return " ".join(parts).strip() + + @classmethod + def _first_page_blocks(cls, content_list: list, md_text: str) -> list: + """Return usable first-page blocks, falling back to markdown text.""" + blocks = [ + block for block in content_list + if isinstance(block, dict) + and block.get("page_idx", 99) == 0 + and str(block.get("text") or "").strip() + ][:20] + return blocks or cls._md_to_first_page(md_text) + @staticmethod def _md_to_first_page(text: str) -> list: """Convert first-page text to pseudo content blocks.""" diff --git a/citationclaw/core/pipeline_adapter.py b/citationclaw/core/pipeline_adapter.py index c9194e0..34c2a4e 100644 --- a/citationclaw/core/pipeline_adapter.py +++ b/citationclaw/core/pipeline_adapter.py @@ -107,7 +107,11 @@ def to_legacy_record( if api_authors_snapshot: api_lines = [] for a in api_authors_snapshot: - api_lines.append(f"{a.get('name','')} | {a.get('affiliation','') or '未知'} | {ScholarSearchAgent._normalize_country(a.get('country',''))}") + api_lines.append( + f"{a.get('name', '')} | " + f"{a.get('affiliation', '') or '未知'} | " + f"{ScholarSearchAgent._normalize_country(a.get('country') or '')}" + ) api_affil_str = "\n".join(api_lines) # Build PDF-only snapshot string diff --git a/citationclaw/core/scholar_search_agent.py b/citationclaw/core/scholar_search_agent.py index c6ae6ca..08e553e 100644 --- a/citationclaw/core/scholar_search_agent.py +++ b/citationclaw/core/scholar_search_agent.py @@ -80,7 +80,7 @@ def _ensure_client(self): self._client = AsyncOpenAI( api_key=self._api_key, base_url=base, - http_client=make_async_client(timeout=120.0), + http_client=make_async_client(timeout=300.0), ) async def search_paper_authors(self, paper_title: str, authors: List[dict]) -> List[ScholarResult]: @@ -127,14 +127,14 @@ async def search_paper_authors(self, paper_title: str, authors: List[dict]) -> L model=self._model, messages=[{"role": "user", "content": prompt}], temperature=0.1, - timeout=60.0, + timeout=300.0, ), - timeout=90.0, # Hard asyncio timeout as safety net + timeout=300.0, # Hard asyncio timeout as safety net ) text = response.choices[0].message.content.strip() return self._parse_response(text) - except asyncio.TimeoutError: - self._log(f" ⚠ 搜索LLM超时 (90s)") + except _aio.TimeoutError: + self._log(" ⚠ 搜索LLM超时 (300s)") return [] except Exception as e: self._log(f" ⚠ 搜索LLM调用失败: {e}") @@ -206,9 +206,9 @@ def _clean_field(raw: str) -> str: return s.strip() @staticmethod - def _normalize_country(raw: str) -> str: - """Normalize country names to Chinese.""" - s = raw.strip() + def _normalize_country(raw: Optional[str]) -> str: + """Normalize country names to Chinese, tolerating missing API values.""" + s = str(raw or "").strip() # Remove parenthetical codes like "(CN)" "（US）" s = re.sub(r'[（(]\s*[A-Z]{2,3}\s*[）)]', '', s).strip() # Map common codes and English names to Chinese diff --git a/citationclaw/skills/phase4_citation_extract.py b/citationclaw/skills/phase4_citation_extract.py index f0d5831..0a79cde 100644 --- a/citationclaw/skills/phase4_citation_extract.py +++ b/citationclaw/skills/phase4_citation_extract.py @@ -270,7 +270,7 @@ async def _llm_extract( client = AsyncOpenAI( api_key=ctx.config.effective_light_api_key(), base_url=(ctx.config.effective_light_base_url() or "").rstrip("/") + "/", - http_client=make_async_client(timeout=60.0), + http_client=make_async_client(timeout=600.0), ) parsed_paragraphs = self._build_paragraphs(contexts) diff --git a/citationclaw/templates/index.html b/citationclaw/templates/index.html index cc73fe2..923cc6c 100644 --- a/citationclaw/templates/index.html +++ b/citationclaw/templates/index.html @@ -157,6 +157,7 @@

论文被引画像分析