review: the complete v0.2.10 line — agent tools, local chat, Flash default

.github/workflows/tests.yml

@@ -31,5 +31,5 @@ jobs:
           cache: pip
       - run: pip install -r requirements.txt pytest
       - if: matrix.agent-frameworks == 'with'
-        run: pip install openai-agents claude-agent-sdk
+        run: pip install openai-agents claude-agent-sdk anthropic
       - run: python -m pytest -q

.gitignore

@@ -6,3 +6,4 @@ __pycache__
 logs/
 .pageindex/
 dist/
+*.doc_id

README.md

@@ -173,9 +173,10 @@ python3 run_pageindex.py --pdf_path /path/to/your/document.pdf
 <details>
 <summary>Optional parameters</summary>
 <br>
-You can customize the processing with additional optional arguments:
+You can customize the processing with additional optional arguments (the structure-tuning flags below require <code>--mode standard</code>):
 
 ```
+--mode                  Processing mode: flash (default) or standard
 --model                 LLM model to use (default: gpt-4o-2024-11-20)
 --toc-check-pages       Pages to check for table of contents (default: 20)
 --max-pages-per-node    Max pages per node (default: 10)
@@ -199,13 +200,13 @@ python3 run_pageindex.py --md_path /path/to/your/document.md
 </details>
 
 > ### ⚡ PageIndex Flash *(preview)*
-> **PageIndex Flash** ([`pageindex/flash`](pageindex/flash)) generates tree structures from PDFs in seconds. Structure extraction is purely heuristic-based, no LLM needed. LLM is only used to generate node summaries.
+> **PageIndex Flash** ([`pageindex/flash`](pageindex/flash)) generates tree structures from PDFs in seconds. Structure extraction is purely heuristic-based, no LLM needed. An LLM is used only for node summaries and the optimization's expansion pass.
 >
 > ```bash
-> python3 run_pageindex.py --flash --pdf_path /path/to/your/document.pdf
+> python3 run_pageindex.py --mode flash --pdf_path /path/to/your/document.pdf
 > ```
 >
-> Add `--optimize` to refine the tree structure for more efficient retrieval (with an LLM expansion pass).
+> Tree optimization for retrieval (a deterministic merge, then an LLM expansion pass) is on by default; pass `--optimize off` to disable.
 
 ## 🚀 Agentic Vectorless RAG: An Example
 

examples/agentic_vectorless_rag_demo.py

@@ -6,83 +6,52 @@
 chunking, PageIndex builds a hierarchical tree index and uses agentic LLM
 reasoning for human-like, context-aware retrieval.
 
-Agent tools:
-  - get_document()           — document metadata (status, page count, etc.)
-  - get_document_structure() — tree structure index of a document
-  - get_page_content()       — retrieve text content of specific pages
+The agent tools come straight from the SDK — ``client.as_openai_tools()``
+exposes the PageIndex tool contract (browse_documents, get_document,
+get_document_structure, get_page_content) and ``client.agent_instructions()``
+provides the retrieval playbook, so the whole agent is a few lines. Swap
+``PageIndexLocalClient()`` for ``PageIndexCloudClient(api_key=...)`` and the
+same code runs against the cloud.
 
 Steps:
   1 — Index a PDF locally and view its tree structure index
   2 — View document metadata
   3 — Ask a question (agent reasons over the index and auto-calls tools)
 
-Requirements: pip install openai-agents; OPENAI_API_KEY in the environment.
+Requirements: pip install "pageindex[openai]"; OPENAI_API_KEY in the environment.
 """
 import sys
-import json
 import asyncio
 import concurrent.futures
 from pathlib import Path
 import requests
 
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
-from agents import Agent, Runner, function_tool, set_tracing_disabled
-from agents.model_settings import ModelSettings
+from agents import Agent, Runner, set_tracing_disabled
 from agents.stream_events import RawResponsesStreamEvent, RunItemStreamEvent
 from openai.types.responses import ResponseTextDeltaEvent, ResponseReasoningSummaryTextDeltaEvent
 
-from pageindex import PageIndexClient
+from pageindex import PageIndexAPIError, PageIndexLocalClient
 import pageindex.utils as utils
 
 PDF_URL = "https://arxiv.org/pdf/2603.15031"
 
 _EXAMPLES_DIR = Path(__file__).parent
 PDF_PATH = _EXAMPLES_DIR / "documents" / "attention-residuals.pdf"
+DOC_ID_PATH = _EXAMPLES_DIR / "documents" / "attention-residuals.doc_id"
 STORAGE_PATH = _EXAMPLES_DIR / ".pageindex"
 
-AGENT_SYSTEM_PROMPT = """
-You are PageIndex, a document QA assistant.
-TOOL USE:
-- Call get_document() first to confirm status and page count.
-- Call get_document_structure() to identify relevant page ranges.
-- Call get_page_content(pages="5-7") with tight ranges; never fetch the whole document.
-- Before each tool call, output one short sentence explaining the reason.
-Answer based only on tool output. Be concise.
-"""
-
 
-def query_agent(client: PageIndexClient, doc_id: str, prompt: str, verbose: bool = False) -> str:
+def query_agent(client: PageIndexLocalClient, doc_id: str, prompt: str, verbose: bool = False) -> str:
     """Run a document QA agent using the OpenAI Agents SDK.
 
     Streams text output token-by-token and returns the full answer string.
     Tool calls are always printed; verbose=True also prints arguments and output previews.
     """
-
-    @function_tool
-    def get_document() -> str:
-        """Get document metadata: status, page count, name, and description."""
-        return json.dumps(client.get_document(doc_id))
-
-    @function_tool
-    def get_document_structure() -> str:
-        """Get the document's full tree structure (without text) to find relevant sections."""
-        return json.dumps(client.get_document_structure(doc_id), ensure_ascii=False)
-
-    @function_tool
-    def get_page_content(pages: str) -> str:
-        """
-        Get the text content of specific pages.
-        Use tight ranges: e.g. '5-7' for pages 5 to 7, '3,8' for pages 3 and 8, '12' for page 12.
-        """
-        return json.dumps(client.get_page_content(doc_id, pages), ensure_ascii=False)
-
     agent = Agent(
-        name="PageIndex",
-        instructions=AGENT_SYSTEM_PROMPT,
-        tools=[get_document, get_document_structure, get_page_content],
-        model=getattr(client, "retrieve_model", None),
-        # model_settings=ModelSettings(reasoning={"effort": "low", "summary": "auto"}),  # Uncomment to enable reasoning
+        **client.openai_agent_config(doc_id=doc_id),
+        # model_settings=ModelSettings(reasoning={"effort": "low", "summary": "auto"}),  # from agents.model_settings import ModelSettings
     )
 
     async def _run():
@@ -152,21 +121,33 @@ async def _run():
         print("Download complete.\n")
 
     # Setup: local mode — no PageIndex API key needed, your LLM key does the work
-    client = PageIndexClient(storage_path=str(STORAGE_PATH))
+    client = PageIndexLocalClient(storage_path=str(STORAGE_PATH))
 
     # Step 1: Index PDF and view tree structure
     print("=" * 60)
     print("Step 1: Index PDF and view tree structure")
     print("=" * 60)
-    doc_id = next(
-        (doc["id"] for doc in client.list_documents(limit=100)["documents"]
-         if doc["name"] == PDF_PATH.name),
-        None,
-    )
+    doc_id = None
+    if DOC_ID_PATH.exists():
+        cached = DOC_ID_PATH.read_text().strip()
+        try:
+            client.get_document(cached)
+            doc_id = cached
+        except PageIndexAPIError:
+            DOC_ID_PATH.unlink()
+    if doc_id is None:
+        # The .doc_id cache is gitignored — on a fresh clone with an
+        # existing store, find the already-indexed copy by name instead of
+        # re-indexing it.
+        doc_id = next(
+            (doc["id"] for doc in client.list_documents(limit=100)["documents"]
+             if doc["name"] == PDF_PATH.name), None)
     if doc_id:
+        DOC_ID_PATH.write_text(doc_id)
         print(f"\nLoaded cached doc_id: {doc_id}")
     else:
-        doc_id = client.submit_document(str(PDF_PATH))["doc_id"]
+        doc_id = client.submit_document(str(PDF_PATH), wait=True)["doc_id"]
+        DOC_ID_PATH.write_text(doc_id)
         print(f"\nIndexed. doc_id: {doc_id}")
     print("\nTree Structure (top-level sections):")
     structure = client.get_tree(doc_id, node_summary=True)["result"]

pageindex/__init__.py

@@ -18,26 +18,36 @@
 ]
 
 _LAZY = {
+    "page_index": ".page_index_classic",
+    "page_index_main": ".page_index_classic",
     "page_index_flash": ".flash",
     "optimize_tree": ".tree_optimize",
     "md_to_tree": ".page_index_md",
 }
-_SUBMODULES = {"client", "cloud_api", "errors", "flash", "local_api",
-               "local_store", "page_index_classic", "page_index_md", "tree_optimize",
-               "utils"}
-
+_SUBMODULES = {"agent_tools", "client", "cloud_api", "errors", "flash",
+               "integrations", "local_api", "local_chat", "local_store",
+               "mcp_bridge", "page_index_classic", "page_index_md",
+               "tree_optimize", "utils"}
 
 def __getattr__(name):
     if name.startswith("_"):
+        # Dunder probes (copy, pickle, inspect) are the frequent unknown
+        # names — they must not trigger the classic import below.
         raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
     import importlib
     if name in _SUBMODULES:
         return importlib.import_module(f".{name}", __name__)
-    module = importlib.import_module(_LAZY.get(name, ".page_index_classic"), __name__)
+    # Pre-0.2.10 compat: unknown names fall through to the classic module,
+    # whose public surface (ConfigLoader, count_tokens, ...) resolved as
+    # package attributes. A non-underscore typo pays one classic import
+    # before its AttributeError — not worth an allowlist.
+    module = importlib.import_module(_LAZY.get(name, ".page_index_classic"),
+                                     __name__)
     try:
         value = getattr(module, name)
     except AttributeError:
-        raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None
+        raise AttributeError(
+            f"module {__name__!r} has no attribute {name!r}") from None
     globals()[name] = value
     return value
 

pageindex/_version.py

@@ -0,0 +1,10 @@
+"""Installed-package version, shared by every surface that reports it upstream."""
+from __future__ import annotations
+
+
+def sdk_version() -> str:
+    try:
+        from importlib.metadata import version
+        return version("pageindex")
+    except Exception:
+        return "0.0.0"