feat: add lightweight provenance profiler

__init__.py

@@ -478,6 +478,7 @@ def _apply_artifact_dedup_canary_to_api_messages(
     if mods is None:
         return None
     _LLMContent = mods["models"]._LLMContent
+    ArtifactSpanLink = mods["artifact_dedup_canary"].ArtifactSpanLink
     apply_artifact_dedup_canary = mods["artifact_dedup_canary"].apply_artifact_dedup_canary
 
     llm_items = []
@@ -501,10 +502,38 @@ def _apply_artifact_dedup_canary_to_api_messages(
     if not llm_items:
         return None
 
+    llm_index_by_message_index = {msg_idx: llm_idx for llm_idx, msg_idx in enumerate(message_indexes)}
+    span_links = []
+    for msg_idx, msg in enumerate(api_messages):
+        raw_links = msg.get("contextpilot_span_links") if isinstance(msg, dict) else None
+        if isinstance(msg, dict):
+            msg.pop("contextpilot_span_links", None)
+        if not isinstance(raw_links, list):
+            continue
+        for raw in raw_links:
+            if not isinstance(raw, dict):
+                continue
+            try:
+                src_msg = int(raw["source_message_index"])
+                tgt_msg = int(raw.get("target_message_index", msg_idx))
+                span_links.append(
+                    ArtifactSpanLink(
+                        source_index=llm_index_by_message_index[src_msg],
+                        source_start=int(raw["source_start"]),
+                        source_end=int(raw["source_end"]),
+                        target_index=llm_index_by_message_index[tgt_msg],
+                        target_start=int(raw["target_start"]),
+                        target_end=int(raw["target_end"]),
+                    )
+                )
+            except (KeyError, TypeError, ValueError, IndexError):
+                continue
+
     result = apply_artifact_dedup_canary(
         llm_items,
         salt=salt,
         min_block_chars=40,
+        span_links=span_links,
     )
     if result and result.mutated:
         for item, idx in zip(llm_items, message_indexes):

contextpilot/__init__.py

@@ -17,70 +17,87 @@
     >>> results = pipeline.run(queries=["What is AI?"])
 
 See docs/reference/api.md for detailed documentation.
-"""
-
-from .pipeline import (
-    RAGPipeline,
-    RetrieverConfig,
-    OptimizerConfig,
-    InferenceConfig,
-    PipelineConfig,
-)
-
-from .context_index import (
-    ContextIndex,
-    IndexResult,
-)
-
-from .context_ordering import (
-    IntraContextOrderer,
-)
 
-from .server.live_index import ContextPilot
-
-from .dedup import (
-    dedup_chat_completions,
-    dedup_responses_api,
-    DedupResult,
-)
-
-from .api import optimize, optimize_batch
+Imports are lazy (PEP 562): the heavy RAG stack (``pipeline`` -> ``context_index``
+-> ``scipy``) is only pulled in when one of its names is first accessed. This
+keeps lightweight, dependency-free consumers -- such as the standalone token
+monitor / provenance profiler in :mod:`contextpilot.hermes_opportunities` --
+importable inside minimal environments where SciPy and friends are absent.
+"""
+from __future__ import annotations
 
-from .retriever import (
-    BM25Retriever,
-    FAISSRetriever,
-    FAISS_AVAILABLE,
-    Mem0Retriever,
-    create_mem0_corpus_map,
-    MEM0_AVAILABLE,
-)
+import importlib
+from typing import TYPE_CHECKING
 
 __version__ = "0.4.1"
 
-__all__ = [
+# Map each public name to the submodule that defines it. Submodules are imported
+# on first attribute access, so importing ``contextpilot`` (or any lightweight
+# subpackage like ``hermes_opportunities``) never eagerly drags in SciPy/NumPy.
+_LAZY_EXPORTS = {
     # High-level pipeline API
-    "RAGPipeline",
-    "RetrieverConfig",
-    "OptimizerConfig",
-    "InferenceConfig",
-    "PipelineConfig",
+    "RAGPipeline": ".pipeline",
+    "RetrieverConfig": ".pipeline",
+    "OptimizerConfig": ".pipeline",
+    "InferenceConfig": ".pipeline",
+    "PipelineConfig": ".pipeline",
     # Core components
-    "ContextIndex",
-    "IndexResult",
-    "IntraContextOrderer",
-    "ContextPilot",
+    "ContextIndex": ".context_index",
+    "IndexResult": ".context_index",
+    "IntraContextOrderer": ".context_ordering",
+    "ContextPilot": ".server.live_index",
     # Deduplication
-    "dedup_chat_completions",
-    "dedup_responses_api",
-    "DedupResult",
+    "dedup_chat_completions": ".dedup",
+    "dedup_responses_api": ".dedup",
+    "DedupResult": ".dedup",
     # Convenience functions
-    "optimize",
-    "optimize_batch",
+    "optimize": ".api",
+    "optimize_batch": ".api",
     # Retrievers
-    "BM25Retriever",
-    "FAISSRetriever",
-    "FAISS_AVAILABLE",
-    "Mem0Retriever",
-    "create_mem0_corpus_map",
-    "MEM0_AVAILABLE",
-]
+    "BM25Retriever": ".retriever",
+    "FAISSRetriever": ".retriever",
+    "FAISS_AVAILABLE": ".retriever",
+    "Mem0Retriever": ".retriever",
+    "create_mem0_corpus_map": ".retriever",
+    "MEM0_AVAILABLE": ".retriever",
+}
+
+__all__ = list(_LAZY_EXPORTS)
+
+
+def __getattr__(name: str):
+    """Lazily resolve a public name to its (heavy) submodule on first access."""
+    module_name = _LAZY_EXPORTS.get(name)
+    if module_name is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    module = importlib.import_module(module_name, __name__)
+    value = getattr(module, name)
+    globals()[name] = value  # cache so subsequent lookups skip the import machinery
+    return value
+
+
+def __dir__():
+    return sorted(list(globals()) + __all__)
+
+
+if TYPE_CHECKING:  # pragma: no cover - import-time hints for type checkers only
+    from .api import optimize, optimize_batch
+    from .context_index import ContextIndex, IndexResult
+    from .context_ordering import IntraContextOrderer
+    from .dedup import DedupResult, dedup_chat_completions, dedup_responses_api
+    from .pipeline import (
+        InferenceConfig,
+        OptimizerConfig,
+        PipelineConfig,
+        RAGPipeline,
+        RetrieverConfig,
+    )
+    from .retriever import (
+        FAISS_AVAILABLE,
+        MEM0_AVAILABLE,
+        BM25Retriever,
+        FAISSRetriever,
+        Mem0Retriever,
+        create_mem0_corpus_map,
+    )
+    from .server.live_index import ContextPilot

contextpilot/hermes_opportunities/__init__.py

@@ -78,18 +78,22 @@
     PromptDuplicateBlock,
     PromptDuplicateShadow,
     PromptDuplicateTypeCount,
+    ProvenanceProfile,
+    ProvenanceSourceStat,
     RepeatedBlock,
     RouterCandidateBlock,
     RouterLabelCount,
     RouterReasonCount,
     TelemetryCoverage,
     ToolSizeStat,
     TypeCount,
+    UNKNOWN_SOURCE,
     WorkerRoutingShadow,
     _est_tokens,
     _LLMContent,
     _ToolMessage,
 )
+from .provenance import build_provenance_profile
 from .privacy import (
     FORBIDDEN_OUTPUT_KEYS,
     _assert_no_forbidden_keys,
@@ -163,7 +167,12 @@
     "ParentAggregationGroup",
     "ArtifactKindStat",
     "ParentAggregationArtifacts",
+    "ProvenanceSourceStat",
+    "ProvenanceProfile",
     "OpportunityReport",
+    # provenance profile (token-monitor view)
+    "UNKNOWN_SOURCE",
+    "build_provenance_profile",
     # loaders
     "load_tool_messages",
     "load_llm_bound_content",