IPO-006..010: factor scoring, dashboard, screener orchestration, enrichment, and the fail-closed AI extraction agent

AGENTS.md

@@ -74,8 +74,8 @@ review → `/code-review` + `/security-review`; everything → `/using-superpowe
 | `security/` | Secret redaction + prompt-injection quarantine. |
 | `config/` | Typed runtime settings from env (`get_settings`, `AppSettings`, `SettingsError`). |
 | `fundamentals/`, `technical/`, `sixty_seven/` | The three AI-assisted subsystems. |
-| `ipo/` | IPO domain: SEBI filing ingestion, verified content-addressed document cache, manual extraction records, deterministic ratio engine, immutable score/recommendation history (IPO-001…005). |
-| `jobs/` | Headless CLIs (daily scan, forward-return computation, IPO filing ingestion). |
+| `ipo/` | IPO domain: SEBI filing ingestion, verified content-addressed document cache, manual extraction records, deterministic ratio engine, immutable score/recommendation history, factor derivation + hard caution flags (`scoring/`), read-only dashboard builder, quarantined SerpAPI enrichment (`sources/enrichment.py`), and the fail-closed AI extraction agent (`agents/`, `documents/table_extractor.py`, `documents/section_classifier.py`) (IPO-001…010). |
+| `jobs/` | Headless CLIs (daily scan, forward-return computation, IPO filing ingestion, and the idempotent IPO scan/download/enrich/extract/score pipeline). |
 | `admin/`, `auth/`, `notifications/`, `data_quality/` | Config overrides, OIDC gate, alerts, candle-quality receipts. |
 | `screener_registry.py`, `scanner_base.py`, `indicators.py`, `daily_data_loader.py`, `universe_*` | Screener framework, indicators, candle cache, universe management. |
 
@@ -235,11 +235,17 @@ allowlist gate. Full details and the **accepted residual risks**:
   [technical-analysis-ai](docs/architecture/components/technical-analysis-ai.md) ·
   [sixty-seven-ka-funda-ai](docs/architecture/components/sixty-seven-ka-funda-ai.md)
 - **IPO subsystem:** [ipo-screener LLD](docs/architecture/components/ipo-screener.md) ·
+  [ipo-extraction-ai LLD](docs/architecture/components/ipo-extraction-ai.md) ·
   [ipo-001 domain + score contract](docs/architecture/ipo-001-domain-score-contract.md) ·
   [ipo-002 SEBI ingestion](docs/architecture/ipo-002-sebi-filing-ingestion.md) ·
   [ipo-003 document cache](docs/architecture/ipo-003-document-downloader-cache.md) ·
   [ipo-004 manual extraction](docs/architecture/ipo-004-manual-extraction-mvp.md) ·
-  [ipo-005 ratio engine](docs/architecture/ipo-005-ratio-engine.md)
+  [ipo-005 ratio engine](docs/architecture/ipo-005-ratio-engine.md) ·
+  [ipo-006 factors + flags](docs/architecture/ipo-006-factor-derivation-and-verdict.md) ·
+  [ipo-007 dashboard](docs/architecture/ipo-007-dashboard.md) ·
+  [ipo-008 orchestration](docs/architecture/ipo-008-screener-orchestration.md) ·
+  [ipo-009 enrichment](docs/architecture/ipo-009-serpapi-enrichment.md) ·
+  [ipo-010 AI extraction](docs/architecture/ipo-010-ai-extraction-proposals.md)
 - **Observability / audit / config:** [observability](docs/architecture/components/observability.md) ·
   [audit-log](docs/architecture/components/audit-log.md) ·
   [obs-003 design](docs/architecture/obs-003-audit-log.md) ·
@@ -270,6 +276,14 @@ python -m backend.jobs.compute_forward_returns
 
 # IPO filing ingestion (SEBI listings -> filing inventory)
 python -m backend.jobs.scan_ipo_filings
+
+# Full IPO screener: scan -> download -> enrich -> score (idempotent re-runs;
+# add --extract to also draft AI extraction proposals for admin review)
+python -m backend.jobs.run_ipo_screener
+
+# Revisit reviewed extraction history for one issue. This still skips an
+# existing pending proposal and any semantically identical regenerated payload.
+python -m backend.jobs.run_ipo_screener --force-extract --issue-id 42
 ```
 
 See the [operations runbook](docs/operations.md) for scheduling, Docker/Compose, Render

app.py

@@ -155,6 +155,7 @@
     _render_history_run_details,
 )
 from ui.ipo_manual_page import _render_ipo_manual_page
+from ui.ipo_page import _render_ipo_page
 from ui.parameter_controls import (  # noqa: F401
     _apply_param_overrides,
     _param_state_key,
@@ -572,11 +573,15 @@ def main() -> None:
     # file must never prevent an operator from inspecting past runs.
     # "Validation / Signal Performance" is a read-only analytical view (like Scan
     # history) available to every authenticated user, not an admin-only page.
+    # "IPO screener" (IPO-007) is the same kind of read-only analytical view:
+    # every authenticated user can inspect verdicts, while the re-score action
+    # inside the page is additionally gated on MANAGE_IPO_DATA.
     view_options = [
         "Scanner",
         "Scan history",
         "Scan comparison",
         "Validation / Signal Performance",
+        "IPO screener",
     ]
     if role_has_capability(current_role, VIEW_HEALTH):
         # AUTH-003: the admin tier sees the operate-the-system pages — health, the
@@ -616,6 +621,12 @@ def main() -> None:
             can_export=role_has_capability(current_role, EXPORT_RESULTS)
         )
         return
+    if view == "IPO screener":
+        _render_ipo_page(
+            can_rescore=role_has_capability(current_role, MANAGE_IPO_DATA),
+            user_email=current_email,
+        )
+        return
     # AUTH-003 defense in depth: the view list already hides these from non-admins,
     # but the handler re-checks the capability before rendering — a stale rerun or a
     # crafted request cannot reach an admin page the UI never offered.

backend/ipo/__init__.py

@@ -30,13 +30,27 @@
     IpoShareUnit,
 )
 from backend.ipo.models import (
+    CitedFinancialFact,
+    CitedTextEvidence,
     Confidence,
+    DebtReductionPurposeEvidence,
+    DebtReductionPurposeStatus,
     FactorAssessment,
     FinancialPeriodType,
+    IpoCautionFlag,
+    IpoCautionFlagReport,
+    IpoCautionFlagStatus,
     IpoDocumentData,
     IpoDocumentParseStatus,
     IpoDocumentRecord,
+    IpoEnrichmentBatchUsability,
+    IpoEnrichmentSignalData,
+    IpoEnrichmentSignalRecord,
+    IpoEnrichmentSignalType,
     IpoEvaluationRecord,
+    IpoEvidenceAuthority,
+    IpoExtractionProposalRecord,
+    IpoExtractionProposalStatus,
     IpoFilingData,
     IpoFinancialData,
     IpoFinancialRecord,
@@ -52,11 +66,13 @@
     IpoSubscriptionRecord,
     IpoValidationError,
     Recommendation,
+    ScoreBreakdownItem,
     SebiFiling,
     SebiFilingCategory,
 )
 from backend.ipo.repository import (
     IpoNotFoundError,
+    approve_extraction_proposal,
     create_document,
     create_financial,
     create_issue,
@@ -72,41 +88,97 @@
     get_evaluation,
     get_financial,
     get_issue,
+    get_latest_evaluation,
     get_latest_filing_date,
     get_latest_ipo_ratios,
     get_latest_manual_profile,
     get_latest_recommendation,
+    get_latest_subscription,
     get_manual_extraction,
     get_subscription,
     ingest_filings,
     list_documents,
+    list_enrichment_signals,
     list_evaluations,
+    list_extraction_proposals,
     list_financials,
     list_issues,
     list_manual_extractions,
     list_subscriptions,
+    load_ipo_factor_inputs_snapshot,
+    record_enrichment_signals,
+    reject_extraction_proposal,
+    submit_extraction_proposal,
     submit_manual_extraction,
     update_document,
     update_financial,
     update_issue,
     update_subscription,
 )
-from backend.ipo.scorecard import score_ipo
+from backend.ipo.scoring.caution_flags import (
+    CAUTION_FLAG_ORDER,
+    CAUTION_FLAGS_VERSION,
+    evaluate_caution_flags,
+)
+from backend.ipo.scoring.factor_derivation import (
+    FACTOR_MODEL_VERSION,
+    IpoFactorInputs,
+    derive_debt_reduction_purpose_evidence,
+    derive_score_input,
+)
+from backend.ipo.scoring.recommendation import (
+    INSUFFICIENT_VERIFIED_DATA,
+    build_recommendation,
+)
+from backend.ipo.scoring.score_model import score_ipo
+from backend.ipo.scoring.service import (
+    SCREENER_MODEL_VERSION,
+    IpoRescoreOutcome,
+    rescore_issue,
+)
+from backend.ipo.sources.enrichment import (
+    ENRICHMENT_AUTHORITY_POLICY_VERSION,
+    ENRICHMENT_SOURCE_POLICY,
+    IpoEnrichmentOutcome,
+    collect_enrichment_signals,
+)
 from backend.ipo.sources.sebi import fetch_sebi_filings
-from backend.ipo.verdict import build_recommendation
 
 __all__ = [
+    "CAUTION_FLAGS_VERSION",
+    "CAUTION_FLAG_ORDER",
+    "ENRICHMENT_AUTHORITY_POLICY_VERSION",
+    "ENRICHMENT_SOURCE_POLICY",
+    "FACTOR_MODEL_VERSION",
+    "INSUFFICIENT_VERIFIED_DATA",
+    "SCREENER_MODEL_VERSION",
+    "CitedFinancialFact",
+    "CitedTextEvidence",
     "Confidence",
+    "DebtReductionPurposeEvidence",
+    "DebtReductionPurposeStatus",
     "FactorAssessment",
     "FinancialPeriodType",
     "IpoAmountUnit",
+    "IpoCautionFlag",
+    "IpoCautionFlagReport",
+    "IpoCautionFlagStatus",
     "IpoDocumentData",
     "IpoDocumentDownloadError",
     "IpoDocumentDownloadErrorCode",
     "IpoDocumentDownloadResult",
     "IpoDocumentParseStatus",
     "IpoDocumentRecord",
+    "IpoEnrichmentBatchUsability",
+    "IpoEnrichmentOutcome",
+    "IpoEnrichmentSignalData",
+    "IpoEnrichmentSignalRecord",
+    "IpoEnrichmentSignalType",
     "IpoEvaluationRecord",
+    "IpoEvidenceAuthority",
+    "IpoExtractionProposalRecord",
+    "IpoExtractionProposalStatus",
+    "IpoFactorInputs",
     "IpoFilingData",
     "IpoFinancialData",
     "IpoFinancialRecord",
@@ -126,6 +198,7 @@
     "IpoRatioReceipt",
     "IpoRatioStatus",
     "IpoRecommendationResult",
+    "IpoRescoreOutcome",
     "IpoScoreInput",
     "IpoScoreResult",
     "IpoShareUnit",
@@ -134,10 +207,13 @@
     "IpoSubscriptionRecord",
     "IpoValidationError",
     "Recommendation",
+    "ScoreBreakdownItem",
     "SebiFiling",
     "SebiFilingCategory",
+    "approve_extraction_proposal",
     "build_recommendation",
     "calculate_ipo_ratios",
+    "collect_enrichment_signals",
     "create_document",
     "create_financial",
     "create_issue",
@@ -147,27 +223,39 @@
     "delete_financial",
     "delete_issue",
     "delete_subscription",
+    "derive_debt_reduction_purpose_evidence",
+    "derive_score_input",
     "download_document",
+    "evaluate_caution_flags",
     "evaluate_issue",
     "fetch_sebi_filings",
     "get_document",
     "get_evaluation",
     "get_financial",
     "get_issue",
+    "get_latest_evaluation",
     "get_latest_filing_date",
     "get_latest_ipo_ratios",
     "get_latest_manual_profile",
     "get_latest_recommendation",
+    "get_latest_subscription",
     "get_manual_extraction",
     "get_subscription",
     "ingest_filings",
     "list_documents",
+    "list_enrichment_signals",
     "list_evaluations",
+    "list_extraction_proposals",
     "list_financials",
     "list_issues",
     "list_manual_extractions",
     "list_subscriptions",
+    "load_ipo_factor_inputs_snapshot",
+    "record_enrichment_signals",
+    "reject_extraction_proposal",
+    "rescore_issue",
     "score_ipo",
+    "submit_extraction_proposal",
     "submit_manual_extraction",
     "update_document",
     "update_financial",

backend/ipo/agents/__init__.py

@@ -0,0 +1,22 @@
+"""AI agents that draft IPO evidence proposals for human review (IPO-010).
+
+Beginner note: nothing in this package can write evidence. Agents produce
+*proposals* that an administrator must approve before scoring ever sees a
+number, so a hallucinated value is a review-queue item, not a verdict input.
+"""
+
+from __future__ import annotations
+
+from backend.ipo.agents.financial_extractor import (
+    EXTRACTOR_MODEL_VERSION,
+    IpoExtractionError,
+    IpoExtractionErrorReceipt,
+    propose_extraction,
+)
+
+__all__ = [
+    "EXTRACTOR_MODEL_VERSION",
+    "IpoExtractionError",
+    "IpoExtractionErrorReceipt",
+    "propose_extraction",
+]