From fbacd16622d20fcf40b487ac4852437f0343fa8b Mon Sep 17 00:00:00 2001 From: Colin Son Date: Wed, 3 Jun 2026 21:10:03 -0500 Subject: [PATCH 1/2] Add blueprint release summary --- src/casecrawler/export/blueprint_release.py | 132 ++++++++++++++++ tests/test_blueprint_release_summary.py | 158 ++++++++++++++++++++ 2 files changed, 290 insertions(+) create mode 100644 src/casecrawler/export/blueprint_release.py create mode 100644 tests/test_blueprint_release_summary.py diff --git a/src/casecrawler/export/blueprint_release.py b/src/casecrawler/export/blueprint_release.py new file mode 100644 index 0000000..602addd --- /dev/null +++ b/src/casecrawler/export/blueprint_release.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +import json +from collections import Counter +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from casecrawler.models.blueprint import ReleaseReadinessTier +from casecrawler.storage.dataset_store import DatasetStore + + +def build_blueprint_release_summary( + store: DatasetStore, + dataset_id: str, + *, + generated_at: str | None = None, +) -> dict[str, Any]: + blueprints = store.list_blueprints(dataset_id=dataset_id, limit=100_000) + records = store.list_records(dataset_id=dataset_id, limit=None) + judge_reports = store.list_judge_reports(dataset_id=dataset_id, limit=100_000) + attempts = store.list_generation_attempts(dataset_id=dataset_id, limit=100_000) + + validation_reports = { + blueprint.blueprint_id: store.get_blueprint_validation_report( + blueprint.blueprint_id + ) + for blueprint in blueprints + } + materialized_blueprint_ids = sorted( + { + str(record.metadata["blueprint_id"]) + for record in records + if record.metadata.get("blueprint_id") + } + ) + ready_blueprint_ids = sorted( + blueprint_id + for blueprint_id, report in validation_reports.items() + if report is not None and report.research_release_ready + ) + missing_materialized = sorted( + set(ready_blueprint_ids) - set(materialized_blueprint_ids) + ) + + return { + "dataset_id": dataset_id, + "generated_at": generated_at or datetime.now(timezone.utc).isoformat(), + "blueprint_count": len(blueprints), + "validation_report_count": sum( + 1 for report in validation_reports.values() if report is not None + ), + "research_release_ready_count": len(ready_blueprint_ids), + "materialized_record_count": sum( + 1 for record in records if record.metadata.get("blueprint_id") + ), + "materialized_blueprint_ids": materialized_blueprint_ids, + "missing_materialized_blueprint_ids": missing_materialized, + "judge_report_count": len(judge_reports), + "passing_judge_report_count": sum(1 for report in judge_reports if report.passed), + "attempt_counts": dict(Counter(attempt.status.value for attempt in attempts)), + "role_attempt_counts": dict(Counter(attempt.role.value for attempt in attempts)), + "tier_counts": _tier_counts(validation_reports), + "organ_system_counts": dict( + Counter(blueprint.organ_system for blueprint in blueprints) + ), + "setting_counts": dict(Counter(blueprint.setting for blueprint in blueprints)), + "non_ready_blueprints": _non_ready_blueprints(validation_reports), + } + + +def write_blueprint_release_summary( + store: DatasetStore, + dataset_id: str, + output: str | Path, + *, + generated_at: str | None = None, +) -> dict[str, Any]: + summary = build_blueprint_release_summary( + store, + dataset_id, + generated_at=generated_at, + ) + output_path = Path(output) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n") + return summary + + +def _tier_counts(validation_reports: dict) -> dict[str, int]: + counts = Counter() + for report in validation_reports.values(): + if report is None: + counts["missing"] += 1 + else: + counts[report.tier.value] += 1 + for tier in ReleaseReadinessTier: + counts.setdefault(tier.value, 0) + return dict(counts) + + +def _non_ready_blueprints(validation_reports: dict) -> list[dict[str, Any]]: + non_ready = [] + for blueprint_id, report in sorted(validation_reports.items()): + if report is not None and report.research_release_ready: + continue + if report is None: + non_ready.append( + { + "blueprint_id": blueprint_id, + "tier": "missing", + "issues": [ + { + "field": "blueprint_validation_report", + "message": "No validation report is persisted.", + } + ], + } + ) + continue + non_ready.append( + { + "blueprint_id": blueprint_id, + "tier": report.tier.value, + "issues": report.issues, + "schema_valid": report.schema_valid, + "clinically_plausible": report.clinically_plausible, + "grounded": report.grounded, + "judge_validated": report.judge_validated, + } + ) + return non_ready diff --git a/tests/test_blueprint_release_summary.py b/tests/test_blueprint_release_summary.py new file mode 100644 index 0000000..fc9fb78 --- /dev/null +++ b/tests/test_blueprint_release_summary.py @@ -0,0 +1,158 @@ +import json + +from casecrawler.export.blueprint_release import ( + build_blueprint_release_summary, + write_blueprint_release_summary, +) +from casecrawler.generation.blueprint_materializer import BlueprintMaterializer +from casecrawler.models.blueprint import ( + BlueprintEvidence, + BlueprintValidationReport, + ClinicalBlueprint, + GenerationAttempt, + GenerationAttemptStatus, + GenerationRole, + JudgeReport, + ReleaseReadinessTier, +) +from casecrawler.storage.dataset_store import DatasetStore + + +def _blueprint(blueprint_id: str, *, organ_system: str = "cardiovascular"): + return ClinicalBlueprint( + blueprint_id=blueprint_id, + dataset_id="ds-1", + cohort_plan_id="plan-1", + archetype_name="anticoagulation decision", + organ_system=organ_system, + setting="outpatient", + patient={"age": 72, "sex": "female"}, + chief_concern="Atrial fibrillation anticoagulation follow-up.", + diagnoses=[ + { + "name": "atrial fibrillation", + "supporting_findings": ["ECG confirms AF"], + } + ], + evidence=BlueprintEvidence( + supported_claims=["AF anticoagulation requires renal-dose review."], + citations=[{"source": "dailymed", "claim": "renal-dose review"}], + ), + ) + + +def _judge_report(blueprint_id: str, *, passed: bool = True) -> JudgeReport: + return JudgeReport( + report_id=f"judge-{blueprint_id}", + dataset_id="ds-1", + artifact_id=blueprint_id, + role=GenerationRole.JUDGE, + score=0.93 if passed else 0.41, + passed=passed, + rubric="blueprint_plausibility", + ) + + +def _validation( + blueprint_id: str, + *, + tier: ReleaseReadinessTier, + judge_report: JudgeReport | None = None, +) -> BlueprintValidationReport: + return BlueprintValidationReport( + blueprint_id=blueprint_id, + tier=tier, + schema_valid=True, + clinically_plausible=tier + in { + ReleaseReadinessTier.CLINICALLY_PLAUSIBLE, + ReleaseReadinessTier.JUDGE_VALIDATED, + ReleaseReadinessTier.RESEARCH_RELEASE_READY, + }, + grounded=tier == ReleaseReadinessTier.RESEARCH_RELEASE_READY, + judge_validated=tier + in { + ReleaseReadinessTier.JUDGE_VALIDATED, + ReleaseReadinessTier.RESEARCH_RELEASE_READY, + }, + judge_reports=[judge_report] if judge_report is not None else [], + issues=[] if tier == ReleaseReadinessTier.RESEARCH_RELEASE_READY else [ + {"field": "evidence.citations", "message": "missing citation"} + ], + ) + + +def test_build_blueprint_release_summary_aggregates_readiness_and_materialization( + tmp_path, +): + store = DatasetStore(db_path=str(tmp_path / "datasets.db")) + ready_blueprint = _blueprint("bp-ready") + draft_blueprint = _blueprint("bp-draft", organ_system="endocrine") + ready_judge = _judge_report("bp-ready") + draft_judge = _judge_report("bp-draft", passed=False) + store.save_blueprint(ready_blueprint) + store.save_blueprint(draft_blueprint) + store.save_judge_report(ready_judge) + store.save_judge_report(draft_judge) + store.save_blueprint_validation_report( + _validation( + "bp-ready", + tier=ReleaseReadinessTier.RESEARCH_RELEASE_READY, + judge_report=ready_judge, + ) + ) + store.save_blueprint_validation_report( + _validation( + "bp-draft", + tier=ReleaseReadinessTier.SCHEMA_VALID, + judge_report=draft_judge, + ) + ) + store.save_generation_attempt( + GenerationAttempt( + attempt_id="attempt-1", + dataset_id="ds-1", + role=GenerationRole.REPAIR, + status=GenerationAttemptStatus.REPAIR_REQUESTED, + provider="openai", + model="repair-model", + artifact_id="bp-draft", + ) + ) + BlueprintMaterializer(created_at="2026-05-06T10:00:00").materialize( + ready_blueprint, + validation_report=store.get_blueprint_validation_report("bp-ready"), + store=store, + require_release_ready=True, + ) + + summary = build_blueprint_release_summary(store, "ds-1") + + assert summary["dataset_id"] == "ds-1" + assert summary["blueprint_count"] == 2 + assert summary["validation_report_count"] == 2 + assert summary["research_release_ready_count"] == 1 + assert summary["materialized_record_count"] == 1 + assert summary["materialized_blueprint_ids"] == ["bp-ready"] + assert summary["missing_materialized_blueprint_ids"] == [] + assert summary["judge_report_count"] == 2 + assert summary["passing_judge_report_count"] == 1 + assert summary["attempt_counts"]["repair_requested"] == 1 + assert summary["tier_counts"]["research_release_ready"] == 1 + assert summary["tier_counts"]["schema_valid"] == 1 + assert summary["organ_system_counts"] == { + "cardiovascular": 1, + "endocrine": 1, + } + assert summary["non_ready_blueprints"][0]["blueprint_id"] == "bp-draft" + + +def test_write_blueprint_release_summary_writes_json(tmp_path): + store = DatasetStore(db_path=str(tmp_path / "datasets.db")) + store.save_blueprint(_blueprint("bp-1")) + output = tmp_path / "summary.json" + + summary = write_blueprint_release_summary(store, "ds-1", output) + + assert json.loads(output.read_text()) == summary + assert summary["blueprint_count"] == 1 From 18088b446be9d40316caa25dde56c5d0eef133d6 Mon Sep 17 00:00:00 2001 From: Colin Son Date: Wed, 3 Jun 2026 21:18:34 -0500 Subject: [PATCH 2/2] Harden blueprint release summary scans --- src/casecrawler/export/blueprint_release.py | 7 +++-- src/casecrawler/storage/dataset_store.py | 24 ++++++++------ tests/test_blueprint_release_summary.py | 1 + tests/test_blueprint_storage.py | 35 +++++++++++++++++++++ 4 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/casecrawler/export/blueprint_release.py b/src/casecrawler/export/blueprint_release.py index 602addd..d4c1c63 100644 --- a/src/casecrawler/export/blueprint_release.py +++ b/src/casecrawler/export/blueprint_release.py @@ -16,10 +16,10 @@ def build_blueprint_release_summary( *, generated_at: str | None = None, ) -> dict[str, Any]: - blueprints = store.list_blueprints(dataset_id=dataset_id, limit=100_000) + blueprints = store.list_blueprints(dataset_id=dataset_id, limit=None) records = store.list_records(dataset_id=dataset_id, limit=None) - judge_reports = store.list_judge_reports(dataset_id=dataset_id, limit=100_000) - attempts = store.list_generation_attempts(dataset_id=dataset_id, limit=100_000) + judge_reports = store.list_judge_reports(dataset_id=dataset_id, limit=None) + attempts = store.list_generation_attempts(dataset_id=dataset_id, limit=None) validation_reports = { blueprint.blueprint_id: store.get_blueprint_validation_report( @@ -89,6 +89,7 @@ def write_blueprint_release_summary( def _tier_counts(validation_reports: dict) -> dict[str, int]: counts = Counter() + counts["missing"] = 0 for report in validation_reports.values(): if report is None: counts["missing"] += 1 diff --git a/src/casecrawler/storage/dataset_store.py b/src/casecrawler/storage/dataset_store.py index 25bce96..3f718a9 100644 --- a/src/casecrawler/storage/dataset_store.py +++ b/src/casecrawler/storage/dataset_store.py @@ -345,7 +345,7 @@ def list_blueprints( self, dataset_id: str | None = None, cohort_plan_id: str | None = None, - limit: int = 1000, + limit: int | None = 1000, ) -> list[ClinicalBlueprint]: query = "SELECT blueprint_json FROM clinical_blueprints WHERE 1=1" params: list = [] @@ -355,8 +355,10 @@ def list_blueprints( if cohort_plan_id: query += " AND cohort_plan_id = ?" params.append(cohort_plan_id) - query += " ORDER BY blueprint_id LIMIT ?" - params.append(limit) + query += " ORDER BY blueprint_id" + if limit is not None: + query += " LIMIT ?" + params.append(limit) rows = self._conn.execute(query, params).fetchall() return [ ClinicalBlueprint.model_validate_json(row["blueprint_json"]) @@ -394,7 +396,7 @@ def list_generation_attempts( dataset_id: str | None = None, artifact_id: str | None = None, role: GenerationRole | str | None = None, - limit: int = 1000, + limit: int | None = 1000, ) -> list[GenerationAttempt]: query = "SELECT attempt_json FROM generation_attempts WHERE 1=1" params: list = [] @@ -408,8 +410,10 @@ def list_generation_attempts( role_value = role.value if isinstance(role, GenerationRole) else role query += " AND role = ?" params.append(role_value) - query += " ORDER BY attempt_id LIMIT ?" - params.append(limit) + query += " ORDER BY attempt_id" + if limit is not None: + query += " LIMIT ?" + params.append(limit) rows = self._conn.execute(query, params).fetchall() return [ GenerationAttempt.model_validate_json(row["attempt_json"]) @@ -486,7 +490,7 @@ def list_judge_reports( self, dataset_id: str | None = None, artifact_id: str | None = None, - limit: int = 1000, + limit: int | None = 1000, ) -> list[JudgeReport]: query = "SELECT report_json FROM judge_reports WHERE 1=1" params: list = [] @@ -496,8 +500,10 @@ def list_judge_reports( if artifact_id: query += " AND artifact_id = ?" params.append(artifact_id) - query += " ORDER BY report_id LIMIT ?" - params.append(limit) + query += " ORDER BY report_id" + if limit is not None: + query += " LIMIT ?" + params.append(limit) rows = self._conn.execute(query, params).fetchall() return [JudgeReport.model_validate_json(row["report_json"]) for row in rows] diff --git a/tests/test_blueprint_release_summary.py b/tests/test_blueprint_release_summary.py index fc9fb78..c015733 100644 --- a/tests/test_blueprint_release_summary.py +++ b/tests/test_blueprint_release_summary.py @@ -140,6 +140,7 @@ def test_build_blueprint_release_summary_aggregates_readiness_and_materializatio assert summary["attempt_counts"]["repair_requested"] == 1 assert summary["tier_counts"]["research_release_ready"] == 1 assert summary["tier_counts"]["schema_valid"] == 1 + assert summary["tier_counts"]["missing"] == 0 assert summary["organ_system_counts"] == { "cardiovascular": 1, "endocrine": 1, diff --git a/tests/test_blueprint_storage.py b/tests/test_blueprint_storage.py index c4cf1fd..da3dce3 100644 --- a/tests/test_blueprint_storage.py +++ b/tests/test_blueprint_storage.py @@ -199,6 +199,41 @@ def test_dataset_store_saves_blueprint_with_attempt(tmp_path): assert store.get_generation_attempt("attempt-1") == attempt +def test_dataset_store_blueprint_audit_lists_support_unbounded_limit(tmp_path): + store = DatasetStore(db_path=str(tmp_path / "datasets.db")) + bp_1 = _blueprint("bp-1") + bp_2 = _blueprint("bp-2") + report = JudgeReport( + report_id="judge-1", + dataset_id="ds-1", + artifact_id="bp-1", + role=GenerationRole.JUDGE, + score=0.92, + passed=True, + rubric="blueprint_plausibility", + ) + attempt = GenerationAttempt( + attempt_id="attempt-1", + dataset_id="ds-1", + role=GenerationRole.JUDGE, + status=GenerationAttemptStatus.SUCCEEDED, + provider="openai", + model="gpt-4.1-mini", + artifact_id="bp-1", + ) + + store.save_blueprint(bp_1) + store.save_blueprint(bp_2) + store.save_judge_report(report) + store.save_generation_attempt(attempt) + + assert store.list_blueprints(dataset_id="ds-1", limit=None) == [bp_1, bp_2] + assert store.list_judge_reports(dataset_id="ds-1", limit=None) == [report] + assert store.list_generation_attempts(dataset_id="ds-1", limit=None) == [ + attempt + ] + + def test_dataset_manifest_includes_blueprint_persistence_counts(tmp_path): store = DatasetStore(db_path=str(tmp_path / "datasets.db")) store.save_record(_record())