Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions backend/api/screen/router.py
Original file line number Diff line number Diff line change
Expand Up @@ -2215,7 +2215,10 @@ async def get_screening_metrics(
try:
ids = await run_in_threadpool(
screening_eligibility_service.list_eligible_ids,
criteria=cp, table_name=table_name, target_stage=step_norm,
criteria=cp,
table_name=table_name,
target_stage=step_norm,
repair_decisions=False,
)
except Exception as e:
raise HTTPException(
Expand Down Expand Up @@ -2768,7 +2771,10 @@ async def get_screening_calibration(
try:
ids = await run_in_threadpool(
screening_eligibility_service.list_eligible_ids,
criteria=cp, table_name=table_name, target_stage=step_norm,
criteria=cp,
table_name=table_name,
target_stage=step_norm,
repair_decisions=False,
)
except Exception as e:
raise HTTPException(
Expand Down Expand Up @@ -3070,7 +3076,10 @@ async def get_live_confidence_histogram(
try:
ids = await run_in_threadpool(
screening_eligibility_service.list_eligible_ids,
criteria=cp, table_name=table_name, target_stage=step_norm,
criteria=cp,
table_name=table_name,
target_stage=step_norm,
repair_decisions=False,
)
except Exception as e:
raise HTTPException(
Expand Down Expand Up @@ -3213,7 +3222,10 @@ async def get_calibration_samples(
try:
ids = await run_in_threadpool(
screening_eligibility_service.list_eligible_ids,
criteria=cp, table_name=table_name, target_stage=step_norm,
criteria=cp,
table_name=table_name,
target_stage=step_norm,
repair_decisions=False,
)
except Exception as e:
raise HTTPException(
Expand Down
23 changes: 19 additions & 4 deletions backend/api/services/screening_eligibility_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,11 @@ def _answer_object(value: Any) -> dict[str, Any]:


def selected_answer(row: dict[str, Any], question: str) -> str | None:
"""Return the effective answer using human-over-AI precedence."""
"""Return the effective answer using human-over-main-AI precedence.

The critical agent is advisory: disagreements are routed to human review
but do not change the main screening agent's progression decision.
"""
core = snake_case(question, max_len=56)
if not core:
return None
Expand Down Expand Up @@ -90,17 +94,28 @@ def list_eligible_ids(
criteria: dict[str, Any] | None,
table_name: str,
target_stage: str,
repair_decisions: bool = True,
) -> list[int]:
"""Return citation IDs eligible for ``target_stage``.

Progression callers should keep ``repair_decisions=True`` so derived
decision caches are refreshed before they are used. Read-only reporting
callers can set it to ``False`` to avoid turning concurrent metrics
requests into repeated, table-wide writes.
"""
stage = str(target_stage or '').strip().lower()
if stage not in {'l1', 'l2', 'extract'}:
raise ValueError(f'Unsupported screening stage: {target_stage!r}')

if stage == 'l1':
return self.repository.list_citation_ids(None, table_name)

# Later stages depend on derived decisions. Repair first and fail loudly
# if repair fails; an empty list would be a misleading valid result.
self.repository.backfill_human_decisions(criteria or {}, table_name)
# Later stages depend on derived decisions. Progression paths repair
# first and fail loudly; reporting paths only read the existing cache.
if repair_decisions:
self.repository.backfill_human_decisions(
criteria or {}, table_name,
)
source_stage = 'l1' if stage == 'l2' else 'l2'
return self.repository.list_citation_ids(source_stage, table_name)

Expand Down
72 changes: 72 additions & 0 deletions backend/tests/screening_eligibility_service_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,11 @@
if 'psycopg' not in sys.modules:
psycopg = types.ModuleType('psycopg')
psycopg.connect = Mock()
psycopg_rows = types.ModuleType('psycopg.rows')
psycopg_rows.dict_row = Mock()
psycopg.rows = psycopg_rows
sys.modules['psycopg'] = psycopg
sys.modules['psycopg.rows'] = psycopg_rows

from api.services.screening_eligibility_service import ScreeningEligibilityService
from api.services.screening_eligibility_service import compute_screening_decisions
Expand Down Expand Up @@ -44,6 +48,20 @@ def test_ai_answer_is_used_when_human_answer_is_missing(self) -> None:
('include', 'include'),
)

def test_critical_ai_answer_does_not_override_screening_ai_answer(self) -> None:
row = {
'llm_relevant_population': {
'selected': 'Include',
'critical': {'selected': 'Exclude'},
},
'llm_eligible_study_design': {'selected': 'Include'},
}

self.assertEqual(
compute_screening_decisions(row, CRITERIA),
('include', 'include'),
)

def test_l2_decision_is_cumulative_and_missing_answers_exclude(self) -> None:
row = {'human_relevant_population': {'selected': 'Include'}}

Expand Down Expand Up @@ -93,6 +111,60 @@ def test_l1_does_not_run_unnecessary_repair(self) -> None:
None, 'screening_table',
)

def test_l2_read_only_scope_does_not_repair_decisions(self) -> None:
repository = Mock()
repository.list_citation_ids.return_value = [4, 9]
service = ScreeningEligibilityService(repository)

result = service.list_eligible_ids(
criteria=CRITERIA,
table_name='screening_table',
target_stage='l2',
repair_decisions=False,
)

self.assertEqual(result, [4, 9])
repository.backfill_human_decisions.assert_not_called()
repository.list_citation_ids.assert_called_once_with(
'l1', 'screening_table',
)

def test_extract_read_only_scope_uses_l2_decisions(self) -> None:
repository = Mock()
repository.list_citation_ids.return_value = [12]
service = ScreeningEligibilityService(repository)

result = service.list_eligible_ids(
criteria=CRITERIA,
table_name='screening_table',
target_stage='extract',
repair_decisions=False,
)

self.assertEqual(result, [12])
repository.backfill_human_decisions.assert_not_called()
repository.list_citation_ids.assert_called_once_with(
'l2', 'screening_table',
)

def test_read_only_scope_is_not_affected_by_repair_failure(self) -> None:
repository = Mock()
repository.backfill_human_decisions.side_effect = RuntimeError(
'repair failed',
)
repository.list_citation_ids.return_value = [4]
service = ScreeningEligibilityService(repository)

result = service.list_eligible_ids(
criteria=CRITERIA,
table_name='screening_table',
target_stage='l2',
repair_decisions=False,
)

self.assertEqual(result, [4])
repository.backfill_human_decisions.assert_not_called()

def test_repair_failure_is_not_hidden_as_an_empty_scope(self) -> None:
repository = Mock()
repository.backfill_human_decisions.side_effect = RuntimeError(
Expand Down
Loading