fixed data

backend/Dockerfile

@@ -9,7 +9,7 @@ WORKDIR /app
 RUN apt-get clean && \
     rm -rf /var/lib/apt/lists/* && \
     apt-get update && \
-    apt-get install -y \
+    apt-get install -y --no-install-recommends \
     gcc \
     g++ \
     git \

backend/api/citations/router.py

@@ -31,7 +31,7 @@
     rispy = None
 
 from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, status
-from fastapi.responses import Response
+from fastapi.responses import Response, StreamingResponse
 from fastapi.concurrency import run_in_threadpool
 from pydantic import BaseModel
 
@@ -920,8 +920,15 @@ async def upload_citation_fulltext(
     sr_id: str,
     citation_id: int,
     file: UploadFile = File(...),
+    mode: str = 'replace',
     current_user: dict[str, Any] = Depends(get_current_active_user),
 ):
+    mode = str(mode or '').strip().lower()
+    if mode not in {'replace', 'supplementary'}:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="mode must be 'replace' or 'supplementary'",
+        )
     # The backend is authoritative: a PDF-linkage worker and manual upload must
     # never compete for an SR. A final conditional worker update remains the
     # race-safe fallback for uploads that began just before job creation.
@@ -1002,16 +1009,24 @@ async def upload_citation_fulltext(
         )
 
     try:
-        from ..services.fulltext_attachment_service import attach_fulltext_document
-        result = await attach_fulltext_document(
-            citation_id=citation_id,
-            table_name=table_name,
-            user_id=str(current_user['id']),
-            filename=file.filename,
-            content=content,
-            source='manual',
-            replace=True,
+        from ..services.fulltext_attachment_service import (
+            add_supplementary_document, attach_fulltext_document,
         )
+        if mode == 'supplementary':
+            result = await add_supplementary_document(
+                citation_id=citation_id, table_name=table_name,
+                user_id=str(current_user['id']), filename=file.filename, content=content,
+            )
+        else:
+            result = await attach_fulltext_document(
+                citation_id=citation_id,
+                table_name=table_name,
+                user_id=str(current_user['id']),
+                filename=file.filename,
+                content=content,
+                source='manual',
+                replace=True,
+            )
     except ValueError as exc:
         raise HTTPException(status_code=400, detail=str(exc)) from exc
     except Exception as e:
@@ -1033,6 +1048,100 @@ async def upload_citation_fulltext(
     }
 
 
+@router.get('/{sr_id}/citations/{citation_id}/fulltext-documents')
+async def get_fulltext_documents(
+    sr_id: str, citation_id: int,
+    current_user: dict[str, Any] = Depends(get_current_active_user),
+):
+    _, screening = await load_sr_and_check(sr_id, current_user, srdb_service)
+    table_name = (screening or {}).get('table_name') or 'citations'
+    row = await run_in_threadpool(cits_dp_service.get_citation_by_id, citation_id, table_name)
+    if not row:
+        raise HTTPException(status_code=404, detail='Citation not found')
+    from ..services.fulltext_attachment_service import ensure_legacy_document, list_fulltext_documents
+    await run_in_threadpool(ensure_legacy_document, citation_id, table_name, row)
+    documents = await run_in_threadpool(list_fulltext_documents, citation_id, table_name)
+    return {'documents': documents, 'title': row.get('title') or row.get('citation') or ''}
+
+
+@router.get('/{sr_id}/citations/{citation_id}/fulltext-documents/{document_id}/download')
+async def download_fulltext_document(
+    sr_id: str, citation_id: int, document_id: str,
+    current_user: dict[str, Any] = Depends(get_current_active_user),
+):
+    """Download a citation attachment after checking SR membership and linkage."""
+    _, screening = await load_sr_and_check(sr_id, current_user, srdb_service)
+    table_name = (screening or {}).get('table_name') or 'citations'
+    row = await run_in_threadpool(cits_dp_service.get_citation_by_id, citation_id, table_name)
+    if not row:
+        raise HTTPException(status_code=404, detail='Citation not found')
+    from ..services.fulltext_attachment_service import ensure_legacy_document, get_fulltext_document
+    from ..services.storage import storage_service
+    await run_in_threadpool(ensure_legacy_document, citation_id, table_name, row)
+    document = await run_in_threadpool(
+        get_fulltext_document, citation_id, table_name, document_id,
+    )
+    if not document:
+        raise HTTPException(
+            status_code=404, detail='Full-text document not found',
+        )
+    try:
+        content, filename = await storage_service.get_bytes_by_path(document['storage_path'])
+    except FileNotFoundError as exc:
+        raise HTTPException(
+            status_code=404, detail='Full-text file not found in storage',
+        ) from exc
+    except ValueError as exc:
+        raise HTTPException(
+            status_code=400, detail='Invalid full-text storage path',
+        ) from exc
+    return StreamingResponse(
+        iter([content]), media_type='application/pdf',
+        headers={
+            'Content-Disposition': f'inline; filename="{filename or document["filename"]}"',
+            'Cache-Control': 'no-store, max-age=0',
+        },
+    )
+
+
+class ActivateDocumentRequest(BaseModel):
+    document_id: str
+
+
+@router.post('/{sr_id}/citations/{citation_id}/fulltext-documents/activate')
+async def activate_document(
+    sr_id: str, citation_id: int, payload: ActivateDocumentRequest,
+    current_user: dict[str, Any] = Depends(get_current_active_user),
+):
+    _, screening = await load_sr_and_check(sr_id, current_user, srdb_service)
+    table_name = (screening or {}).get('table_name') or 'citations'
+    from ..services.fulltext_attachment_service import activate_fulltext_document
+    activated = await run_in_threadpool(
+        activate_fulltext_document, citation_id, table_name, payload.document_id,
+    )
+    if not activated:
+        raise HTTPException(
+            status_code=404, detail='Full-text document not found',
+        )
+    return {'status': 'success'}
+
+
+@router.delete('/{sr_id}/citations/{citation_id}/fulltext-documents/{document_id}')
+async def delete_document(
+    sr_id: str, citation_id: int, document_id: str,
+    current_user: dict[str, Any] = Depends(get_current_active_user),
+):
+    _, screening = await load_sr_and_check(sr_id, current_user, srdb_service)
+    table_name = (screening or {}).get('table_name') or 'citations'
+    from ..services.fulltext_attachment_service import delete_fulltext_document
+    deleted, was_active = await delete_fulltext_document(citation_id, table_name, document_id)
+    if not deleted:
+        raise HTTPException(
+            status_code=404, detail='Full-text document not found',
+        )
+    return {'status': 'success', 'was_active': was_active}
+
+
 # Helper to list fulltext URLs - delegated to backend.api.core.postgres.list_fulltext_urls
 
 
@@ -1076,6 +1185,9 @@ async def hard_delete_screening_resources(sr_id: str, current_user: dict[str, An
     # 1) collect fulltext URLs from the screening DB
     try:
         urls = await run_in_threadpool(cits_dp_service.list_fulltext_urls, table_name)
+        from ..services.fulltext_attachment_service import list_registered_storage_paths
+        registered_urls = await run_in_threadpool(list_registered_storage_paths, table_name)
+        urls = list(dict.fromkeys([*(urls or []), *(registered_urls or [])]))
     except RuntimeError as rexc:
         raise HTTPException(
             status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=str(rexc),
@@ -1154,6 +1266,13 @@ async def hard_delete_screening_resources(sr_id: str, current_user: dict[str, An
     # 3) drop the screening table
     try:
         await run_in_threadpool(cits_dp_service.drop_table, table_name)
+        try:
+            from ..services.fulltext_attachment_service import delete_document_registry
+            await run_in_threadpool(delete_document_registry, table_name)
+        except Exception:
+            # The citation table is authoritative; stale registry rows are harmless
+            # and can be reconciled separately if registry cleanup is unavailable.
+            pass
         table_dropped = True
     except RuntimeError as rexc:
         raise HTTPException(

backend/api/extract/router.py

@@ -701,6 +701,20 @@ async def _run_docint():
             [f"[{i}] {x}" for i, x in enumerate(full_text_arr)],
         )
 
+        # A structurally valid PDF can still yield no machine-readable text
+        # (for example, an image-only scan or an unsupported text encoding).
+        # Do not persist this as a successful extraction: downstream screening
+        # would otherwise misdiagnose absent context as an LLM response failure.
+        if not full_text_str.strip():
+            raise HTTPException(
+                status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+                detail=(
+                    'No machine-readable text could be extracted from this PDF. '
+                    'The file may be scanned, image-only, or use an unsupported '
+                    'encoding. Upload a searchable/OCR-processed PDF and try again.'
+                ),
+            )
+
         # -------------------------
         # Upload Azure DI artifacts
         # -------------------------
@@ -825,6 +839,17 @@ async def _run_docint():
         coords_for_overlay = list(annotations) + list(artifact_coords)
         updated1 = await run_in_threadpool(cits_dp_service.update_text_column, citation_id, 'fulltext', full_text_str, table_name)
         updated2 = await run_in_threadpool(cits_dp_service.update_text_column, citation_id, 'fulltext_md5', current_md5, table_name)
+        try:
+            from ..services.fulltext_attachment_service import (
+                ensure_legacy_document, record_active_extracted_text,
+            )
+            await run_in_threadpool(ensure_legacy_document, citation_id, table_name, row)
+            await run_in_threadpool(
+                record_active_extracted_text, citation_id, table_name, full_text_str,
+            )
+        except Exception:
+            # Multi-document indexing is additive and must not fail extraction.
+            pass
         updated3 = await run_in_threadpool(cits_dp_service.update_jsonb_column, citation_id, 'fulltext_coords', coords_for_overlay, table_name)
         updated4 = await run_in_threadpool(cits_dp_service.update_jsonb_column, citation_id, 'fulltext_pages', pages, table_name)
         updated5 = await run_in_threadpool(cits_dp_service.update_jsonb_column, citation_id, 'fulltext_figures', fulltext_figures, table_name)

backend/api/screen/agentic_utils.py

@@ -119,7 +119,26 @@ def parse_agent_xml(text: str) -> ParsedAgentXML:
 
 
 class AgentResponseError(ValueError):
-    """Raised when an LLM response still violates its stage contract after repair."""
+    """Raised when an LLM response still violates its stage contract after repair.
+
+    The repaired response is retained so callers that support partial suggestions
+    can persist the usable fields and show precise warnings to reviewers.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        raw_response: str = '',
+        parsed: ParsedAgentXML | None = None,
+        metadata: object | None = None,
+        missing_fields: list[str] | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.raw_response = raw_response
+        self.parsed = parsed
+        self.metadata = metadata
+        self.missing_fields = list(missing_fields or [])
 
 
 def validate_agent_response(
@@ -138,7 +157,12 @@ def validate_agent_response(
     return missing
 
 
-def build_repair_prompt(*, raw_response: str, stage: AgentStage) -> str:
+def build_repair_prompt(
+    *,
+    raw_response: str,
+    stage: AgentStage,
+    original_prompt: str = '',
+) -> str:
     """Ask the model to reformat an incomplete response without changing its judgment."""
     if stage == 'screening':
         schema = (
@@ -151,10 +175,16 @@ def build_repair_prompt(*, raw_response: str, stage: AgentStage) -> str:
             '<answer>exact selected option</answer>\n'
             '<confidence>number from 0 to 1</confidence>'
         )
+    context = original_prompt.strip()
     return f"""Your previous response did not match the required output contract.
-Preserve the same judgment, but return ONLY these XML tags with valid, non-empty values:
+Use the original task and allowed options below. Preserve the same judgment when it
+is recoverable; otherwise make the best supported judgment from the original task.
+Return ONLY these XML tags with valid, non-empty values:
 {schema}
 
+Original task and allowed options:
+{context or '(not available)'}
+
 Previous response:
 {raw_response}
 """
@@ -173,14 +203,22 @@ async def call_and_parse_agent_response(
     if not missing:
         return raw, parsed, metadata, False
 
-    repair_prompt = build_repair_prompt(raw_response=raw, stage=stage)
+    repair_prompt = build_repair_prompt(
+        raw_response=raw,
+        stage=stage,
+        original_prompt=prompt,
+    )
     repaired_raw, repaired_metadata = await call_llm(repair_prompt)
     repaired = parse_agent_xml(repaired_raw)
     repaired_missing = validate_agent_response(repaired, stage=stage)
     if repaired_missing:
         fields = ', '.join(repaired_missing)
         raise AgentResponseError(
             f'{stage.capitalize()} agent response missing or invalid fields after repair: {fields}',
+            raw_response=repaired_raw,
+            parsed=repaired,
+            metadata=repaired_metadata,
+            missing_fields=repaired_missing,
         )
     return repaired_raw, repaired, repaired_metadata, True