[codex] add evidence-first TrustDocument extraction

NOTICE

@@ -32,6 +32,8 @@ This software has runtime dependencies on the following open-source libraries
 - Jackson (com.fasterxml.jackson.*)             — Apache License 2.0
 - SLF4J API (org.slf4j:slf4j-api)                — MIT License
 - Apache PDFBox (org.apache.pdfbox:pdfbox)       — Apache License 2.0
+- OpenDataLoader PDF Core (org.opendataloader:opendataloader-pdf-core)
+                                                    — Apache License 2.0
 - Apache POI (org.apache.poi:poi-ooxml)          — Apache License 2.0
 - Failsafe (dev.failsafe:failsafe)               — Apache License 2.0
 - Apache Commons Text (org.apache.commons:*)     — Apache License 2.0

README.md

@@ -16,13 +16,13 @@
 [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE)
 [![Java](https://img.shields.io/badge/Java-25+-007396?logo=openjdk)](https://openjdk.org)
 
-**Auditable LLM extraction for Java.** DocTruth turns PDFs, DOCX, XLSX, and CSV files into schema-bound structured output with field-level source citations, optional PDF bounding boxes, confidence scores, provenance, and PROV-O audit JSON.
+**Auditable LLM extraction for Java.** DocTruth turns documents into TrustDocument evidence, schema-bound structured output, field-level source citations, optional PDF bounding boxes, confidence scores, provenance, and PROV-O audit JSON.
 
 DocTruth is for teams that need to answer one question reliably:
 
 > Where did this extracted value come from?
 
-The core boundary is simple: source document in, validated structured output plus evidence trail out.
+The core boundary is simple: source document in, TrustDocument plus validated structured output and evidence trail out. For PDFs, DocTruth uses OpenDataLoader as the default parser backend while keeping the public SDK, CLI JSON, citations, confidence, provenance, and audit contracts under DocTruth.
 
 It is framework-agnostic and fits into plain Java, Spring Boot, LangChain4j, Spring AI, Quarkus, Micronaut, or any Java service that already calls OpenAI, Anthropic, Gemini, DeepSeek, or an OpenAI-compatible model endpoint.
 
@@ -65,6 +65,16 @@ java -version
 
 ## Quick Start
 
+First prove the document evidence surface without an LLM key:
+
+```bash
+doctruth parse contract.pdf --format json -o trust-document.json
+doctruth profile contract.pdf --json
+```
+
+The TrustDocument JSON includes the source SHA-256, page count, parser backend,
+page/line anchors, optional PDF bounding boxes, text units, tables, and figures.
+
 ```java
 import ai.doctruth.DocTruth;
 import java.math.BigDecimal;
@@ -104,6 +114,8 @@ smoke tests. Parser and schema inspection do not require an LLM key.
 mvn package -DskipTests
 java -jar target/doctruth-java-0.2.0-alpha-all.jar parse examples/no-llm-parse/sample-contract.csv
 java -jar target/doctruth-java-0.2.0-alpha-all.jar parse examples/no-llm-parse/sample-contract.csv --json -o parsed.json
+java -jar target/doctruth-java-0.2.0-alpha-all.jar parse contract.pdf --format json -o trust-document.json
+java -jar target/doctruth-java-0.2.0-alpha-all.jar profile contract.pdf --json
 java -jar target/doctruth-java-0.2.0-alpha-all.jar schema examples/pydantic-interop/resume.schema.json
 ```
 
@@ -127,7 +139,8 @@ doctruth version
   <img src="docs/assets/capabilities.png" alt="DocTruth capabilities: parse, assemble context, extract with LLM providers, validate schema, attach evidence, and export audit JSON">
 </p>
 
-- Parses PDF, DOCX, XLSX, and CSV into sections with source locations; PDF text sections include page-normalized bounding boxes when layout data is available.
+- Parses PDF, DOCX, XLSX, and CSV into evidence-bearing sections; PDF defaults to the OpenDataLoader backend and emits page-normalized bounding boxes when layout data is available.
+- Normalizes parser output into DocTruth's TrustDocument JSON contract for source identity, parser provenance, page/line anchors, content units, tables, and visual evidence.
 - Extracts Java records or JSON Schema-bound objects through LLM providers.
 - Validates structured output locally and retries repairable failures.
 - Matches extracted fields back to exact source quotes.
@@ -159,11 +172,16 @@ var result = DocTruth.withProvider(provider)
         .extractJson("Extract contract terms", schema)
         .requireCitation("partyA")
         .requireCitation("totalValue")
+        .withEvidenceFirst()
         .withEvidence()
         .withMaxRetries(2)
         .runJson();
 ```
 
+`withEvidenceFirst()` asks the provider for `{ "value": ..., "exactQuote": ... }`
+at each schema leaf, then DocTruth unwraps the value, validates it against the
+original schema, and verifies the quote against the parsed document.
+
 If a team already owns Pydantic v2 models, export them to JSON Schema at build
 time and treat the output as a normal schema file. DocTruth does not import
 Python in Java production.
@@ -196,7 +214,8 @@ var local = DocTruth.withProvider(LlmProviders.openAiCompatible(
 
 ```bash
 doctruth init
-doctruth parse contract.pdf --bboxes
+doctruth parse contract.pdf --format json -o trust-document.json
+doctruth profile contract.pdf --json
 doctruth schema contract.schema.json
 doctruth doctor
 doctruth extract contract.pdf -s contract.schema.json
@@ -210,6 +229,7 @@ doctruth audit .doctruth/runs/<run-id>/audit.json
   - [No-LLM parse example](examples/no-llm-parse/)
   - [Install DocTruth CLI](docs/install.md)
   - [CLI](docs/cli.md)
+  - [Parser profiling](docs/parser-profiling.md)
   - [Evidence schema](docs/evidence-schema.md)
 - Integrate:
   - [Java integration guide](docs/java-integration.md)

docs/cli.md

@@ -74,7 +74,16 @@ backend only for compatibility checks or parser debugging:
 doctruth parse contract.pdf --parser pdfbox
 ```
 
-Write parsed sections as JSON:
+Write DocTruth's audit-ready TrustDocument JSON:
+
+```bash
+doctruth parse contract.pdf --format json -o trust-document.json
+```
+
+The output includes the source SHA-256, page count, parser backend, ordered
+content units, page/line anchors, tables, and optional PDF bounding boxes.
+
+Write the lower-level parsed section JSON for parser debugging:
 
 ```bash
 doctruth parse contract.pdf --json -o parsed.json
@@ -86,6 +95,55 @@ Show that bbox recovery is enabled in the summary:
 doctruth parse contract.pdf --bboxes
 ```
 
+### Profile
+
+Measure local parser latency and heap movement without an LLM key:
+
+```bash
+doctruth profile contract.pdf --iterations 3 --json
+```
+
+Include output serialization cost when comparing parser plus JSON rendering:
+
+```bash
+doctruth profile contract.pdf --iterations 3 --include-output trust-json --json
+```
+
+Measure the streaming file-writer path used by `parse -o`:
+
+```bash
+doctruth profile contract.pdf --iterations 3 --include-output trust-json-file --json
+```
+
+The JSON includes:
+
+```text
+parser
+iterations
+fileSizeBytes
+sectionCount
+includeOutput
+parseLatencyMillis
+outputLatencyMillis
+coldLatencyMillis
+warmAverageLatencyMillis
+coldOutputLatencyMillis
+warmAverageOutputLatencyMillis
+profiledOutputChars
+profiledOutputBytes
+heapUsedBeforeBytes
+heapUsedAfterBytes
+heapDeltaBytes
+```
+
+`trust-json` and `parsed-json` measure string rendering. `trust-json-file` and
+`parsed-json-file` measure temporary file writer output and report bytes.
+Warm latency is a subsequent full parse in the same JVM, not a persistent
+parser-worker residency metric.
+
+Use this command to compare `opendataloader` and `pdfbox` on a named corpus
+before proposing Rust, Go, or other native optimization work.
+
 ### Schema
 
 Check a JSON Schema:
@@ -112,8 +170,9 @@ By default, DocTruth:
 
 - reads provider/model/output defaults from `doctruth.yml` when present
 - uses `openai` as the default provider
-- requires citations for top-level schema fields
-- writes `result.json` and `audit.json` to `.doctruth/runs/<run-id>/`
+- requires citations for schema leaf fields
+- writes `trust-document.json`, `result.json`, `audit.json`, and `manifest.json`
+  to `.doctruth/runs/<run-id>/`
 
 Common overrides:
 
@@ -124,8 +183,16 @@ doctruth extract contract.pdf -s contract.schema.json --model gpt-4o-mini
 doctruth extract contract.pdf -s contract.schema.json --base-url http://localhost:11434/v1
 doctruth extract contract.pdf -s contract.schema.json --allow-uncited
 doctruth extract contract.pdf -s contract.schema.json --require partyA,totalValue
+doctruth extract contract.pdf -s contract.schema.json --evidence-first
 ```
 
+`--evidence-first` wraps each schema leaf as `{ "value": ..., "exactQuote": ... }`
+for the provider response. DocTruth unwraps `value` into `result.json`, validates
+it against the original schema, and uses `exactQuote` for citation matching.
+
+`audit.json` derivations include the TrustDocument `docId` and `unit` id when a
+field citation maps to a parsed unit.
+
 Provider keys:
 
 | Provider | Env var |

docs/install.md

@@ -5,7 +5,7 @@ try/debug/inspect path: it lets a Java team verify the core promise before
 writing integration code:
 
 ```text
-document -> parsed sections with source locations -> schema check -> audit output
+document -> TrustDocument evidence -> schema check -> audit output
 ```
 
 ## SDK Install
@@ -63,7 +63,7 @@ Check the install:
 ```bash
 doctruth version
 doctruth doctor
-doctruth parse examples/no-llm-parse/sample-contract.csv
+doctruth parse contract.pdf --format json -o trust-document.json
 ```
 
 If `java` is not on `PATH`, point the launcher at your Java 25 runtime:
@@ -86,13 +86,15 @@ java -version
 No provider key is required for parser and schema inspection:
 
 ```bash
-doctruth parse examples/no-llm-parse/sample-contract.csv
-doctruth parse examples/no-llm-parse/sample-contract.csv --json -o parsed.json
+doctruth parse contract.pdf --format json -o trust-document.json
+doctruth profile contract.pdf --json
 doctruth schema examples/pydantic-interop/resume.schema.json
 ```
 
 This is the recommended first-run path. It proves the document evidence surface
-before a user spends time configuring model keys.
+before a user spends time configuring model keys. For PDF, the default backend
+is OpenDataLoader; `doctruth profile` gives local latency and heap evidence for
+the same backend.
 
 ## Extraction Run
 
@@ -107,6 +109,11 @@ doctruth audit .doctruth/runs/<run-id>/audit.json
 Use `--provider`, `--model`, and `--base-url` only when the defaults are not
 enough.
 
+Each extraction run writes `trust-document.json`, `result.json`, `audit.json`,
+and `manifest.json`. Use `--evidence-first` when the provider should return
+`value` plus `exactQuote` for each schema leaf before DocTruth validates and
+cites the result.
+
 The CLI and SDK use the same parser, citation, provenance, and audit primitives.
 
 ## GitHub Release Artifacts

docs/parser-profiling.md

@@ -0,0 +1,63 @@
+# Parser Profiling
+
+DocTruth profiles parser backends before proposing Rust, Go, or native
+optimization work. The default PDF backend is OpenDataLoader.
+
+Run a local profile:
+
+```bash
+doctruth profile contract.pdf --iterations 3 --json
+```
+
+Include JSON rendering cost when you are profiling parser plus output overhead:
+
+```bash
+doctruth profile contract.pdf --iterations 3 --include-output trust-json --json
+```
+
+Measure the streaming file-writer path used by `doctruth parse -o`:
+
+```bash
+doctruth profile contract.pdf --iterations 3 --include-output trust-json-file --json
+```
+
+Compare the legacy PDFBox backend on the same file:
+
+```bash
+doctruth profile contract.pdf --parser pdfbox --iterations 3 --json
+```
+
+The command reports:
+
+```text
+parser
+iterations
+fileSizeBytes
+sectionCount
+includeOutput
+parseLatencyMillis
+outputLatencyMillis
+coldLatencyMillis
+warmAverageLatencyMillis
+coldOutputLatencyMillis
+warmAverageOutputLatencyMillis
+profiledOutputChars
+profiledOutputBytes
+heapUsedBeforeBytes
+heapUsedAfterBytes
+heapDeltaBytes
+```
+
+`includeOutput` defaults to `parser-only`. Use `trust-json` for DocTruth's
+audit-ready TrustDocument JSON or `parsed-json` for the compatibility parser
+debug format. Use `trust-json-file` or `parsed-json-file` when profiling the
+streaming writer path instead of string materialization.
+
+`warmAverageLatencyMillis` means subsequent full parses in the same JVM. It is
+not a persistent parser-worker or model-residency measurement; the OpenDataLoader
+backend still tears down its static runtime after each parse.
+
+Treat these numbers as evidence for the named machine, corpus, parser backend,
+DocTruth version, and command. They are not global product thresholds. Native
+work should target a measured bottleneck while preserving the DocTruth
+TrustDocument JSON contract.

src/main/java/ai/doctruth/Citation.java

@@ -7,7 +7,8 @@
  * The verifiable evidence anchor for a single extracted field. A {@code Citation} ties an
  * extracted value back to a specific span of the source document plus the exact quote
  * recovered from that span, an optional visual bounding box, and a confidence score for the
- * page-attribution match itself.
+ * page-attribution match itself. When available, {@code source} links the quote to
+ * the TrustDocument unit that generated the audit entry.
  *
  * <p>Invariants (enforced by the compact constructor):
  *
@@ -19,22 +20,30 @@
  *       fuzzy / Levenshtein-style matchers.
  *   <li>{@code boundingBox} is non-null; use {@link Optional#empty()} when the source format
  *       has no reliable page geometry.
+ *   <li>{@code source} is non-null; use {@link Optional#empty()} when this citation was not
+ *       produced from a TrustDocument-backed parser run.
  * </ul>
  *
  * @param location   the source-document span this citation points at.
  * @param exactQuote the literal text recovered from the source that justified the value.
  * @param matchScore similarity of {@code exactQuote} to the substring at {@code location};
  *                   downstream matchers should treat {@code matchScore < 0.85} as a warning.
  * @param boundingBox optional page-normalized visual region for PDF-originated text.
+ * @param source optional TrustDocument source identity.
  * @since 0.1.0
  */
 public record Citation(
-        SourceLocation location, String exactQuote, double matchScore, Optional<BoundingBox> boundingBox) {
+        SourceLocation location,
+        String exactQuote,
+        double matchScore,
+        Optional<BoundingBox> boundingBox,
+        Optional<CitationSource> source) {
 
     public Citation {
         Objects.requireNonNull(location, "location");
         Objects.requireNonNull(exactQuote, "exactQuote");
         Objects.requireNonNull(boundingBox, "boundingBox");
+        Objects.requireNonNull(source, "source");
         if (exactQuote.isBlank()) {
             throw new IllegalArgumentException("exactQuote must not be blank");
         }
@@ -47,6 +56,13 @@ public record Citation(
      * Backward-compat constructor — leaves the visual bounding box absent.
      */
     public Citation(SourceLocation location, String exactQuote, double matchScore) {
-        this(location, exactQuote, matchScore, Optional.empty());
+        this(location, exactQuote, matchScore, Optional.empty(), Optional.empty());
+    }
+
+    /**
+     * Backward-compat constructor — leaves TrustDocument source identity absent.
+     */
+    public Citation(SourceLocation location, String exactQuote, double matchScore, Optional<BoundingBox> boundingBox) {
+        this(location, exactQuote, matchScore, boundingBox, Optional.empty());
     }
 }

src/main/java/ai/doctruth/CitationSource.java

@@ -0,0 +1,24 @@
+package ai.doctruth;
+
+import java.util.Objects;
+
+/**
+ * TrustDocument source identity for a citation.
+ *
+ * @param docId  TrustDocument document id.
+ * @param unitId TrustDocument unit id inside {@code docId}.
+ * @since 0.2.0
+ */
+public record CitationSource(String docId, String unitId) {
+
+    public CitationSource {
+        Objects.requireNonNull(docId, "docId");
+        Objects.requireNonNull(unitId, "unitId");
+        if (docId.isBlank()) {
+            throw new IllegalArgumentException("docId must not be blank");
+        }
+        if (unitId.isBlank()) {
+            throw new IllegalArgumentException("unitId must not be blank");
+        }
+    }
+}