From e37643bf6aa1df12c4a8198575b588e54323b3b3 Mon Sep 17 00:00:00 2001 From: JamesbbBriz Date: Sun, 5 Jul 2026 13:00:29 +0800 Subject: [PATCH 1/2] feat: add evidence-first TrustDocument extraction --- NOTICE | 2 + README.md | 28 +- docs/cli.md | 73 +++- docs/install.md | 17 +- docs/parser-profiling.md | 63 ++++ src/main/java/ai/doctruth/Citation.java | 22 +- src/main/java/ai/doctruth/CitationSource.java | 24 ++ src/main/java/ai/doctruth/Confidence.java | 5 +- .../DocumentJsonExtractionBuilder.java | 4 + .../java/ai/doctruth/EvidenceFirstJson.java | 108 ++++++ .../ai/doctruth/JsonExtractionBuilder.java | 22 +- .../doctruth/JsonExtractionBuilderState.java | 45 ++- .../OpenDataLoaderPdfDocumentParser.java | 107 +++++- .../doctruth/OpenDataLoaderPdfGeometry.java | 49 ++- .../doctruth/OpenDataLoaderSectionMapper.java | 58 ++-- src/main/java/ai/doctruth/ParserRun.java | 19 ++ src/main/java/ai/doctruth/TrustDocument.java | 104 ++++++ .../java/ai/doctruth/TrustDocumentJson.java | 228 +++++++++++++ .../java/ai/doctruth/TrustDocumentParser.java | 26 ++ .../java/ai/doctruth/TrustDocumentSource.java | 28 ++ src/main/java/ai/doctruth/TrustUnit.java | 37 ++ .../java/ai/doctruth/TrustUnitEvidence.java | 22 ++ .../java/ai/doctruth/cli/DocTruthCli.java | 1 + .../java/ai/doctruth/cli/DocumentParsers.java | 10 + .../java/ai/doctruth/cli/ExtractCommand.java | 58 +++- .../java/ai/doctruth/cli/ParseCommand.java | 117 +++++-- .../ai/doctruth/cli/ParsedDocumentJson.java | 142 +++++--- .../java/ai/doctruth/cli/ProfileCommand.java | 296 ++++++++++++++++ src/main/java/ai/doctruth/cli/Usage.java | 8 +- .../internal/audit/ProvOExporter.java | 4 + .../internal/citation/CitationMatcher.java | 41 +-- .../java/ai/doctruth/CitationSourceTest.java | 45 +++ .../ai/doctruth/EvidenceFirstJsonTest.java | 108 ++++++ .../JsonSchemaExtractionContractTest.java | 44 +++ .../OpenDataLoaderPdfDocumentParserTest.java | 145 ++++++++ .../OpenDataLoaderSectionMapperTest.java | 83 +++++ .../ai/doctruth/TrustDocumentParserTest.java | 176 ++++++++++ .../java/ai/doctruth/cli/CliSupportTest.java | 17 + .../cli/DocTruthCliExtractContractTest.java | 206 +++++++++++ .../cli/DocTruthCliTrustDocumentTest.java | 319 ++++++++++++++++++ .../cli/ParseCommandPerformanceTest.java | 136 ++++++++ .../ai/doctruth/public-api-snapshot.txt | 83 ++++- 42 files changed, 2963 insertions(+), 167 deletions(-) create mode 100644 docs/parser-profiling.md create mode 100644 src/main/java/ai/doctruth/CitationSource.java create mode 100644 src/main/java/ai/doctruth/EvidenceFirstJson.java create mode 100644 src/main/java/ai/doctruth/ParserRun.java create mode 100644 src/main/java/ai/doctruth/TrustDocument.java create mode 100644 src/main/java/ai/doctruth/TrustDocumentJson.java create mode 100644 src/main/java/ai/doctruth/TrustDocumentParser.java create mode 100644 src/main/java/ai/doctruth/TrustDocumentSource.java create mode 100644 src/main/java/ai/doctruth/TrustUnit.java create mode 100644 src/main/java/ai/doctruth/TrustUnitEvidence.java create mode 100644 src/main/java/ai/doctruth/cli/ProfileCommand.java create mode 100644 src/test/java/ai/doctruth/CitationSourceTest.java create mode 100644 src/test/java/ai/doctruth/EvidenceFirstJsonTest.java create mode 100644 src/test/java/ai/doctruth/TrustDocumentParserTest.java create mode 100644 src/test/java/ai/doctruth/cli/DocTruthCliExtractContractTest.java create mode 100644 src/test/java/ai/doctruth/cli/DocTruthCliTrustDocumentTest.java create mode 100644 src/test/java/ai/doctruth/cli/ParseCommandPerformanceTest.java diff --git a/NOTICE b/NOTICE index 03ac33bc..97ea21bc 100644 --- a/NOTICE +++ b/NOTICE @@ -32,6 +32,8 @@ This software has runtime dependencies on the following open-source libraries - Jackson (com.fasterxml.jackson.*) — Apache License 2.0 - SLF4J API (org.slf4j:slf4j-api) — MIT License - Apache PDFBox (org.apache.pdfbox:pdfbox) — Apache License 2.0 +- OpenDataLoader PDF Core (org.opendataloader:opendataloader-pdf-core) + — Apache License 2.0 - Apache POI (org.apache.poi:poi-ooxml) — Apache License 2.0 - Failsafe (dev.failsafe:failsafe) — Apache License 2.0 - Apache Commons Text (org.apache.commons:*) — Apache License 2.0 diff --git a/README.md b/README.md index 22ffa01e..7498d8ad 100644 --- a/README.md +++ b/README.md @@ -16,13 +16,13 @@ [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE) [![Java](https://img.shields.io/badge/Java-25+-007396?logo=openjdk)](https://openjdk.org) -**Auditable LLM extraction for Java.** DocTruth turns PDFs, DOCX, XLSX, and CSV files into schema-bound structured output with field-level source citations, optional PDF bounding boxes, confidence scores, provenance, and PROV-O audit JSON. +**Auditable LLM extraction for Java.** DocTruth turns documents into TrustDocument evidence, schema-bound structured output, field-level source citations, optional PDF bounding boxes, confidence scores, provenance, and PROV-O audit JSON. DocTruth is for teams that need to answer one question reliably: > Where did this extracted value come from? -The core boundary is simple: source document in, validated structured output plus evidence trail out. +The core boundary is simple: source document in, TrustDocument plus validated structured output and evidence trail out. For PDFs, DocTruth uses OpenDataLoader as the default parser backend while keeping the public SDK, CLI JSON, citations, confidence, provenance, and audit contracts under DocTruth. It is framework-agnostic and fits into plain Java, Spring Boot, LangChain4j, Spring AI, Quarkus, Micronaut, or any Java service that already calls OpenAI, Anthropic, Gemini, DeepSeek, or an OpenAI-compatible model endpoint. @@ -65,6 +65,16 @@ java -version ## Quick Start +First prove the document evidence surface without an LLM key: + +```bash +doctruth parse contract.pdf --format json -o trust-document.json +doctruth profile contract.pdf --json +``` + +The TrustDocument JSON includes the source SHA-256, page count, parser backend, +page/line anchors, optional PDF bounding boxes, text units, tables, and figures. + ```java import ai.doctruth.DocTruth; import java.math.BigDecimal; @@ -104,6 +114,8 @@ smoke tests. Parser and schema inspection do not require an LLM key. mvn package -DskipTests java -jar target/doctruth-java-0.2.0-alpha-all.jar parse examples/no-llm-parse/sample-contract.csv java -jar target/doctruth-java-0.2.0-alpha-all.jar parse examples/no-llm-parse/sample-contract.csv --json -o parsed.json +java -jar target/doctruth-java-0.2.0-alpha-all.jar parse contract.pdf --format json -o trust-document.json +java -jar target/doctruth-java-0.2.0-alpha-all.jar profile contract.pdf --json java -jar target/doctruth-java-0.2.0-alpha-all.jar schema examples/pydantic-interop/resume.schema.json ``` @@ -127,7 +139,8 @@ doctruth version DocTruth capabilities: parse, assemble context, extract with LLM providers, validate schema, attach evidence, and export audit JSON

-- Parses PDF, DOCX, XLSX, and CSV into sections with source locations; PDF text sections include page-normalized bounding boxes when layout data is available. +- Parses PDF, DOCX, XLSX, and CSV into evidence-bearing sections; PDF defaults to the OpenDataLoader backend and emits page-normalized bounding boxes when layout data is available. +- Normalizes parser output into DocTruth's TrustDocument JSON contract for source identity, parser provenance, page/line anchors, content units, tables, and visual evidence. - Extracts Java records or JSON Schema-bound objects through LLM providers. - Validates structured output locally and retries repairable failures. - Matches extracted fields back to exact source quotes. @@ -159,11 +172,16 @@ var result = DocTruth.withProvider(provider) .extractJson("Extract contract terms", schema) .requireCitation("partyA") .requireCitation("totalValue") + .withEvidenceFirst() .withEvidence() .withMaxRetries(2) .runJson(); ``` +`withEvidenceFirst()` asks the provider for `{ "value": ..., "exactQuote": ... }` +at each schema leaf, then DocTruth unwraps the value, validates it against the +original schema, and verifies the quote against the parsed document. + If a team already owns Pydantic v2 models, export them to JSON Schema at build time and treat the output as a normal schema file. DocTruth does not import Python in Java production. @@ -196,7 +214,8 @@ var local = DocTruth.withProvider(LlmProviders.openAiCompatible( ```bash doctruth init -doctruth parse contract.pdf --bboxes +doctruth parse contract.pdf --format json -o trust-document.json +doctruth profile contract.pdf --json doctruth schema contract.schema.json doctruth doctor doctruth extract contract.pdf -s contract.schema.json @@ -210,6 +229,7 @@ doctruth audit .doctruth/runs//audit.json - [No-LLM parse example](examples/no-llm-parse/) - [Install DocTruth CLI](docs/install.md) - [CLI](docs/cli.md) + - [Parser profiling](docs/parser-profiling.md) - [Evidence schema](docs/evidence-schema.md) - Integrate: - [Java integration guide](docs/java-integration.md) diff --git a/docs/cli.md b/docs/cli.md index c44eca32..6a18fc66 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -74,7 +74,16 @@ backend only for compatibility checks or parser debugging: doctruth parse contract.pdf --parser pdfbox ``` -Write parsed sections as JSON: +Write DocTruth's audit-ready TrustDocument JSON: + +```bash +doctruth parse contract.pdf --format json -o trust-document.json +``` + +The output includes the source SHA-256, page count, parser backend, ordered +content units, page/line anchors, tables, and optional PDF bounding boxes. + +Write the lower-level parsed section JSON for parser debugging: ```bash doctruth parse contract.pdf --json -o parsed.json @@ -86,6 +95,55 @@ Show that bbox recovery is enabled in the summary: doctruth parse contract.pdf --bboxes ``` +### Profile + +Measure local parser latency and heap movement without an LLM key: + +```bash +doctruth profile contract.pdf --iterations 3 --json +``` + +Include output serialization cost when comparing parser plus JSON rendering: + +```bash +doctruth profile contract.pdf --iterations 3 --include-output trust-json --json +``` + +Measure the streaming file-writer path used by `parse -o`: + +```bash +doctruth profile contract.pdf --iterations 3 --include-output trust-json-file --json +``` + +The JSON includes: + +```text +parser +iterations +fileSizeBytes +sectionCount +includeOutput +parseLatencyMillis +outputLatencyMillis +coldLatencyMillis +warmAverageLatencyMillis +coldOutputLatencyMillis +warmAverageOutputLatencyMillis +profiledOutputChars +profiledOutputBytes +heapUsedBeforeBytes +heapUsedAfterBytes +heapDeltaBytes +``` + +`trust-json` and `parsed-json` measure string rendering. `trust-json-file` and +`parsed-json-file` measure temporary file writer output and report bytes. +Warm latency is a subsequent full parse in the same JVM, not a persistent +parser-worker residency metric. + +Use this command to compare `opendataloader` and `pdfbox` on a named corpus +before proposing Rust, Go, or other native optimization work. + ### Schema Check a JSON Schema: @@ -112,8 +170,9 @@ By default, DocTruth: - reads provider/model/output defaults from `doctruth.yml` when present - uses `openai` as the default provider -- requires citations for top-level schema fields -- writes `result.json` and `audit.json` to `.doctruth/runs//` +- requires citations for schema leaf fields +- writes `trust-document.json`, `result.json`, `audit.json`, and `manifest.json` + to `.doctruth/runs//` Common overrides: @@ -124,8 +183,16 @@ doctruth extract contract.pdf -s contract.schema.json --model gpt-4o-mini doctruth extract contract.pdf -s contract.schema.json --base-url http://localhost:11434/v1 doctruth extract contract.pdf -s contract.schema.json --allow-uncited doctruth extract contract.pdf -s contract.schema.json --require partyA,totalValue +doctruth extract contract.pdf -s contract.schema.json --evidence-first ``` +`--evidence-first` wraps each schema leaf as `{ "value": ..., "exactQuote": ... }` +for the provider response. DocTruth unwraps `value` into `result.json`, validates +it against the original schema, and uses `exactQuote` for citation matching. + +`audit.json` derivations include the TrustDocument `docId` and `unit` id when a +field citation maps to a parsed unit. + Provider keys: | Provider | Env var | diff --git a/docs/install.md b/docs/install.md index 54bf8a16..c2af9086 100644 --- a/docs/install.md +++ b/docs/install.md @@ -5,7 +5,7 @@ try/debug/inspect path: it lets a Java team verify the core promise before writing integration code: ```text -document -> parsed sections with source locations -> schema check -> audit output +document -> TrustDocument evidence -> schema check -> audit output ``` ## SDK Install @@ -63,7 +63,7 @@ Check the install: ```bash doctruth version doctruth doctor -doctruth parse examples/no-llm-parse/sample-contract.csv +doctruth parse contract.pdf --format json -o trust-document.json ``` If `java` is not on `PATH`, point the launcher at your Java 25 runtime: @@ -86,13 +86,15 @@ java -version No provider key is required for parser and schema inspection: ```bash -doctruth parse examples/no-llm-parse/sample-contract.csv -doctruth parse examples/no-llm-parse/sample-contract.csv --json -o parsed.json +doctruth parse contract.pdf --format json -o trust-document.json +doctruth profile contract.pdf --json doctruth schema examples/pydantic-interop/resume.schema.json ``` This is the recommended first-run path. It proves the document evidence surface -before a user spends time configuring model keys. +before a user spends time configuring model keys. For PDF, the default backend +is OpenDataLoader; `doctruth profile` gives local latency and heap evidence for +the same backend. ## Extraction Run @@ -107,6 +109,11 @@ doctruth audit .doctruth/runs//audit.json Use `--provider`, `--model`, and `--base-url` only when the defaults are not enough. +Each extraction run writes `trust-document.json`, `result.json`, `audit.json`, +and `manifest.json`. Use `--evidence-first` when the provider should return +`value` plus `exactQuote` for each schema leaf before DocTruth validates and +cites the result. + The CLI and SDK use the same parser, citation, provenance, and audit primitives. ## GitHub Release Artifacts diff --git a/docs/parser-profiling.md b/docs/parser-profiling.md new file mode 100644 index 00000000..e49f431a --- /dev/null +++ b/docs/parser-profiling.md @@ -0,0 +1,63 @@ +# Parser Profiling + +DocTruth profiles parser backends before proposing Rust, Go, or native +optimization work. The default PDF backend is OpenDataLoader. + +Run a local profile: + +```bash +doctruth profile contract.pdf --iterations 3 --json +``` + +Include JSON rendering cost when you are profiling parser plus output overhead: + +```bash +doctruth profile contract.pdf --iterations 3 --include-output trust-json --json +``` + +Measure the streaming file-writer path used by `doctruth parse -o`: + +```bash +doctruth profile contract.pdf --iterations 3 --include-output trust-json-file --json +``` + +Compare the legacy PDFBox backend on the same file: + +```bash +doctruth profile contract.pdf --parser pdfbox --iterations 3 --json +``` + +The command reports: + +```text +parser +iterations +fileSizeBytes +sectionCount +includeOutput +parseLatencyMillis +outputLatencyMillis +coldLatencyMillis +warmAverageLatencyMillis +coldOutputLatencyMillis +warmAverageOutputLatencyMillis +profiledOutputChars +profiledOutputBytes +heapUsedBeforeBytes +heapUsedAfterBytes +heapDeltaBytes +``` + +`includeOutput` defaults to `parser-only`. Use `trust-json` for DocTruth's +audit-ready TrustDocument JSON or `parsed-json` for the compatibility parser +debug format. Use `trust-json-file` or `parsed-json-file` when profiling the +streaming writer path instead of string materialization. + +`warmAverageLatencyMillis` means subsequent full parses in the same JVM. It is +not a persistent parser-worker or model-residency measurement; the OpenDataLoader +backend still tears down its static runtime after each parse. + +Treat these numbers as evidence for the named machine, corpus, parser backend, +DocTruth version, and command. They are not global product thresholds. Native +work should target a measured bottleneck while preserving the DocTruth +TrustDocument JSON contract. diff --git a/src/main/java/ai/doctruth/Citation.java b/src/main/java/ai/doctruth/Citation.java index efab12c1..5734cac2 100644 --- a/src/main/java/ai/doctruth/Citation.java +++ b/src/main/java/ai/doctruth/Citation.java @@ -7,7 +7,8 @@ * The verifiable evidence anchor for a single extracted field. A {@code Citation} ties an * extracted value back to a specific span of the source document plus the exact quote * recovered from that span, an optional visual bounding box, and a confidence score for the - * page-attribution match itself. + * page-attribution match itself. When available, {@code source} links the quote to + * the TrustDocument unit that generated the audit entry. * *

Invariants (enforced by the compact constructor): * @@ -19,6 +20,8 @@ * fuzzy / Levenshtein-style matchers. *

  • {@code boundingBox} is non-null; use {@link Optional#empty()} when the source format * has no reliable page geometry. + *
  • {@code source} is non-null; use {@link Optional#empty()} when this citation was not + * produced from a TrustDocument-backed parser run. * * * @param location the source-document span this citation points at. @@ -26,15 +29,21 @@ * @param matchScore similarity of {@code exactQuote} to the substring at {@code location}; * downstream matchers should treat {@code matchScore < 0.85} as a warning. * @param boundingBox optional page-normalized visual region for PDF-originated text. + * @param source optional TrustDocument source identity. * @since 0.1.0 */ public record Citation( - SourceLocation location, String exactQuote, double matchScore, Optional boundingBox) { + SourceLocation location, + String exactQuote, + double matchScore, + Optional boundingBox, + Optional source) { public Citation { Objects.requireNonNull(location, "location"); Objects.requireNonNull(exactQuote, "exactQuote"); Objects.requireNonNull(boundingBox, "boundingBox"); + Objects.requireNonNull(source, "source"); if (exactQuote.isBlank()) { throw new IllegalArgumentException("exactQuote must not be blank"); } @@ -47,6 +56,13 @@ public record Citation( * Backward-compat constructor — leaves the visual bounding box absent. */ public Citation(SourceLocation location, String exactQuote, double matchScore) { - this(location, exactQuote, matchScore, Optional.empty()); + this(location, exactQuote, matchScore, Optional.empty(), Optional.empty()); + } + + /** + * Backward-compat constructor — leaves TrustDocument source identity absent. + */ + public Citation(SourceLocation location, String exactQuote, double matchScore, Optional boundingBox) { + this(location, exactQuote, matchScore, boundingBox, Optional.empty()); } } diff --git a/src/main/java/ai/doctruth/CitationSource.java b/src/main/java/ai/doctruth/CitationSource.java new file mode 100644 index 00000000..8920bf83 --- /dev/null +++ b/src/main/java/ai/doctruth/CitationSource.java @@ -0,0 +1,24 @@ +package ai.doctruth; + +import java.util.Objects; + +/** + * TrustDocument source identity for a citation. + * + * @param docId TrustDocument document id. + * @param unitId TrustDocument unit id inside {@code docId}. + * @since 0.2.0 + */ +public record CitationSource(String docId, String unitId) { + + public CitationSource { + Objects.requireNonNull(docId, "docId"); + Objects.requireNonNull(unitId, "unitId"); + if (docId.isBlank()) { + throw new IllegalArgumentException("docId must not be blank"); + } + if (unitId.isBlank()) { + throw new IllegalArgumentException("unitId must not be blank"); + } + } +} diff --git a/src/main/java/ai/doctruth/Confidence.java b/src/main/java/ai/doctruth/Confidence.java index dde4b706..167b8772 100644 --- a/src/main/java/ai/doctruth/Confidence.java +++ b/src/main/java/ai/doctruth/Confidence.java @@ -3,9 +3,8 @@ import java.util.Objects; /** - * A confidence score for a single extracted field, plus a free-form rationale. The score is - * typically LLM-self-reported (a prior production extraction system-style {@code Field(ge=0.0, le=1.0)}) — the library - * does NOT compute it from signals; it carries through what the model emits. + * A confidence score for a single extracted field, plus a free-form rationale. Scores can come from + * model output or DocTruth-owned evidence signals such as source quote matching. * *

    Invariants (enforced by the compact constructor): * diff --git a/src/main/java/ai/doctruth/DocumentJsonExtractionBuilder.java b/src/main/java/ai/doctruth/DocumentJsonExtractionBuilder.java index 2b362482..4de6a2d9 100644 --- a/src/main/java/ai/doctruth/DocumentJsonExtractionBuilder.java +++ b/src/main/java/ai/doctruth/DocumentJsonExtractionBuilder.java @@ -30,6 +30,10 @@ public DocumentJsonExtractionBuilder requireCitation(String fieldPath) { return new DocumentJsonExtractionBuilder(delegate.requireCitation(fieldPath), document); } + public DocumentJsonExtractionBuilder withEvidenceFirst() { + return new DocumentJsonExtractionBuilder(delegate.withEvidenceFirst(), document); + } + public DocumentJsonExtractionBuilder withMaxRetries(int n) { return new DocumentJsonExtractionBuilder(delegate.withMaxRetries(n), document); } diff --git a/src/main/java/ai/doctruth/EvidenceFirstJson.java b/src/main/java/ai/doctruth/EvidenceFirstJson.java new file mode 100644 index 00000000..2cca0745 --- /dev/null +++ b/src/main/java/ai/doctruth/EvidenceFirstJson.java @@ -0,0 +1,108 @@ +package ai.doctruth; + +import java.util.LinkedHashMap; +import java.util.Map; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +final class EvidenceFirstJson { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private EvidenceFirstJson() { + throw new AssertionError("no instances"); + } + + static JsonNode responseSchema(JsonNode schema) { + if (schema == null || schema.isMissingNode() || schema.isNull()) { + return leafSchema(schema); + } + if ("object".equals(schema.path("type").asText()) && schema.path("properties").isObject()) { + ObjectNode copy = schema.deepCopy(); + ObjectNode properties = MAPPER.createObjectNode(); + schema.path("properties").fields().forEachRemaining(e -> properties.set(e.getKey(), responseSchema(e.getValue()))); + copy.set("properties", properties); + return copy; + } + if ("array".equals(schema.path("type").asText()) && schema.has("items")) { + ObjectNode copy = schema.deepCopy(); + copy.set("items", responseSchema(schema.path("items"))); + return copy; + } + return leafSchema(schema); + } + + static JsonNode unwrap(JsonNode evidenceNode) { + if (isEvidenceLeaf(evidenceNode)) { + return evidenceNode.path("value").deepCopy(); + } + if (evidenceNode != null && evidenceNode.isObject()) { + ObjectNode out = MAPPER.createObjectNode(); + evidenceNode.fields().forEachRemaining(e -> out.set(e.getKey(), unwrap(e.getValue()))); + return out; + } + if (evidenceNode != null && evidenceNode.isArray()) { + ArrayNode out = MAPPER.createArrayNode(); + evidenceNode.forEach(item -> out.add(unwrap(item))); + return out; + } + return evidenceNode == null ? MAPPER.nullNode() : evidenceNode.deepCopy(); + } + + static Map quoteMap(JsonNode evidenceNode) { + var out = new LinkedHashMap(); + collectQuotes("", evidenceNode, out); + return Map.copyOf(out); + } + + private static ObjectNode leafSchema(JsonNode valueSchema) { + ObjectNode wrapper = MAPPER.createObjectNode(); + wrapper.put("type", "object"); + ObjectNode properties = MAPPER.createObjectNode(); + properties.set("value", valueSchema == null || valueSchema.isMissingNode() ? MAPPER.createObjectNode() : valueSchema.deepCopy()); + ObjectNode quote = MAPPER.createObjectNode(); + quote.put("type", "string"); + quote.put("minLength", 1); + properties.set("exactQuote", quote); + wrapper.set("properties", properties); + ArrayNode required = MAPPER.createArrayNode(); + required.add("value"); + required.add("exactQuote"); + wrapper.set("required", required); + wrapper.put("additionalProperties", false); + return wrapper; + } + + private static boolean isEvidenceLeaf(JsonNode node) { + return node != null && node.isObject() && node.has("value") && node.has("exactQuote"); + } + + private static void collectQuotes(String path, JsonNode node, Map out) { + if (node == null || node.isNull() || node.isMissingNode()) { + return; + } + if (isEvidenceLeaf(node)) { + String quote = node.path("exactQuote").asText(); + if (!quote.isBlank()) { + out.put(path, quote); + } + return; + } + if (node.isObject()) { + node.fields().forEachRemaining(e -> collectQuotes(joinPath(path, e.getKey()), e.getValue(), out)); + return; + } + if (node.isArray()) { + for (int i = 0; i < node.size(); i++) { + collectQuotes(path + "[" + i + "]", node.get(i), out); + } + } + } + + private static String joinPath(String parent, String child) { + return parent.isEmpty() ? child : parent + "." + child; + } +} diff --git a/src/main/java/ai/doctruth/JsonExtractionBuilder.java b/src/main/java/ai/doctruth/JsonExtractionBuilder.java index fb3ceaa6..0d18b1e7 100644 --- a/src/main/java/ai/doctruth/JsonExtractionBuilder.java +++ b/src/main/java/ai/doctruth/JsonExtractionBuilder.java @@ -74,6 +74,10 @@ public JsonExtractionBuilder requireCitation(String fieldPath) { return copy(state.requireCitation(fieldPath)); } + public JsonExtractionBuilder withEvidenceFirst() { + return copy(state.withEvidenceFirst()); + } + public ExtractionResult runJson(ParsedDocument doc) throws ExtractionException { Objects.requireNonNull(doc, "doc"); String repairContext = null; @@ -81,9 +85,14 @@ public ExtractionResult runJson(ParsedDocument doc) throws ExtractionE var request = requestFor(doc, repairContext); ProviderResponse response = callProvider(request); try { - JsonNode value = parseJson(response.rawJson(), retry); + JsonNode rawValue = parseJson(response.rawJson(), retry); + if (state.evidenceFirst) { + JsonSchemaValidator.validate(rawValue, EvidenceFirstJson.responseSchema(schema.node()), retry); + } + JsonNode value = state.evidenceFirst ? EvidenceFirstJson.unwrap(rawValue) : rawValue; JsonSchemaValidator.validate(value, schema.node(), retry); - Map citations = citations(value, doc, retry); + Object citationSource = state.evidenceFirst ? EvidenceFirstJson.quoteMap(rawValue) : value; + Map citations = citations(citationSource, doc, retry); return result(response, value, citations, retry); } catch (ExtractionException e) { if (retry == state.maxRetries) { @@ -112,8 +121,9 @@ private ProviderRequest requestFor(ParsedDocument doc, String repairContext) thr if (repairContext != null && !repairContext.isBlank()) { userPrompt = userPrompt + repairInstructions(repairContext); } + JsonNode responseSchema = state.evidenceFirst ? EvidenceFirstJson.responseSchema(schema.node()) : schema.node(); return new ProviderRequest( - prompt, userPrompt, schema.node(), new ProviderOptions(state.maxRetries, DEFAULT_TIMEOUT)); + prompt, userPrompt, responseSchema, new ProviderOptions(state.maxRetries, DEFAULT_TIMEOUT)); } private static String repairInstructions(String repairContext) { @@ -145,12 +155,12 @@ private static JsonNode parseJson(String rawJson, int retries) throws Extraction } } - private Map citations(JsonNode value, ParsedDocument doc, int retries) + private Map citations(Object citationSource, ParsedDocument doc, int retries) throws ExtractionException { if (!state.recordProvenance && !state.recordConfidence && state.requiredCitations.isEmpty()) { return Map.of(); } - Map matched = new CitationMatcher().matchAll(value, doc); + Map matched = new CitationMatcher().matchAll(citationSource, doc); requireStrongCitations(matched, retries); if (state.recordProvenance || state.recordConfidence) { return matched; @@ -194,7 +204,7 @@ private static Map confidenceFrom(Map cita return citations.entrySet().stream() .collect(Collectors.toUnmodifiableMap( Map.Entry::getKey, - e -> new Confidence(e.getValue().matchScore(), "source citation matched for JSON field"))); + e -> new Confidence(e.getValue().matchScore(), "source evidence quote matched for JSON field"))); } private static String renderUserPrompt(ParsedDocument doc) { diff --git a/src/main/java/ai/doctruth/JsonExtractionBuilderState.java b/src/main/java/ai/doctruth/JsonExtractionBuilderState.java index f15bf820..2421eb35 100644 --- a/src/main/java/ai/doctruth/JsonExtractionBuilderState.java +++ b/src/main/java/ai/doctruth/JsonExtractionBuilderState.java @@ -14,6 +14,7 @@ final class JsonExtractionBuilderState { final ContextStrategy contextStrategy; final Instant sourcePublishedAt; final Set requiredCitations; + final boolean evidenceFirst; private JsonExtractionBuilderState( boolean recordProvenance, @@ -22,7 +23,8 @@ private JsonExtractionBuilderState( int maxRetries, ContextStrategy contextStrategy, Instant sourcePublishedAt, - Set requiredCitations) { + Set requiredCitations, + boolean evidenceFirst) { this.recordProvenance = recordProvenance; this.recordBitemporal = recordBitemporal; this.recordConfidence = recordConfidence; @@ -30,10 +32,11 @@ private JsonExtractionBuilderState( this.contextStrategy = contextStrategy; this.sourcePublishedAt = sourcePublishedAt; this.requiredCitations = Set.copyOf(requiredCitations); + this.evidenceFirst = evidenceFirst; } static JsonExtractionBuilderState defaults() { - return new JsonExtractionBuilderState(false, false, false, 0, null, null, Set.of()); + return new JsonExtractionBuilderState(false, false, false, 0, null, null, Set.of(), false); } JsonExtractionBuilderState withProvenance() { @@ -44,7 +47,8 @@ JsonExtractionBuilderState withProvenance() { maxRetries, contextStrategy, sourcePublishedAt, - requiredCitations); + requiredCitations, + evidenceFirst); } JsonExtractionBuilderState withBitemporal() { @@ -55,7 +59,8 @@ JsonExtractionBuilderState withBitemporal() { maxRetries, contextStrategy, sourcePublishedAt, - requiredCitations); + requiredCitations, + evidenceFirst); } JsonExtractionBuilderState withConfidence() { @@ -66,7 +71,8 @@ JsonExtractionBuilderState withConfidence() { maxRetries, contextStrategy, sourcePublishedAt, - requiredCitations); + requiredCitations, + evidenceFirst); } JsonExtractionBuilderState withMaxRetries(int retries) { @@ -80,7 +86,8 @@ JsonExtractionBuilderState withMaxRetries(int retries) { retries, contextStrategy, sourcePublishedAt, - requiredCitations); + requiredCitations, + evidenceFirst); } JsonExtractionBuilderState withContextStrategy(ContextStrategy strategy) { @@ -91,7 +98,8 @@ JsonExtractionBuilderState withContextStrategy(ContextStrategy strategy) { maxRetries, Objects.requireNonNull(strategy, "contextStrategy"), sourcePublishedAt, - requiredCitations); + requiredCitations, + evidenceFirst); } JsonExtractionBuilderState withSourcePublishedAt(Instant publishedAt) { @@ -102,7 +110,8 @@ JsonExtractionBuilderState withSourcePublishedAt(Instant publishedAt) { maxRetries, contextStrategy, Objects.requireNonNull(publishedAt, "sourcePublishedAt"), - requiredCitations); + requiredCitations, + evidenceFirst); } JsonExtractionBuilderState requireCitation(String fieldPath) { @@ -119,7 +128,20 @@ JsonExtractionBuilderState requireCitation(String fieldPath) { maxRetries, contextStrategy, sourcePublishedAt, - next); + next, + evidenceFirst); + } + + JsonExtractionBuilderState withEvidenceFirst() { + return copy( + recordProvenance, + recordBitemporal, + recordConfidence, + maxRetries, + contextStrategy, + sourcePublishedAt, + requiredCitations, + true); } private JsonExtractionBuilderState copy( @@ -129,8 +151,9 @@ private JsonExtractionBuilderState copy( int retries, ContextStrategy context, Instant publishedAt, - Set citations) { + Set citations, + boolean nextEvidenceFirst) { return new JsonExtractionBuilderState( - provenance, bitemporal, confidence, retries, context, publishedAt, citations); + provenance, bitemporal, confidence, retries, context, publishedAt, citations, nextEvidenceFirst); } } diff --git a/src/main/java/ai/doctruth/OpenDataLoaderPdfDocumentParser.java b/src/main/java/ai/doctruth/OpenDataLoaderPdfDocumentParser.java index 8f2fda8e..8df5e0fa 100644 --- a/src/main/java/ai/doctruth/OpenDataLoaderPdfDocumentParser.java +++ b/src/main/java/ai/doctruth/OpenDataLoaderPdfDocumentParser.java @@ -6,6 +6,7 @@ import java.nio.file.Path; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.Comparator; import java.util.HexFormat; import java.util.List; @@ -13,7 +14,7 @@ import java.util.Optional; import java.util.OptionalInt; -import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.ObjectMapper; import org.opendataloader.pdf.api.Config; import org.opendataloader.pdf.api.OpenDataLoaderPDF; @@ -24,6 +25,9 @@ final class OpenDataLoaderPdfDocumentParser { private static final Logger LOG = LoggerFactory.getLogger(OpenDataLoaderPdfDocumentParser.class); private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final Object OPENDATALOADER_LOCK = new Object(); + private static OpenDataLoaderRunner openDataLoaderRunner = + (pdfPath, outputDir, config) -> OpenDataLoaderPDF.processFile(pdfPath.toString(), config); static { suppressOpenDataLoaderInfoLogging(); @@ -53,25 +57,30 @@ static ParsedDocument parse(Path pdfPath) throws ParseException { Path outputDir = null; try { outputDir = Files.createTempDirectory("doctruth-opendataloader-"); - var geometry = OpenDataLoaderPdfGeometry.read(pdfPath); + var geometry = OpenDataLoaderPdfGeometry.lazy(pdfPath); Config config = config(outputDir); - try (var suppression = JulInfoSuppression.open()) { - suppression.keepAlive(); - OpenDataLoaderPDF.processFile(pdfPath.toString(), config); + synchronized (OPENDATALOADER_LOCK) { + try { + try (var suppression = JulInfoSuppression.open()) { + suppression.keepAlive(); + openDataLoaderRunner.process(pdfPath, outputDir, config); + } + } finally { + OpenDataLoaderPDF.shutdown(); + } } Path jsonPath = outputDir.resolve(jsonFilename(pdfPath)); if (!Files.isRegularFile(jsonPath)) { throw new IOException("OpenDataLoader did not produce JSON output: " + jsonPath); } - JsonNode root = MAPPER.readTree(jsonPath.toFile()); - int pageCount = root.path("number of pages").asInt(geometry.pageCount()); - var metadata = new DocumentMetadata(pdfPath.getFileName().toString(), pageCount, Optional.empty()); + var parsed = readOpenDataLoaderJson(jsonPath, geometry); + var metadata = new DocumentMetadata(pdfPath.getFileName().toString(), parsed.pageCount(), Optional.empty()); var docId = "sha256:" + sha256Hex(pdfPath); - var sections = new OpenDataLoaderSectionMapper(geometry).map(root.path("kids")); + var sections = parsed.sections(); LOG.debug( "parsed pdf path={} backend=opendataloader pages={} sections={}", pdfPath, - pageCount, + parsed.pageCount(), sections.size()); return new ParsedDocument(docId, sections, metadata); } catch (IOException | RuntimeException e) { @@ -82,11 +91,87 @@ static ParsedDocument parse(Path pdfPath) throws ParseException { OptionalInt.empty(), e); } finally { - OpenDataLoaderPDF.shutdown(); deleteRecursively(outputDir); } } + static ParsedOpenDataLoaderJson readOpenDataLoaderJson(Path jsonPath, OpenDataLoaderPdfGeometry geometry) + throws IOException { + int pageCount = 0; + var sections = new ArrayList(); + var mapper = new OpenDataLoaderSectionMapper(geometry); + try (var parser = MAPPER.getFactory().createParser(jsonPath.toFile())) { + JsonToken token = parser.nextToken(); + if (token == null) { + throw new IOException("OpenDataLoader JSON is empty: " + jsonPath); + } + if (token != JsonToken.START_OBJECT) { + throw new IOException("OpenDataLoader JSON root must be an object: " + token); + } + while (true) { + token = parser.nextToken(); + if (token == JsonToken.END_OBJECT) { + break; + } + if (token == null) { + throw new IOException("OpenDataLoader JSON ended before the root object closed"); + } + if (token != JsonToken.FIELD_NAME) { + throw new IOException("OpenDataLoader JSON root fields must be named fields: " + token); + } + String field = parser.currentName(); + token = parser.nextToken(); + if (token == null) { + throw new IOException("OpenDataLoader JSON field has no value: " + field); + } + if ("number of pages".equals(field)) { + pageCount = parser.getValueAsInt(0); + } else if ("kids".equals(field)) { + if (token != JsonToken.START_ARRAY) { + throw new IOException("OpenDataLoader kids field must be an array: " + token); + } + while (true) { + token = parser.nextToken(); + if (token == JsonToken.END_ARRAY) { + break; + } + if (token == null) { + throw new IOException("OpenDataLoader JSON kids array ended before it closed"); + } + if (token != JsonToken.START_OBJECT) { + throw new IOException("OpenDataLoader kids entries must be objects: " + token); + } + mapper.append(MAPPER.readTree(parser), sections); + } + } else { + parser.skipChildren(); + } + } + } + return new ParsedOpenDataLoaderJson(pageCount > 0 ? pageCount : geometry.pageCount(), List.copyOf(sections)); + } + + record ParsedOpenDataLoaderJson(int pageCount, List sections) {} + + @FunctionalInterface + interface OpenDataLoaderRunner { + void process(Path pdfPath, Path outputDir, Config config) throws IOException; + } + + static AutoCloseable useOpenDataLoaderRunnerForTesting(OpenDataLoaderRunner runner) { + Objects.requireNonNull(runner, "runner"); + OpenDataLoaderRunner previous; + synchronized (OPENDATALOADER_LOCK) { + previous = openDataLoaderRunner; + openDataLoaderRunner = runner; + } + return () -> { + synchronized (OPENDATALOADER_LOCK) { + openDataLoaderRunner = previous; + } + }; + } + private static Config config(Path outputDir) { var config = new Config(); config.setOutputFolder(outputDir.toString()); diff --git a/src/main/java/ai/doctruth/OpenDataLoaderPdfGeometry.java b/src/main/java/ai/doctruth/OpenDataLoaderPdfGeometry.java index 40084283..8cffa981 100644 --- a/src/main/java/ai/doctruth/OpenDataLoaderPdfGeometry.java +++ b/src/main/java/ai/doctruth/OpenDataLoaderPdfGeometry.java @@ -1,15 +1,31 @@ package ai.doctruth; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Path; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.Optional; import org.apache.pdfbox.Loader; import org.apache.pdfbox.pdmodel.common.PDRectangle; -record OpenDataLoaderPdfGeometry(Map pages) { +final class OpenDataLoaderPdfGeometry { + + private final Map pages; + private final GeometryLoader loader; + private OpenDataLoaderPdfGeometry loaded; + + OpenDataLoaderPdfGeometry(Map pages) { + this.pages = Map.copyOf(Objects.requireNonNull(pages, "pages")); + this.loader = null; + } + + private OpenDataLoaderPdfGeometry(GeometryLoader loader) { + this.pages = null; + this.loader = Objects.requireNonNull(loader, "loader"); + } static OpenDataLoaderPdfGeometry read(Path pdfPath) throws IOException { var pages = new HashMap(); @@ -22,12 +38,39 @@ static OpenDataLoaderPdfGeometry read(Path pdfPath) throws IOException { return new OpenDataLoaderPdfGeometry(pages); } + static OpenDataLoaderPdfGeometry lazy(Path pdfPath) { + return lazy(() -> read(pdfPath)); + } + + static OpenDataLoaderPdfGeometry lazy(GeometryLoader loader) { + return new OpenDataLoaderPdfGeometry(loader); + } + int pageCount() { - return pages.size(); + return materialized().pages.size(); } Optional page(int pageNumber) { - return Optional.ofNullable(pages.get(pageNumber)); + return Optional.ofNullable(materialized().pages.get(pageNumber)); + } + + private synchronized OpenDataLoaderPdfGeometry materialized() { + if (loader == null) { + return this; + } + if (loaded == null) { + try { + loaded = loader.load(); + } catch (IOException e) { + throw new UncheckedIOException("failed to read PDF page geometry", e); + } + } + return loaded; + } + + @FunctionalInterface + interface GeometryLoader { + OpenDataLoaderPdfGeometry load() throws IOException; } record PageGeometry(double width, double height) {} diff --git a/src/main/java/ai/doctruth/OpenDataLoaderSectionMapper.java b/src/main/java/ai/doctruth/OpenDataLoaderSectionMapper.java index ded4a621..a7ef67eb 100644 --- a/src/main/java/ai/doctruth/OpenDataLoaderSectionMapper.java +++ b/src/main/java/ai/doctruth/OpenDataLoaderSectionMapper.java @@ -18,17 +18,17 @@ final class OpenDataLoaderSectionMapper { } List map(JsonNode kids) { - var sections = new ArrayList(); if (kids == null || !kids.isArray()) { - return sections; + return new ArrayList<>(); } + var sections = new ArrayList(kids.size()); for (JsonNode kid : kids) { append(kid, sections); } return sections; } - private void append(JsonNode node, List sections) { + void append(JsonNode node, List sections) { String type = node.path("type").asText(""); switch (type) { case "paragraph", "heading", "list item", "line", "text chunk", "text block" -> @@ -59,8 +59,8 @@ private void appendList(JsonNode node, List sections) { } private void appendTable(JsonNode node, List sections) { - var rows = new ArrayList>(); JsonNode tableRows = node.path("rows"); + var rows = new ArrayList>(tableRows.isArray() ? tableRows.size() : 0); if (tableRows.isArray()) { for (JsonNode row : tableRows) { rows.add(tableCells(row)); @@ -70,8 +70,9 @@ private void appendTable(JsonNode node, List sections) { } private List tableCells(JsonNode row) { - var cells = new ArrayList(); - for (JsonNode cell : row.path("cells")) { + JsonNode cellsNode = row.path("cells"); + var cells = new ArrayList(cellsNode.isArray() ? cellsNode.size() : 0); + for (JsonNode cell : cellsNode) { cells.add(textFrom(cell).trim()); } return cells; @@ -92,43 +93,46 @@ private void appendArray(JsonNode nodes, List sections) { } private String textFrom(JsonNode node) { + var text = new StringBuilder(); + appendTextFrom(node, text); + return text.toString(); + } + + private boolean appendTextFrom(JsonNode node, StringBuilder text) { if (node == null || node.isMissingNode() || node.isNull()) { - return ""; + return false; } if (node.isArray()) { - return textFromArray(node); + return appendTextFromArray(node, text); } String direct = node.path("content").asText(""); if (!direct.isBlank()) { - return direct; + appendTextSegment(text, direct.trim()); + return true; } - var parts = new ArrayList(); - collectText(node.path("kids"), parts); - collectText(node.path("list items"), parts); - return String.join("\n", parts); + boolean appended = appendTextFromArray(node.path("kids"), text); + return appendTextFromArray(node.path("list items"), text) || appended; } - private String textFromArray(JsonNode nodes) { - var parts = new ArrayList(); + private boolean appendTextFromArray(JsonNode nodes, StringBuilder text) { + if (!nodes.isArray()) { + return false; + } + boolean appended = false; for (JsonNode child : nodes) { - String childText = textFrom(child).trim(); - if (!childText.isEmpty()) { - parts.add(childText); - } + appended = appendTextFrom(child, text) || appended; } - return String.join("\n", parts); + return appended; } - private void collectText(JsonNode nodes, List parts) { - if (!nodes.isArray()) { + private static void appendTextSegment(StringBuilder text, String segment) { + if (segment.isEmpty()) { return; } - for (JsonNode child : nodes) { - String childText = textFrom(child).trim(); - if (!childText.isEmpty()) { - parts.add(childText); - } + if (!text.isEmpty()) { + text.append('\n'); } + text.append(segment); } private SourceLocation location(int page) { diff --git a/src/main/java/ai/doctruth/ParserRun.java b/src/main/java/ai/doctruth/ParserRun.java new file mode 100644 index 00000000..a4743328 --- /dev/null +++ b/src/main/java/ai/doctruth/ParserRun.java @@ -0,0 +1,19 @@ +package ai.doctruth; + +import java.util.Objects; + +/** + * Parser provenance attached to a TrustDocument. + * + * @param backend parser backend identifier, for example {@code opendataloader}. + * @since 0.2.0 + */ +public record ParserRun(String backend) { + + public ParserRun { + Objects.requireNonNull(backend, "backend"); + if (backend.isBlank()) { + throw new IllegalArgumentException("backend must not be blank"); + } + } +} diff --git a/src/main/java/ai/doctruth/TrustDocument.java b/src/main/java/ai/doctruth/TrustDocument.java new file mode 100644 index 00000000..38b659b9 --- /dev/null +++ b/src/main/java/ai/doctruth/TrustDocument.java @@ -0,0 +1,104 @@ +package ai.doctruth; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * DocTruth's canonical parser output for audit-ready extraction workflows. + * + * @param schemaVersion stable TrustDocument schema version. + * @param docId parser document id. + * @param source source file identity. + * @param parserRun parser provenance. + * @param units ordered audit-addressable content units. + * @since 0.2.0 + */ +public record TrustDocument( + String schemaVersion, + String docId, + TrustDocumentSource source, + ParserRun parserRun, + List units) { + + public static final String SCHEMA_VERSION = "doctruth.trust-document.v1"; + + public TrustDocument { + Objects.requireNonNull(schemaVersion, "schemaVersion"); + Objects.requireNonNull(docId, "docId"); + Objects.requireNonNull(source, "source"); + Objects.requireNonNull(parserRun, "parserRun"); + Objects.requireNonNull(units, "units"); + if (schemaVersion.isBlank()) { + throw new IllegalArgumentException("schemaVersion must not be blank"); + } + if (docId.isBlank()) { + throw new IllegalArgumentException("docId must not be blank"); + } + units = List.copyOf(units); + } + + public static TrustDocument fromParsed(ParsedDocument parsed, Path sourcePath, PdfParserBackend backend) { + Objects.requireNonNull(parsed, "parsed"); + Objects.requireNonNull(sourcePath, "sourcePath"); + Objects.requireNonNull(backend, "backend"); + return fromParsed(parsed, sourcePath, backend.id()); + } + + public static TrustDocument fromParsed(ParsedDocument parsed, Path sourcePath, String backend) { + Objects.requireNonNull(parsed, "parsed"); + Objects.requireNonNull(sourcePath, "sourcePath"); + Objects.requireNonNull(backend, "backend"); + var source = new TrustDocumentSource( + sourceFilename(sourcePath), sha256FromDocId(parsed.docId()), parsed.metadata().pageCount()); + return new TrustDocument(SCHEMA_VERSION, parsed.docId(), source, new ParserRun(backend), units(parsed)); + } + + private static String sourceFilename(Path sourcePath) { + Path filename = sourcePath.getFileName(); + if (filename == null || filename.toString().isBlank()) { + throw new IllegalArgumentException("sourcePath must include a filename"); + } + return filename.toString(); + } + + private static String sha256FromDocId(String docId) { + if (docId.startsWith("sha256:")) { + return docId.substring("sha256:".length()); + } + throw new IllegalArgumentException("ParsedDocument docId must be sha256-backed"); + } + + private static List units(ParsedDocument parsed) { + var units = new ArrayList(parsed.sections().size()); + for (int i = 0; i < parsed.sections().size(); i++) { + units.add(unit("u" + (i + 1), parsed.sections().get(i))); + } + return units; + } + + private static TrustUnit unit(String id, ParsedSection section) { + return switch (section) { + case TextSection text -> new TrustUnit( + id, + "text", + text.text(), + List.of(), + new TrustUnitEvidence(text.location(), text.boundingBox(), Optional.of(text.kind()))); + case TableSection table -> new TrustUnit( + id, + "table", + "", + table.rows(), + new TrustUnitEvidence(table.location(), Optional.empty(), Optional.empty())); + case FigureSection figure -> new TrustUnit( + id, + "figure", + figure.caption(), + List.of(), + new TrustUnitEvidence(figure.location(), Optional.empty(), Optional.empty())); + }; + } +} diff --git a/src/main/java/ai/doctruth/TrustDocumentJson.java b/src/main/java/ai/doctruth/TrustDocumentJson.java new file mode 100644 index 00000000..d31532c6 --- /dev/null +++ b/src/main/java/ai/doctruth/TrustDocumentJson.java @@ -0,0 +1,228 @@ +package ai.doctruth; + +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * JSON renderer for the stable TrustDocument contract. + * + * @since 0.2.0 + */ +public final class TrustDocumentJson { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final JsonFactory JSON = MAPPER.getFactory(); + + private TrustDocumentJson() { + throw new AssertionError("no instances"); + } + + public static String toJson(TrustDocument doc) { + var out = new StringWriter(); + writeJson(doc, out); + return out.toString(); + } + + public static void writeJson(TrustDocument doc, Writer writer) { + try { + try (JsonGenerator json = jsonGenerator(writer)) { + writeDocument(json, doc); + } + } catch (IOException e) { + throw new IllegalStateException("failed to serialize TrustDocument", e); + } + } + + public static String toJson(ParsedDocument doc, Path source, PdfParserBackend backend) { + Objects.requireNonNull(backend, "backend"); + return toJson(doc, source, backend.id()); + } + + public static String toJson(ParsedDocument doc, Path source, String backend) { + var out = new StringWriter(); + writeJson(doc, source, backend, out); + return out.toString(); + } + + public static void writeJson(ParsedDocument doc, Path source, PdfParserBackend backend, Writer writer) { + Objects.requireNonNull(backend, "backend"); + writeJson(doc, source, backend.id(), writer); + } + + public static void writeJson(ParsedDocument doc, Path source, String backend, Writer writer) { + Objects.requireNonNull(source, "source"); + Objects.requireNonNull(backend, "backend"); + try { + try (JsonGenerator json = jsonGenerator(writer)) { + writeParsedDocument(json, doc, source, backend); + } + } catch (IOException e) { + throw new IllegalStateException("failed to serialize TrustDocument", e); + } + } + + public static void writeJson(ParsedDocument doc, Path source, PdfParserBackend backend, Path output) { + Objects.requireNonNull(backend, "backend"); + writeJson(doc, source, backend.id(), output); + } + + public static void writeJson(ParsedDocument doc, Path source, String backend, Path output) { + try { + Path parent = output.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + try (Writer writer = Files.newBufferedWriter(output)) { + writeJson(doc, source, backend, writer); + } + } catch (IOException e) { + throw new IllegalStateException("failed to write TrustDocument", e); + } + } + + private static JsonGenerator jsonGenerator(Writer writer) throws IOException { + JsonGenerator json = JSON.createGenerator(writer); + json.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET); + return json.useDefaultPrettyPrinter(); + } + + private static void writeDocument(JsonGenerator json, TrustDocument doc) throws IOException { + json.writeStartObject(); + json.writeStringField("schemaVersion", doc.schemaVersion()); + json.writeStringField("docId", doc.docId()); + writeSource(json, doc.source()); + writeParserRun(json, doc.parserRun().backend()); + json.writeArrayFieldStart("units"); + for (TrustUnit unit : doc.units()) { + writeUnit(json, unit); + } + json.writeEndArray(); + json.writeEndObject(); + } + + private static void writeParsedDocument(JsonGenerator json, ParsedDocument doc, Path source, String backend) + throws IOException { + json.writeStartObject(); + json.writeStringField("schemaVersion", TrustDocument.SCHEMA_VERSION); + json.writeStringField("docId", doc.docId()); + writeSource(json, new TrustDocumentSource( + sourceFilename(source), sha256FromDocId(doc.docId()), doc.metadata().pageCount())); + writeParserRun(json, backend); + json.writeArrayFieldStart("units"); + int id = 1; + for (ParsedSection section : doc.sections()) { + writeParsedUnit(json, "u" + id++, section); + } + json.writeEndArray(); + json.writeEndObject(); + } + + private static String sourceFilename(Path source) { + Path filename = source.getFileName(); + if (filename == null || filename.toString().isBlank()) { + throw new IllegalArgumentException("source must include a filename"); + } + return filename.toString(); + } + + private static void writeSource(JsonGenerator json, TrustDocumentSource source) throws IOException { + json.writeObjectFieldStart("source"); + json.writeStringField("filename", source.filename()); + json.writeStringField("sha256", source.sha256()); + json.writeNumberField("pageCount", source.pageCount()); + json.writeEndObject(); + } + + private static void writeParserRun(JsonGenerator json, String backend) throws IOException { + json.writeObjectFieldStart("parserRun"); + json.writeStringField("backend", backend); + json.writeEndObject(); + } + + private static void writeUnit(JsonGenerator json, TrustUnit unit) throws IOException { + json.writeStartObject(); + json.writeStringField("id", unit.id()); + json.writeStringField("type", unit.type()); + if (!unit.text().isEmpty()) { + json.writeStringField("text", unit.text()); + } + if (!unit.rows().isEmpty()) { + json.writeObjectField("rows", unit.rows()); + } + if (unit.evidence().blockKind().isPresent()) { + json.writeStringField("blockKind", unit.evidence().blockKind().get().name()); + } + writeLocation(json, unit.evidence().location()); + if (unit.evidence().boundingBox().isPresent()) { + writeBoundingBox(json, unit.evidence().boundingBox().get()); + } + json.writeEndObject(); + } + + private static void writeParsedUnit(JsonGenerator json, String id, ParsedSection section) throws IOException { + switch (section) { + case TextSection text -> { + json.writeStartObject(); + json.writeStringField("id", id); + json.writeStringField("type", "text"); + json.writeStringField("text", text.text()); + json.writeStringField("blockKind", text.kind().name()); + writeLocation(json, text.location()); + if (text.boundingBox().isPresent()) { + writeBoundingBox(json, text.boundingBox().get()); + } + json.writeEndObject(); + } + case TableSection table -> { + json.writeStartObject(); + json.writeStringField("id", id); + json.writeStringField("type", "table"); + json.writeObjectField("rows", table.rows()); + writeLocation(json, table.location()); + json.writeEndObject(); + } + case FigureSection figure -> { + json.writeStartObject(); + json.writeStringField("id", id); + json.writeStringField("type", "figure"); + json.writeStringField("text", figure.caption()); + writeLocation(json, figure.location()); + json.writeEndObject(); + } + } + } + + private static void writeLocation(JsonGenerator json, SourceLocation location) throws IOException { + json.writeObjectFieldStart("location"); + json.writeNumberField("pageStart", location.pageStart()); + json.writeNumberField("pageEnd", location.pageEnd()); + json.writeNumberField("lineStart", location.lineStart()); + json.writeNumberField("lineEnd", location.lineEnd()); + json.writeNumberField("charOffset", location.charOffset()); + json.writeEndObject(); + } + + private static void writeBoundingBox(JsonGenerator json, BoundingBox box) throws IOException { + json.writeObjectFieldStart("boundingBox"); + json.writeNumberField("x0", box.x0()); + json.writeNumberField("y0", box.y0()); + json.writeNumberField("x1", box.x1()); + json.writeNumberField("y1", box.y1()); + json.writeEndObject(); + } + + private static String sha256FromDocId(String docId) { + if (docId.startsWith("sha256:")) { + return docId.substring("sha256:".length()); + } + throw new IllegalArgumentException("ParsedDocument docId must be sha256-backed"); + } +} diff --git a/src/main/java/ai/doctruth/TrustDocumentParser.java b/src/main/java/ai/doctruth/TrustDocumentParser.java new file mode 100644 index 00000000..4975eae7 --- /dev/null +++ b/src/main/java/ai/doctruth/TrustDocumentParser.java @@ -0,0 +1,26 @@ +package ai.doctruth; + +import java.nio.file.Path; +import java.util.Objects; + +/** + * Public SDK entry point for parsing PDFs into DocTruth's audit-ready TrustDocument format. + * + * @since 0.2.0 + */ +public final class TrustDocumentParser { + + private TrustDocumentParser() { + throw new AssertionError("no instances"); + } + + public static TrustDocument parse(Path pdfPath) throws ParseException { + return parse(pdfPath, PdfParserBackend.OPENDATALOADER); + } + + public static TrustDocument parse(Path pdfPath, PdfParserBackend backend) throws ParseException { + Objects.requireNonNull(pdfPath, "pdfPath"); + Objects.requireNonNull(backend, "backend"); + return TrustDocument.fromParsed(PdfDocumentParser.parse(pdfPath, backend), pdfPath, backend); + } +} diff --git a/src/main/java/ai/doctruth/TrustDocumentSource.java b/src/main/java/ai/doctruth/TrustDocumentSource.java new file mode 100644 index 00000000..b753ac25 --- /dev/null +++ b/src/main/java/ai/doctruth/TrustDocumentSource.java @@ -0,0 +1,28 @@ +package ai.doctruth; + +import java.util.Objects; + +/** + * Stable source identity for a TrustDocument. + * + * @param filename original source filename. + * @param sha256 lowercase SHA-256 digest of the source bytes. + * @param pageCount source page count when known by the parser. + * @since 0.2.0 + */ +public record TrustDocumentSource(String filename, String sha256, int pageCount) { + + public TrustDocumentSource { + Objects.requireNonNull(filename, "filename"); + Objects.requireNonNull(sha256, "sha256"); + if (filename.isBlank()) { + throw new IllegalArgumentException("filename must not be blank"); + } + if (!sha256.matches("[0-9a-f]{64}")) { + throw new IllegalArgumentException("sha256 must be a lowercase hex SHA-256 digest"); + } + if (pageCount < 1) { + throw new IllegalArgumentException("pageCount must be >= 1"); + } + } +} diff --git a/src/main/java/ai/doctruth/TrustUnit.java b/src/main/java/ai/doctruth/TrustUnit.java new file mode 100644 index 00000000..8976ca14 --- /dev/null +++ b/src/main/java/ai/doctruth/TrustUnit.java @@ -0,0 +1,37 @@ +package ai.doctruth; + +import java.util.List; +import java.util.Objects; + +/** + * One audit-addressable content unit in a {@link TrustDocument}. + * + * @param id stable unit id within the document. + * @param type unit type: {@code text}, {@code table}, or {@code figure}. + * @param text recovered text or caption; empty for table-only units. + * @param rows table rows; empty for non-table units. + * @param evidence source anchor and visual evidence. + * @since 0.2.0 + */ +public record TrustUnit( + String id, String type, String text, List> rows, TrustUnitEvidence evidence) { + + public TrustUnit { + Objects.requireNonNull(id, "id"); + Objects.requireNonNull(type, "type"); + Objects.requireNonNull(text, "text"); + Objects.requireNonNull(rows, "rows"); + Objects.requireNonNull(evidence, "evidence"); + if (id.isBlank()) { + throw new IllegalArgumentException("id must not be blank"); + } + if (type.isBlank()) { + throw new IllegalArgumentException("type must not be blank"); + } + rows = copyRows(rows); + } + + private static List> copyRows(List> rows) { + return rows.stream().map(List::copyOf).toList(); + } +} diff --git a/src/main/java/ai/doctruth/TrustUnitEvidence.java b/src/main/java/ai/doctruth/TrustUnitEvidence.java new file mode 100644 index 00000000..bb0b08b9 --- /dev/null +++ b/src/main/java/ai/doctruth/TrustUnitEvidence.java @@ -0,0 +1,22 @@ +package ai.doctruth; + +import java.util.Objects; +import java.util.Optional; + +/** + * Source anchor for one TrustDocument unit. + * + * @param location page and line anchor. + * @param boundingBox optional page-normalized visual evidence region. + * @param blockKind layout class for text units, empty otherwise. + * @since 0.2.0 + */ +public record TrustUnitEvidence( + SourceLocation location, Optional boundingBox, Optional blockKind) { + + public TrustUnitEvidence { + Objects.requireNonNull(location, "location"); + Objects.requireNonNull(boundingBox, "boundingBox"); + Objects.requireNonNull(blockKind, "blockKind"); + } +} diff --git a/src/main/java/ai/doctruth/cli/DocTruthCli.java b/src/main/java/ai/doctruth/cli/DocTruthCli.java index 7d44ebc5..66cd5b1d 100644 --- a/src/main/java/ai/doctruth/cli/DocTruthCli.java +++ b/src/main/java/ai/doctruth/cli/DocTruthCli.java @@ -58,6 +58,7 @@ private int runChecked(String[] args) throws CliException { switch (args[0]) { case "init" -> new InitCommand(context).run(args); case "parse" -> new ParseCommand(context).run(args); + case "profile" -> new ProfileCommand(context).run(args); case "schema" -> new SchemaCommand(context).run(args); case "extract" -> new ExtractCommand(context).run(args); case "audit" -> new AuditCommand(context).run(args); diff --git a/src/main/java/ai/doctruth/cli/DocumentParsers.java b/src/main/java/ai/doctruth/cli/DocumentParsers.java index 0c9e9edd..750706c1 100644 --- a/src/main/java/ai/doctruth/cli/DocumentParsers.java +++ b/src/main/java/ai/doctruth/cli/DocumentParsers.java @@ -35,6 +35,16 @@ static ParsedDocument parse(Path path, PdfParserBackend pdfBackend) throws CliEx } } + static String parserId(Path path) { + return switch (extension(path)) { + case "pdf" -> PdfParserBackend.OPENDATALOADER.id(); + case "docx" -> "docx"; + case "xlsx" -> "xlsx"; + case "csv" -> "csv"; + default -> "unknown"; + }; + } + private static String extension(Path path) { String name = path.getFileName().toString(); int dot = name.lastIndexOf('.'); diff --git a/src/main/java/ai/doctruth/cli/ExtractCommand.java b/src/main/java/ai/doctruth/cli/ExtractCommand.java index 063a0828..25a5935c 100644 --- a/src/main/java/ai/doctruth/cli/ExtractCommand.java +++ b/src/main/java/ai/doctruth/cli/ExtractCommand.java @@ -13,12 +13,18 @@ import ai.doctruth.ExtractionException; import ai.doctruth.ExtractionResult; import ai.doctruth.JsonSchema; +import ai.doctruth.ParsedDocument; +import ai.doctruth.TrustDocumentJson; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; final class ExtractCommand { private static final String DEFAULT_PROMPT = "Extract the document fields according to the supplied schema."; + private static final String RUN_SCHEMA_VERSION = "doctruth.extract-run.v1"; + private static final ObjectMapper MAPPER = new ObjectMapper(); private final CliContext context; @@ -34,7 +40,7 @@ void run(String[] args) throws CliException { var provider = context.providers().create(options.providerConfig(config)); var result = runExtraction(doc, schema, provider, options); Path dir = options.out().orElseGet(() -> defaultRunDir(config)); - writeOutputs(dir, result); + writeOutputs(dir, options.document(), doc, result); printSummary(dir, result); } @@ -48,6 +54,9 @@ private static ExtractionResult runExtraction( .withConfidence() .withBitemporal() .withMaxRetries(2); + if (options.evidenceFirst()) { + builder = builder.withEvidenceFirst(); + } if (!options.allowUncited()) { for (String field : options.requiredFields(schema)) { builder = builder.requireCitation(field); @@ -59,16 +68,33 @@ private static ExtractionResult runExtraction( } } - private static void writeOutputs(Path dir, ExtractionResult result) throws CliException { + private static void writeOutputs( + Path dir, Path source, ParsedDocument doc, ExtractionResult result) throws CliException { try { Files.createDirectories(dir); + TrustDocumentJson.writeJson(doc, source, DocumentParsers.parserId(source), dir.resolve("trust-document.json")); Files.writeString(dir.resolve("result.json"), result.value().toPrettyString()); result.toAuditJson(dir.resolve("audit.json")); + Files.writeString(dir.resolve("manifest.json"), manifest(source, doc).toPrettyString()); } catch (IOException e) { throw new CliException("failed to write extraction outputs: " + e.getMessage(), e); } } + private static ObjectNode manifest(Path source, ParsedDocument doc) { + ObjectNode root = MAPPER.createObjectNode(); + root.put("schemaVersion", RUN_SCHEMA_VERSION); + root.put("source", source.toString()); + root.put("docId", doc.docId()); + root.put("parser", DocumentParsers.parserId(source)); + ObjectNode artifacts = MAPPER.createObjectNode(); + artifacts.put("trustDocument", "trust-document.json"); + artifacts.put("result", "result.json"); + artifacts.put("audit", "audit.json"); + root.set("artifacts", artifacts); + return root; + } + private void printSummary(Path dir, ExtractionResult result) { int fields = result.value().isObject() ? result.value().size() : 1; long weak = result.citations().values().stream() @@ -95,6 +121,7 @@ private record ExtractOptions( Optional model, Optional baseUrl, boolean allowUncited, + boolean evidenceFirst, Set require, String prompt) { @@ -109,6 +136,7 @@ static ExtractOptions parse(String[] args) { Optional model = Optional.empty(); Optional baseUrl = Optional.empty(); boolean allowUncited = false; + boolean evidenceFirst = false; Set require = new LinkedHashSet<>(); String prompt = DEFAULT_PROMPT; var cursor = new ArgCursor(args, 2); @@ -121,6 +149,7 @@ static ExtractOptions parse(String[] args) { case "--model" -> model = Optional.of(cursor.next()); case "--base-url" -> baseUrl = Optional.of(URI.create(cursor.next())); case "--allow-uncited" -> allowUncited = true; + case "--evidence-first" -> evidenceFirst = true; case "--require" -> addRequired(require, cursor.next()); case "--prompt" -> prompt = cursor.next(); default -> throw new UsageException("unknown extract option: " + arg); @@ -137,6 +166,7 @@ static ExtractOptions parse(String[] args) { model, baseUrl, allowUncited, + evidenceFirst, Set.copyOf(require), prompt); } @@ -149,14 +179,30 @@ Set requiredFields(JsonSchema schema) { if (!require.isEmpty()) { return require; } - var properties = schema.node().path("properties"); var fields = new LinkedHashSet(); - if (properties.isObject()) { - properties.fieldNames().forEachRemaining(fields::add); - } + collectLeafFields("", schema.node(), fields); return Set.copyOf(fields); } + private static void collectLeafFields(String path, JsonNode schema, Set fields) { + var properties = schema.path("properties"); + if ("object".equals(schema.path("type").asText()) && properties.isObject()) { + properties.fields().forEachRemaining(e -> collectLeafFields(joinPath(path, e.getKey()), e.getValue(), fields)); + return; + } + if ("array".equals(schema.path("type").asText()) && schema.has("items")) { + collectLeafFields(path, schema.path("items"), fields); + return; + } + if (!path.isBlank()) { + fields.add(path); + } + } + + private static String joinPath(String parent, String child) { + return parent.isEmpty() ? child : parent + "." + child; + } + private static void addRequired(Set require, String csv) { for (String field : csv.split(",")) { if (!field.isBlank()) { diff --git a/src/main/java/ai/doctruth/cli/ParseCommand.java b/src/main/java/ai/doctruth/cli/ParseCommand.java index ad915711..3688023b 100644 --- a/src/main/java/ai/doctruth/cli/ParseCommand.java +++ b/src/main/java/ai/doctruth/cli/ParseCommand.java @@ -1,33 +1,89 @@ package ai.doctruth.cli; -import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; +import ai.doctruth.ParsedDocument; import ai.doctruth.PdfParserBackend; +import ai.doctruth.TrustDocumentJson; final class ParseCommand { private final CliContext context; + private final Parser parser; + private final JsonRenderer jsonRenderer; ParseCommand(CliContext context) { + this(context, DocumentParsers::parse, new JsonRenderer() { + @Override + public String render(ParsedDocument doc, Path source, PdfParserBackend parser, boolean trustDocument) + throws CliException { + return renderJson(doc, source, parser, trustDocument); + } + + @Override + public void write( + ParsedDocument doc, Path source, PdfParserBackend parser, boolean trustDocument, Path out) + throws CliException { + writeJson(doc, source, parser, trustDocument, out); + } + }); + } + + ParseCommand(CliContext context, Parser parser, JsonRenderer jsonRenderer) { this.context = context; + this.parser = parser; + this.jsonRenderer = jsonRenderer; } void run(String[] args) throws CliException { var options = ParseOptions.parse(args); - var doc = DocumentParsers.parse(options.document(), options.parser()); - String json = ParsedDocumentJson.toJson(doc); + var doc = parser.parse(options.document(), options.parser()); if (options.out() != null) { - write(options.out(), json); + writeJson(doc, options); } - if (options.json() && options.out() == null) { - context.out().println(json); + if (options.shouldPrintJson() && options.out() == null) { + context.out().println(json(doc, options)); return; } printSummary(options.document(), doc, options); } + private String json(ParsedDocument doc, ParseOptions options) throws CliException { + return jsonRenderer.render( + doc, options.document(), options.parser(), options.format() == OutputFormat.TRUST_DOCUMENT_JSON); + } + + private void writeJson(ParsedDocument doc, ParseOptions options) throws CliException { + jsonRenderer.write( + doc, + options.document(), + options.parser(), + options.format() == OutputFormat.TRUST_DOCUMENT_JSON, + options.out()); + } + + private static String renderJson(ParsedDocument doc, Path source, PdfParserBackend parser, boolean trustDocument) + throws CliException { + if (trustDocument) { + return TrustDocumentJson.toJson(doc, source, parser); + } + return ParsedDocumentJson.toJson(doc); + } + + private static void writeJson( + ParsedDocument doc, Path source, PdfParserBackend parser, boolean trustDocument, Path out) + throws CliException { + if (trustDocument) { + try { + TrustDocumentJson.writeJson(doc, source, parser, out); + } catch (IllegalStateException e) { + throw new CliException("failed to write TrustDocument JSON: " + e.getMessage(), e); + } + return; + } + ParsedDocumentJson.writeJson(doc, out); + } + private void printSummary(Path source, ai.doctruth.ParsedDocument doc, ParseOptions options) { var stats = ParsedDocumentStats.from(doc); context.out().println(source); @@ -43,41 +99,54 @@ private void printSummary(Path source, ai.doctruth.ParsedDocument doc, ParseOpti } } - private static void write(Path out, String json) throws CliException { - try { - Path parent = out.getParent(); - if (parent != null) { - Files.createDirectories(parent); - } - Files.writeString(out, json); - } catch (IOException e) { - throw new CliException("failed to write parsed JSON: " + e.getMessage(), e); - } + @FunctionalInterface + interface Parser { + ParsedDocument parse(Path path, PdfParserBackend parser) throws CliException; + } + + interface JsonRenderer { + String render(ParsedDocument doc, Path source, PdfParserBackend parser, boolean trustDocument) + throws CliException; + + void write(ParsedDocument doc, Path source, PdfParserBackend parser, boolean trustDocument, Path out) + throws CliException; + } + + enum OutputFormat { + PARSED_JSON, + TRUST_DOCUMENT_JSON } - private record ParseOptions(Path document, boolean json, boolean bboxes, Path out, PdfParserBackend parser) { + private record ParseOptions( + Path document, boolean legacyJson, boolean bboxes, Path out, PdfParserBackend parser, OutputFormat format) { static ParseOptions parse(String[] args) { if (args.length < 2) { throw new UsageException( - "usage: doctruth parse [--parser opendataloader|pdfbox] [--json] [--bboxes] [-o parsed.json]"); + "usage: doctruth parse [--parser opendataloader|pdfbox] [--format json|parsed-json] [--json] [--bboxes] [-o parsed.json]"); } Path document = Path.of(args[1]); boolean json = false; boolean bboxes = false; Path out = null; PdfParserBackend parser = PdfParserBackend.OPENDATALOADER; + OutputFormat format = OutputFormat.PARSED_JSON; var cursor = new ArgCursor(args, 2); while (cursor.hasNext()) { String arg = cursor.next(); switch (arg) { case "--json" -> json = true; case "--bboxes" -> bboxes = true; + case "--format" -> format = parseFormat(nextValue(cursor, arg)); case "--parser" -> parser = parseBackend(nextValue(cursor, arg)); case "-o", "--out" -> out = cursor.nextPath(arg); default -> throw new UsageException("unknown parse option: " + arg); } } - return new ParseOptions(document, json, bboxes, out, parser); + return new ParseOptions(document, json, bboxes, out, parser, format); + } + + boolean shouldPrintJson() { + return legacyJson || format == OutputFormat.TRUST_DOCUMENT_JSON; } private static PdfParserBackend parseBackend(String raw) { @@ -88,6 +157,14 @@ private static PdfParserBackend parseBackend(String raw) { } } + private static OutputFormat parseFormat(String raw) { + return switch (raw) { + case "json", "trust-json", "trust-document-json" -> OutputFormat.TRUST_DOCUMENT_JSON; + case "parsed-json" -> OutputFormat.PARSED_JSON; + default -> throw new UsageException("unsupported parse format: " + raw); + }; + } + private static String nextValue(ArgCursor cursor, String option) { if (!cursor.hasNext()) { throw new UsageException(option + " requires a value"); diff --git a/src/main/java/ai/doctruth/cli/ParsedDocumentJson.java b/src/main/java/ai/doctruth/cli/ParsedDocumentJson.java index 91c9c28f..df5d75fb 100644 --- a/src/main/java/ai/doctruth/cli/ParsedDocumentJson.java +++ b/src/main/java/ai/doctruth/cli/ParsedDocumentJson.java @@ -1,15 +1,21 @@ package ai.doctruth.cli; +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; +import java.nio.file.Files; +import java.nio.file.Path; + import ai.doctruth.BoundingBox; import ai.doctruth.FigureSection; import ai.doctruth.ParsedDocument; +import ai.doctruth.ParsedSection; import ai.doctruth.SourceLocation; import ai.doctruth.TableSection; import ai.doctruth.TextSection; +import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; final class ParsedDocumentJson { @@ -22,60 +28,112 @@ private ParsedDocumentJson() { static String toJson(ParsedDocument doc) throws CliException { try { - return MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(toNode(doc)); - } catch (com.fasterxml.jackson.core.JsonProcessingException e) { + var out = new StringWriter(); + writeJson(doc, out); + return out.toString(); + } catch (IOException e) { throw new CliException("failed to serialize parsed document", e); } } - private static ObjectNode toNode(ParsedDocument doc) { - ObjectNode root = MAPPER.createObjectNode(); - root.put("docId", doc.docId()); - ObjectNode metadata = MAPPER.createObjectNode(); - metadata.put("sourceFilename", doc.metadata().sourceFilename()); - metadata.put("pageCount", doc.metadata().pageCount()); - doc.metadata().sourcePublishedAt().ifPresent(t -> metadata.put("sourcePublishedAt", t.toString())); - root.set("metadata", metadata); - ArrayNode sections = MAPPER.createArrayNode(); - doc.sections().forEach(section -> { - switch (section) { - case TextSection text -> sections.add(textNode(text)); - case TableSection table -> sections.add(tableNode(table)); - case FigureSection figure -> sections.add(figureNode(figure)); + static void writeJson(ParsedDocument doc, Path output) throws CliException { + try { + Path parent = output.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + try (Writer writer = Files.newBufferedWriter(output)) { + writeJson(doc, writer); } - }); - root.set("sections", sections); - return root; + } catch (IOException e) { + throw new CliException("failed to write parsed document JSON: " + e.getMessage(), e); + } + } + + private static void writeJson(ParsedDocument doc, Writer writer) throws IOException { + try (JsonGenerator json = MAPPER.getFactory().createGenerator(writer).useDefaultPrettyPrinter()) { + writeDocument(json, doc); + } } - private static ObjectNode textNode(TextSection section) { - ObjectNode node = base("text", section.location()); - node.put("kind", section.kind().name()); - node.put("text", section.text()); - section.boundingBox().ifPresent(box -> node.set("boundingBox", bbox(box))); - return node; + private static void writeDocument(JsonGenerator json, ParsedDocument doc) throws IOException { + json.writeStartObject(); + json.writeStringField("docId", doc.docId()); + json.writeObjectFieldStart("metadata"); + json.writeStringField("sourceFilename", doc.metadata().sourceFilename()); + json.writeNumberField("pageCount", doc.metadata().pageCount()); + if (doc.metadata().sourcePublishedAt().isPresent()) { + json.writeStringField("sourcePublishedAt", doc.metadata().sourcePublishedAt().get().toString()); + } + json.writeEndObject(); + json.writeArrayFieldStart("sections"); + for (ParsedSection section : doc.sections()) { + writeSection(json, section); + } + json.writeEndArray(); + json.writeEndObject(); + } + + private static void writeSection(JsonGenerator json, ParsedSection section) throws IOException { + switch (section) { + case TextSection text -> writeTextSection(json, text); + case TableSection table -> writeTableSection(json, table); + case FigureSection figure -> writeFigureSection(json, figure); + } + } + + private static void writeTextSection(JsonGenerator json, TextSection section) throws IOException { + writeSectionStart(json, "text", section.location()); + json.writeStringField("kind", section.kind().name()); + json.writeStringField("text", section.text()); + if (section.boundingBox().isPresent()) { + writeBoundingBox(json, section.boundingBox().get()); + } + json.writeEndObject(); + } + + private static void writeTableSection(JsonGenerator json, TableSection section) throws IOException { + writeSectionStart(json, "table", section.location()); + json.writeArrayFieldStart("rows"); + for (var row : section.rows()) { + json.writeStartArray(); + for (String cell : row) { + json.writeString(cell); + } + json.writeEndArray(); + } + json.writeEndArray(); + json.writeEndObject(); } - private static ObjectNode tableNode(TableSection section) { - ObjectNode node = base("table", section.location()); - node.set("rows", MAPPER.valueToTree(section.rows())); - return node; + private static void writeFigureSection(JsonGenerator json, FigureSection section) throws IOException { + writeSectionStart(json, "figure", section.location()); + json.writeStringField("caption", section.caption()); + json.writeEndObject(); } - private static ObjectNode figureNode(FigureSection section) { - ObjectNode node = base("figure", section.location()); - node.put("caption", section.caption()); - return node; + private static void writeSectionStart(JsonGenerator json, String type, SourceLocation location) throws IOException { + json.writeStartObject(); + json.writeStringField("type", type); + writeLocation(json, location); } - private static ObjectNode base(String type, SourceLocation location) { - ObjectNode node = MAPPER.createObjectNode(); - node.put("type", type); - node.set("location", MAPPER.valueToTree(location)); - return node; + private static void writeLocation(JsonGenerator json, SourceLocation location) throws IOException { + json.writeObjectFieldStart("location"); + json.writeNumberField("pageStart", location.pageStart()); + json.writeNumberField("pageEnd", location.pageEnd()); + json.writeNumberField("lineStart", location.lineStart()); + json.writeNumberField("lineEnd", location.lineEnd()); + json.writeNumberField("charOffset", location.charOffset()); + json.writeEndObject(); } - private static ObjectNode bbox(BoundingBox box) { - return MAPPER.valueToTree(box); + private static void writeBoundingBox(JsonGenerator json, BoundingBox box) throws IOException { + json.writeObjectFieldStart("boundingBox"); + json.writeNumberField("x0", box.x0()); + json.writeNumberField("y0", box.y0()); + json.writeNumberField("x1", box.x1()); + json.writeNumberField("y1", box.y1()); + json.writeEndObject(); } } diff --git a/src/main/java/ai/doctruth/cli/ProfileCommand.java b/src/main/java/ai/doctruth/cli/ProfileCommand.java new file mode 100644 index 00000000..6e86a224 --- /dev/null +++ b/src/main/java/ai/doctruth/cli/ProfileCommand.java @@ -0,0 +1,296 @@ +package ai.doctruth.cli; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import ai.doctruth.ParsedDocument; +import ai.doctruth.PdfParserBackend; +import ai.doctruth.TrustDocumentJson; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +final class ProfileCommand { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private final CliContext context; + + ProfileCommand(CliContext context) { + this.context = context; + } + + void run(String[] args) throws CliException { + var options = ProfileOptions.parse(args); + var result = measure(options); + if (options.json()) { + context.out().println(result.toJson()); + return; + } + context.out().println(options.document()); + context.out().println("parser: " + result.parser()); + context.out().println("iterations: " + result.iterations()); + context.out().println("file size bytes: " + result.fileSizeBytes()); + context.out().println("sections: " + result.sectionCount()); + context.out().println("include output: " + result.includeOutput().id()); + context.out().println("cold latency ms: " + result.coldLatencyMillis()); + context.out().println("warm avg latency ms: " + result.warmAverageLatencyMillis()); + if (result.includeOutput() != IncludeOutput.PARSER_ONLY) { + context.out().println("cold output latency ms: " + result.coldOutputLatencyMillis()); + context.out().println("warm avg output latency ms: " + result.warmAverageOutputLatencyMillis()); + } + context.out().println("heap used before bytes: " + result.heapUsedBeforeBytes()); + context.out().println("heap used after bytes: " + result.heapUsedAfterBytes()); + } + + private static ProfileResult measure(ProfileOptions options) throws CliException { + long fileSize = fileSize(options.document()); + long heapBefore = usedHeap(); + var parseLatencies = new ArrayList(options.iterations()); + var outputLatencies = new ArrayList(options.iterations()); + int sectionCount = 0; + long outputChars = 0; + long outputBytes = 0; + for (int i = 0; i < options.iterations(); i++) { + long start = System.nanoTime(); + ParsedDocument doc = DocumentParsers.parse(options.document(), options.parser()); + parseLatencies.add(millisSince(start)); + sectionCount = doc.sections().size(); + if (options.includeOutput() != IncludeOutput.PARSER_ONLY) { + long outputStart = System.nanoTime(); + var output = renderOutput(doc, options); + outputChars += output.chars(); + outputBytes += output.bytes(); + outputLatencies.add(millisSince(outputStart)); + } + } + long heapAfter = usedHeap(); + return new ProfileResult( + options.parser().id(), + options.iterations(), + fileSize, + sectionCount, + options.includeOutput(), + copyOf(parseLatencies), + copyOf(outputLatencies), + outputChars, + outputBytes, + heapBefore, + heapAfter); + } + + private static OutputMeasurement renderOutput(ParsedDocument doc, ProfileOptions options) throws CliException { + return switch (options.includeOutput()) { + case PARSER_ONLY -> new OutputMeasurement(0, 0); + case TRUST_JSON -> new OutputMeasurement( + TrustDocumentJson.toJson(doc, options.document(), options.parser()).length(), 0); + case PARSED_JSON -> new OutputMeasurement(ParsedDocumentJson.toJson(doc).length(), 0); + case TRUST_JSON_FILE -> renderFileOutput(output -> TrustDocumentJson.writeJson( + doc, options.document(), options.parser(), output)); + case PARSED_JSON_FILE -> renderFileOutput(output -> ParsedDocumentJson.writeJson(doc, output)); + }; + } + + private static OutputMeasurement renderFileOutput(FileOutputWriter writer) throws CliException { + Path output = null; + try { + output = Files.createTempFile("doctruth-profile-output-", ".json"); + writer.write(output); + return new OutputMeasurement(0, Files.size(output)); + } catch (CliException e) { + throw e; + } catch (java.io.IOException | RuntimeException e) { + throw new CliException("failed to render profile output: " + e.getMessage(), e); + } finally { + if (output != null) { + try { + Files.deleteIfExists(output); + } catch (java.io.IOException e) { + // best effort cleanup for a temporary profile artifact + } + } + } + } + + private static long fileSize(Path path) throws CliException { + try { + return Files.size(path); + } catch (java.io.IOException e) { + throw new CliException("failed to inspect profile document: " + e.getMessage(), e); + } + } + + private static long[] copyOf(List values) { + long[] copy = new long[values.size()]; + for (int i = 0; i < values.size(); i++) { + copy[i] = values.get(i); + } + return copy; + } + + private static long millisSince(long startNanos) { + return Math.max(0, (System.nanoTime() - startNanos) / 1_000_000); + } + + private static long usedHeap() { + Runtime runtime = Runtime.getRuntime(); + return runtime.totalMemory() - runtime.freeMemory(); + } + + @FunctionalInterface + private interface FileOutputWriter { + void write(Path output) throws java.io.IOException, CliException; + } + + private record OutputMeasurement(long chars, long bytes) {} + + private enum IncludeOutput { + PARSER_ONLY("parser-only"), + TRUST_JSON("trust-json"), + PARSED_JSON("parsed-json"), + TRUST_JSON_FILE("trust-json-file"), + PARSED_JSON_FILE("parsed-json-file"); + + private final String id; + + IncludeOutput(String id) { + this.id = id; + } + + String id() { + return id; + } + + static IncludeOutput parse(String raw) { + return switch (raw) { + case "parser-only", "none" -> PARSER_ONLY; + case "trust-json", "json", "trust-document-json" -> TRUST_JSON; + case "parsed-json" -> PARSED_JSON; + case "trust-json-file", "trust-document-json-file", "json-file" -> TRUST_JSON_FILE; + case "parsed-json-file" -> PARSED_JSON_FILE; + default -> throw new UsageException("unsupported profile output mode: " + raw); + }; + } + } + + private record ProfileOptions( + Path document, PdfParserBackend parser, int iterations, boolean json, IncludeOutput includeOutput) { + static ProfileOptions parse(String[] args) { + if (args.length < 2) { + throw new UsageException( + "usage: doctruth profile [--parser opendataloader|pdfbox] [--iterations n] [--include-output parser-only|trust-json|parsed-json] [--json]"); + } + Path document = Path.of(args[1]); + PdfParserBackend parser = PdfParserBackend.OPENDATALOADER; + int iterations = 3; + boolean json = false; + IncludeOutput includeOutput = IncludeOutput.PARSER_ONLY; + var cursor = new ArgCursor(args, 2); + while (cursor.hasNext()) { + String arg = cursor.next(); + switch (arg) { + case "--json" -> json = true; + case "--parser" -> parser = parseBackend(nextValue(cursor, arg)); + case "--iterations" -> iterations = parseIterations(nextValue(cursor, arg)); + case "--include-output" -> includeOutput = IncludeOutput.parse(nextValue(cursor, arg)); + default -> throw new UsageException("unknown profile option: " + arg); + } + } + return new ProfileOptions(document, parser, iterations, json, includeOutput); + } + + private static PdfParserBackend parseBackend(String raw) { + try { + return PdfParserBackend.fromId(raw); + } catch (IllegalArgumentException e) { + throw new UsageException(e.getMessage()); + } + } + + private static int parseIterations(String raw) { + try { + int value = Integer.parseInt(raw); + if (value < 1) { + throw new UsageException("--iterations must be >= 1"); + } + return value; + } catch (NumberFormatException e) { + throw new UsageException("--iterations must be an integer"); + } + } + + private static String nextValue(ArgCursor cursor, String option) { + if (!cursor.hasNext()) { + throw new UsageException(option + " requires a value"); + } + return cursor.next(); + } + } + + private record ProfileResult( + String parser, + int iterations, + long fileSizeBytes, + int sectionCount, + IncludeOutput includeOutput, + long[] parseLatencyMillis, + long[] outputLatencyMillis, + long profiledOutputChars, + long profiledOutputBytes, + long heapUsedBeforeBytes, + long heapUsedAfterBytes) { + long coldLatencyMillis() { + return parseLatencyMillis.length == 0 ? -1 : parseLatencyMillis[0]; + } + + long warmAverageLatencyMillis() { + return warmAverage(parseLatencyMillis); + } + + long coldOutputLatencyMillis() { + return outputLatencyMillis.length == 0 ? -1 : outputLatencyMillis[0]; + } + + long warmAverageOutputLatencyMillis() { + return warmAverage(outputLatencyMillis); + } + + String toJson() throws CliException { + try { + ObjectNode node = MAPPER.createObjectNode(); + node.put("parser", parser); + node.put("iterations", iterations); + node.put("fileSizeBytes", fileSizeBytes); + node.put("sectionCount", sectionCount); + node.put("includeOutput", includeOutput.id()); + node.set("parseLatencyMillis", MAPPER.valueToTree(parseLatencyMillis)); + node.set("outputLatencyMillis", MAPPER.valueToTree(outputLatencyMillis)); + node.put("coldLatencyMillis", coldLatencyMillis()); + node.put("warmAverageLatencyMillis", warmAverageLatencyMillis()); + node.put("coldOutputLatencyMillis", coldOutputLatencyMillis()); + node.put("warmAverageOutputLatencyMillis", warmAverageOutputLatencyMillis()); + node.put("profiledOutputChars", profiledOutputChars); + node.put("profiledOutputBytes", profiledOutputBytes); + node.put("heapUsedBeforeBytes", heapUsedBeforeBytes); + node.put("heapUsedAfterBytes", heapUsedAfterBytes); + node.put("heapDeltaBytes", heapUsedAfterBytes - heapUsedBeforeBytes); + return MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(node); + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { + throw new CliException("failed to serialize profile JSON", e); + } + } + + private static long warmAverage(long[] values) { + if (values.length <= 1) { + return -1; + } + long total = 0; + for (int i = 1; i < values.length; i++) { + total += values[i]; + } + return total / (values.length - 1); + } + } +} diff --git a/src/main/java/ai/doctruth/cli/Usage.java b/src/main/java/ai/doctruth/cli/Usage.java index af341b64..c4ac1d84 100644 --- a/src/main/java/ai/doctruth/cli/Usage.java +++ b/src/main/java/ai/doctruth/cli/Usage.java @@ -12,16 +12,18 @@ static String main() { Usage: doctruth init - doctruth parse [--parser opendataloader|pdfbox] [--json] [--bboxes] [-o parsed.json] + doctruth parse [--parser opendataloader|pdfbox] [--format json|parsed-json] [--json] [--bboxes] [-o parsed.json] + doctruth profile [--parser opendataloader|pdfbox] [--iterations n] [--include-output parser-only|trust-json|trust-json-file|parsed-json|parsed-json-file] [--json] doctruth schema [--json] - doctruth extract -s [-o out/] + doctruth extract -s [--evidence-first] [-o out/] doctruth audit [--json] doctruth doctor [--json] doctruth completion doctruth version Common: - doctruth parse contract.pdf + doctruth parse contract.pdf --format json -o trust-document.json + doctruth profile contract.pdf --include-output trust-json --json doctruth schema contract.schema.json doctruth extract contract.pdf -s contract.schema.json doctruth doctor diff --git a/src/main/java/ai/doctruth/internal/audit/ProvOExporter.java b/src/main/java/ai/doctruth/internal/audit/ProvOExporter.java index a79f065b..3876212c 100644 --- a/src/main/java/ai/doctruth/internal/audit/ProvOExporter.java +++ b/src/main/java/ai/doctruth/internal/audit/ProvOExporter.java @@ -109,6 +109,10 @@ private static ObjectNode derivationEntry(String path, Citation citation) { entry.put("doctruth:matchScore", citation.matchScore()); entry.set("doctruth:sourceLocation", locationNode(citation.location())); citation.boundingBox().ifPresent(box -> entry.set("doctruth:boundingBox", MAPPER.valueToTree(box))); + citation.source().ifPresent(source -> { + entry.put("doctruth:sourceDocId", source.docId()); + entry.put("doctruth:sourceUnitId", source.unitId()); + }); return entry; } diff --git a/src/main/java/ai/doctruth/internal/citation/CitationMatcher.java b/src/main/java/ai/doctruth/internal/citation/CitationMatcher.java index 413c428c..51c66bbc 100644 --- a/src/main/java/ai/doctruth/internal/citation/CitationMatcher.java +++ b/src/main/java/ai/doctruth/internal/citation/CitationMatcher.java @@ -10,6 +10,7 @@ import ai.doctruth.BoundingBox; import ai.doctruth.Citation; +import ai.doctruth.CitationSource; import ai.doctruth.FigureSection; import ai.doctruth.ParsedDocument; import ai.doctruth.ParsedSection; @@ -63,7 +64,7 @@ public Map matchAll(Object value, ParsedDocument doc) { var leaves = new ArrayList(); traverse("", value, leaves); var sections = renderedSections(doc); - var fallback = fallbackLocation(doc); + var fallback = new SourceLocation(1, 1, 1, 1, 0); var out = new LinkedHashMap(); for (var leaf : leaves) { out.put(leaf.path(), matchOne(leaf.value(), leaf.path(), sections, fallback)); @@ -75,7 +76,7 @@ private Citation matchOne(String needle, String path, List sections, S for (var sec : sections) { int idx = sec.text().indexOf(needle); if (idx >= 0) { - return new Citation(sec.location(), needle, 1.0, sec.boundingBox()); + return citation(sec, needle, 1.0); } } var best = bestFuzzy(needle, sections); @@ -96,7 +97,7 @@ private Citation matchOne(String needle, String path, List sections, S private static Citation bestFuzzy(String needle, List sections) { Citation best = null; for (var sec : sections) { - var c = bestFuzzyWindow(needle, sec.text(), sec.location(), sec.boundingBox()); + var c = bestFuzzyWindow(needle, sec); if (c == null) { continue; } @@ -107,8 +108,8 @@ private static Citation bestFuzzy(String needle, List sections) { return best; } - private static Citation bestFuzzyWindow( - String needle, String haystack, SourceLocation loc, Optional boundingBox) { + private static Citation bestFuzzyWindow(String needle, Rendered sec) { + String haystack = sec.text(); if (haystack.isEmpty() || needle.isEmpty()) { return null; } @@ -135,8 +136,16 @@ private static Citation bestFuzzyWindow( if (bestQuote == null || bestQuote.isBlank()) { return null; } - double clamped = Math.max(0.0, Math.min(1.0, bestScore)); - return new Citation(loc, bestQuote, clamped, boundingBox); + return citation(sec, bestQuote, Math.max(0.0, Math.min(1.0, bestScore))); + } + + private static Citation citation(Rendered sec, String exactQuote, double matchScore) { + return new Citation( + sec.location(), + exactQuote, + matchScore, + sec.boundingBox(), + Optional.of(new CitationSource(sec.sourceDocId(), sec.sourceUnitId()))); } private static List candidatePositions(String needle, String haystack) { @@ -166,8 +175,9 @@ private static List candidatePositions(String needle, String haystack) private static List renderedSections(ParsedDocument doc) { var out = new ArrayList(doc.sections().size()); - for (var s : doc.sections()) { - out.add(new Rendered(textOf(s), locationOf(s), boundingBoxOf(s))); + for (int i = 0; i < doc.sections().size(); i++) { + var s = doc.sections().get(i); + out.add(new Rendered(textOf(s), locationOf(s), boundingBoxOf(s), doc.docId(), "u" + (i + 1))); } return out; } @@ -205,10 +215,6 @@ private static Optional boundingBoxOf(ParsedSection s) { }; } - private static SourceLocation fallbackLocation(ParsedDocument doc) { - return new SourceLocation(1, 1, 1, 1, 0); - } - private static void traverse(String path, Object node, List out) { if (node == null) { return; @@ -270,11 +276,7 @@ private static void traverseRecord(String path, Object record, List out) { Object v; try { var accessor = c.getAccessor(); - // Records may be declared in a non-exported package or with non-public - // enclosing classes (common in tests). The component's accessor method - // is conceptually always public, but reflective invocation enforces the - // enclosing class's visibility. Records are not JDK built-ins, so - // setAccessible is permitted per CONTRIBUTING.md §1. + // Handles test records with package-private enclosing classes. accessor.setAccessible(true); v = accessor.invoke(record); } catch (ReflectiveOperationException e) { @@ -291,5 +293,6 @@ private static String joinPath(String parent, String child) { private record Leaf(String path, String value) {} - private record Rendered(String text, SourceLocation location, Optional boundingBox) {} + private record Rendered(String text, SourceLocation location, Optional boundingBox, String sourceDocId, + String sourceUnitId) {} } diff --git a/src/test/java/ai/doctruth/CitationSourceTest.java b/src/test/java/ai/doctruth/CitationSourceTest.java new file mode 100644 index 00000000..3e574e6f --- /dev/null +++ b/src/test/java/ai/doctruth/CitationSourceTest.java @@ -0,0 +1,45 @@ +package ai.doctruth; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.junit.jupiter.api.Test; + +class CitationSourceTest { + + @Test + void acceptsTrustDocumentSourceIdentity() { + var source = new CitationSource("sha256:abc", "u1"); + + assertThat(source.docId()).isEqualTo("sha256:abc"); + assertThat(source.unitId()).isEqualTo("u1"); + } + + @Test + void rejectsNullDocId() { + assertThatThrownBy(() -> new CitationSource(null, "u1")) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("docId"); + } + + @Test + void rejectsNullUnitId() { + assertThatThrownBy(() -> new CitationSource("sha256:abc", null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("unitId"); + } + + @Test + void rejectsBlankDocId() { + assertThatThrownBy(() -> new CitationSource(" ", "u1")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("docId"); + } + + @Test + void rejectsBlankUnitId() { + assertThatThrownBy(() -> new CitationSource("sha256:abc", " ")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("unitId"); + } +} diff --git a/src/test/java/ai/doctruth/EvidenceFirstJsonTest.java b/src/test/java/ai/doctruth/EvidenceFirstJsonTest.java new file mode 100644 index 00000000..e5fecbb1 --- /dev/null +++ b/src/test/java/ai/doctruth/EvidenceFirstJsonTest.java @@ -0,0 +1,108 @@ +package ai.doctruth; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +class EvidenceFirstJsonTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Test + void responseSchemaWrapsObjectLeavesAndArrayItems() throws Exception { + var schema = MAPPER.readTree( + """ + { + "type": "object", + "properties": { + "party": { "type": "string" }, + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "amount": { "type": "number" } + } + } + } + }, + "additionalProperties": false + } + """); + + var wrapped = EvidenceFirstJson.responseSchema(schema); + + assertThat(wrapped.path("properties").path("party").path("properties").has("value")).isTrue(); + assertThat(wrapped.path("properties").path("party").path("properties").has("exactQuote")).isTrue(); + assertThat(wrapped.path("properties") + .path("items") + .path("items") + .path("properties") + .path("amount") + .path("properties") + .path("value") + .path("type") + .asText()) + .isEqualTo("number"); + } + + @Test + void responseSchemaWrapsNullMissingAndPrimitiveSchemasAsLeaves() throws Exception { + var nullWrapped = EvidenceFirstJson.responseSchema(null); + var missingWrapped = EvidenceFirstJson.responseSchema(MAPPER.missingNode()); + var primitiveWrapped = EvidenceFirstJson.responseSchema(MAPPER.readTree("{\"type\":\"boolean\"}")); + + assertThat(nullWrapped.path("properties").path("value").isObject()).isTrue(); + assertThat(missingWrapped.path("properties").path("value").isObject()).isTrue(); + assertThat(primitiveWrapped.path("properties").path("value").path("type").asText()).isEqualTo("boolean"); + } + + @Test + void unwrapConvertsEvidenceLeavesInsideObjectsAndArrays() throws Exception { + var raw = MAPPER.readTree( + """ + { + "party": { "value": "Acme", "exactQuote": "Party: Acme" }, + "lines": [ + { "amount": { "value": 42, "exactQuote": "Amount: 42" } }, + null + ] + } + """); + + var unwrapped = EvidenceFirstJson.unwrap(raw); + + assertThat(unwrapped.path("party").asText()).isEqualTo("Acme"); + assertThat(unwrapped.path("lines").get(0).path("amount").asInt()).isEqualTo(42); + assertThat(unwrapped.path("lines").get(1).isNull()).isTrue(); + } + + @Test + void unwrapNullReturnsJsonNullAndNonEvidenceScalarIsCopied() throws Exception { + assertThat(EvidenceFirstJson.unwrap(null).isNull()).isTrue(); + assertThat(EvidenceFirstJson.unwrap(MAPPER.readTree("\"plain\"")).asText()).isEqualTo("plain"); + } + + @Test + void quoteMapCollectsNestedQuotesAndSkipsBlankQuotes() throws Exception { + var raw = MAPPER.readTree( + """ + { + "party": { "value": "Acme", "exactQuote": "Party: Acme" }, + "lines": [ + { "amount": { "value": 42, "exactQuote": "Amount: 42" } }, + { "memo": { "value": "ignored", "exactQuote": " " } } + ], + "noQuote": { "value": "not a leaf" } + } + """); + + assertThat(EvidenceFirstJson.quoteMap(raw)) + .containsEntry("party", "Party: Acme") + .containsEntry("lines[0].amount", "Amount: 42") + .doesNotContainKey("lines[1].memo") + .doesNotContainKey("noQuote"); + assertThat(EvidenceFirstJson.quoteMap(null)).isEmpty(); + } +} diff --git a/src/test/java/ai/doctruth/JsonSchemaExtractionContractTest.java b/src/test/java/ai/doctruth/JsonSchemaExtractionContractTest.java index 43da0cd1..8e4e6beb 100644 --- a/src/test/java/ai/doctruth/JsonSchemaExtractionContractTest.java +++ b/src/test/java/ai/doctruth/JsonSchemaExtractionContractTest.java @@ -332,6 +332,50 @@ public ProviderResponse complete(ProviderRequest req) { assertThat(result.provenance().sourcePublishedAt()).contains(publishedAt); } + @Test + @DisplayName("evidence-first JSON wraps provider schema, unwraps values, and cites exact quotes") + void evidenceFirstJsonUsesProviderQuotesAsCitations() throws ExtractionException { + var captured = new AtomicReference(); + LlmProvider fake = new AnthropicProvider("test-key") { + @Override + public ProviderResponse complete(ProviderRequest req) { + captured.set(req.responseSchema()); + return response(""" + { + "partyA": { + "value": "Acme", + "exactQuote": "Party A: Acme" + }, + "status": { + "value": "signed", + "exactQuote": "status signed" + }, + "totalValue": { + "value": 1000, + "exactQuote": "value 1000" + } + } + """); + } + }; + + var result = DocTruth.from(fake) + .extractJson("extract contract", JsonSchema.from(CONTRACT_SCHEMA)) + .withEvidenceFirst() + .withProvenance() + .withConfidence() + .runJson(doc("Party A: Acme status signed value 1000")); + + assertThat(captured.get().path("properties").path("partyA").path("properties").has("value")) + .isTrue(); + assertThat(captured.get().path("properties").path("partyA").path("properties").has("exactQuote")) + .isTrue(); + assertThat(result.value().path("partyA").asText()).isEqualTo("Acme"); + assertThat(result.value().path("totalValue").asInt()).isEqualTo(1000); + assertThat(result.citations().get("partyA").exactQuote()).isEqualTo("Party A: Acme"); + assertThat(result.confidence().get("partyA").rationale()).contains("evidence quote"); + } + @Test @DisplayName("builder invariants reject invalid retry and citation arguments") void builderInvariants() { diff --git a/src/test/java/ai/doctruth/OpenDataLoaderPdfDocumentParserTest.java b/src/test/java/ai/doctruth/OpenDataLoaderPdfDocumentParserTest.java index fd5c10ed..0bfb6092 100644 --- a/src/test/java/ai/doctruth/OpenDataLoaderPdfDocumentParserTest.java +++ b/src/test/java/ai/doctruth/OpenDataLoaderPdfDocumentParserTest.java @@ -2,8 +2,13 @@ import static org.assertj.core.api.Assertions.assertThat; +import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDPage; @@ -54,6 +59,146 @@ void defaultPdfBackendIsOpenDataLoader() throws Exception { assertThat(defaultDoc.sections()).isEqualTo(explicitDoc.sections()); } + @Test + void streamsOpenDataLoaderJsonIntoParsedSections() throws Exception { + Path json = tempDir.resolve("sample.json"); + Files.writeString(json, """ + { + "number of pages": 2, + "producer metadata": { "ignored": [1, 2, 3] }, + "kids": [ + { + "type": "heading", + "page number": 1, + "bounding box": [20, 100, 100, 150], + "content": "Streaming Heading" + }, + { + "type": "table", + "page number": 2, + "rows": [ + { "cells": [{ "content": "A1" }, { "content": "B1" }] } + ] + } + ] + } + """); + var geometry = new OpenDataLoaderPdfGeometry( + Map.of(1, new OpenDataLoaderPdfGeometry.PageGeometry(200.0, 400.0))); + + var parsed = OpenDataLoaderPdfDocumentParser.readOpenDataLoaderJson(json, geometry); + + assertThat(parsed.pageCount()).isEqualTo(2); + assertThat(parsed.sections()).hasSize(2); + assertThat((TextSection) parsed.sections().get(0)) + .extracting(TextSection::text, TextSection::kind) + .containsExactly("Streaming Heading", BlockKind.HEADING); + assertThat((TableSection) parsed.sections().get(1)) + .extracting(TableSection::rows) + .isEqualTo(List.of(List.of("A1", "B1"))); + } + + @Test + void rejectsOpenDataLoaderJsonThatDoesNotStartWithObject() throws Exception { + Path json = tempDir.resolve("array-root.json"); + Files.writeString(json, "[]"); + var geometry = new OpenDataLoaderPdfGeometry(Map.of()); + + org.assertj.core.api.Assertions.assertThatThrownBy( + () -> OpenDataLoaderPdfDocumentParser.readOpenDataLoaderJson(json, geometry)) + .isInstanceOf(java.io.IOException.class) + .hasMessageContaining("OpenDataLoader JSON root must be an object"); + } + + @Test + void rejectsOpenDataLoaderJsonWithNonObjectKids() throws Exception { + Path json = tempDir.resolve("non-object-kid.json"); + Files.writeString(json, """ + { + "number of pages": 1, + "kids": [null] + } + """); + var geometry = new OpenDataLoaderPdfGeometry(Map.of()); + + org.assertj.core.api.Assertions.assertThatThrownBy( + () -> OpenDataLoaderPdfDocumentParser.readOpenDataLoaderJson(json, geometry)) + .isInstanceOf(java.io.IOException.class) + .hasMessageContaining("OpenDataLoader kids entries must be objects"); + } + + @Test + void rejectsOpenDataLoaderJsonWithNonArrayKidsField() throws Exception { + Path json = tempDir.resolve("non-array-kids.json"); + Files.writeString(json, """ + { + "number of pages": 1, + "kids": {} + } + """); + var geometry = new OpenDataLoaderPdfGeometry(Map.of()); + + org.assertj.core.api.Assertions.assertThatThrownBy( + () -> OpenDataLoaderPdfDocumentParser.readOpenDataLoaderJson(json, geometry)) + .isInstanceOf(java.io.IOException.class) + .hasMessageContaining("OpenDataLoader kids field must be an array"); + } + + @Test + void serializesOpenDataLoaderProcessLifecycle() throws Exception { + Path first = tempDir.resolve("first.pdf"); + Path second = tempDir.resolve("second.pdf"); + Files.writeString(first, "fake pdf bytes"); + Files.writeString(second, "fake pdf bytes"); + var activeRuns = new AtomicInteger(); + var maxActiveRuns = new AtomicInteger(); + var calls = new AtomicInteger(); + OpenDataLoaderPdfDocumentParser.OpenDataLoaderRunner runner = (pdfPath, outputDir, config) -> { + calls.incrementAndGet(); + int active = activeRuns.incrementAndGet(); + maxActiveRuns.accumulateAndGet(active, Math::max); + try { + sleepBriefly(); + String outputName = pdfPath.getFileName().toString().replaceFirst("\\.[^.]+$", "") + ".json"; + Files.writeString(outputDir.resolve(outputName), """ + { + "number of pages": 1, + "kids": [ + { "type": "heading", "page number": 1, "content": "serialized" } + ] + } + """); + } finally { + activeRuns.decrementAndGet(); + } + }; + + try (var runnerOverride = OpenDataLoaderPdfDocumentParser.useOpenDataLoaderRunnerForTesting(runner)) { + assertThat(runnerOverride).isNotNull(); + var executor = Executors.newFixedThreadPool(2); + try { + var firstResult = executor.submit(() -> OpenDataLoaderPdfDocumentParser.parse(first)); + var secondResult = executor.submit(() -> OpenDataLoaderPdfDocumentParser.parse(second)); + assertThat(firstResult.get(5, TimeUnit.SECONDS).sections()).hasSize(1); + assertThat(secondResult.get(5, TimeUnit.SECONDS).sections()).hasSize(1); + } finally { + executor.shutdownNow(); + } + } + + assertThat(calls).hasValue(2); + assertThat(maxActiveRuns).hasValue(1); + } + + private static void sleepBriefly() throws java.io.IOException { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new java.io.IOException("interrupted while simulating OpenDataLoader", e); + } + } + private static Path writeMultiPagePdf(Path dir, List pageTexts) throws Exception { Path path = dir.resolve("sample.pdf"); try (PDDocument document = new PDDocument()) { diff --git a/src/test/java/ai/doctruth/OpenDataLoaderSectionMapperTest.java b/src/test/java/ai/doctruth/OpenDataLoaderSectionMapperTest.java index 17a42220..347bebee 100644 --- a/src/test/java/ai/doctruth/OpenDataLoaderSectionMapperTest.java +++ b/src/test/java/ai/doctruth/OpenDataLoaderSectionMapperTest.java @@ -3,6 +3,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; @@ -111,4 +112,86 @@ void ignoresMissingArraysBlankTextAndInvalidBoundingBoxes() throws Exception { .map(TextSection.class::cast) .allSatisfy(section -> assertThat(section.boundingBox()).isEmpty()); } + + @Test + void flattensNestedTextWithoutBlankSegments() throws Exception { + var geometry = + new OpenDataLoaderPdfGeometry(Map.of(1, new OpenDataLoaderPdfGeometry.PageGeometry(200.0, 400.0))); + var kids = MAPPER.readTree(""" + [ + { + "type": "paragraph", + "page number": 1, + "kids": [ + { "content": "Alpha" }, + { "content": " " }, + { + "list items": [ + { "content": "Beta" }, + { + "kids": [ + { "content": "Gamma" } + ] + } + ] + } + ] + }, + { + "type": "table", + "page number": 1, + "rows": [ + { + "cells": [ + { + "kids": [ + { "content": "Cell A" }, + { "content": "Cell B" } + ] + } + ] + } + ] + } + ] + """); + + var sections = new OpenDataLoaderSectionMapper(geometry).map(kids); + + assertThat((TextSection) sections.get(0)).extracting(TextSection::text).isEqualTo("Alpha\nBeta\nGamma"); + assertThat((TableSection) sections.get(1)) + .extracting(TableSection::rows) + .isEqualTo(java.util.List.of(java.util.List.of("Cell A\nCell B"))); + } + + @Test + void loadsGeometryOnlyWhenBoundingBoxesArePresent() throws Exception { + var loads = new AtomicInteger(); + var geometry = OpenDataLoaderPdfGeometry.lazy(() -> { + loads.incrementAndGet(); + return new OpenDataLoaderPdfGeometry( + Map.of(1, new OpenDataLoaderPdfGeometry.PageGeometry(200.0, 400.0))); + }); + + new OpenDataLoaderSectionMapper(geometry).map(MAPPER.readTree(""" + [ + { "type": "paragraph", "page number": 1, "content": "No box" } + ] + """)); + + assertThat(loads).hasValue(0); + + new OpenDataLoaderSectionMapper(geometry).map(MAPPER.readTree(""" + [ + { + "type": "paragraph", + "page number": 1, + "content": "Has box", + "bounding box": [20, 100, 100, 150] + } + ] + """)); + + assertThat(loads).hasValue(1); + } } diff --git a/src/test/java/ai/doctruth/TrustDocumentParserTest.java b/src/test/java/ai/doctruth/TrustDocumentParserTest.java new file mode 100644 index 00000000..c0639c0e --- /dev/null +++ b/src/test/java/ai/doctruth/TrustDocumentParserTest.java @@ -0,0 +1,176 @@ +package ai.doctruth; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.lang.reflect.InvocationTargetException; +import java.io.StringWriter; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.font.PDType1Font; +import org.apache.pdfbox.pdmodel.font.Standard14Fonts; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TrustDocumentParserTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @TempDir + Path tempDir; + + @Test + void parsesPdfIntoTrustDocumentWithOpenDataLoaderByDefault() throws Exception { + Path pdf = writePdf("TrustDocument default path"); + + TrustDocument doc = TrustDocumentParser.parse(pdf); + + assertThat(doc.source().filename()).isEqualTo(pdf.getFileName().toString()); + assertThat(doc.source().sha256()).hasSize(64); + assertThat(doc.parserRun().backend()).isEqualTo("opendataloader"); + assertThat(doc.units()).isNotEmpty(); + assertThat(doc.units()).anySatisfy(unit -> assertThat(unit.text()).contains("TrustDocument default path")); + } + + @Test + void jsonCarriesAuditReadySourceParserLocationAndBboxFields() throws Exception { + Path pdf = writePdf("Audit-ready TrustDocument JSON"); + TrustDocument doc = TrustDocumentParser.parse(pdf); + + var tree = MAPPER.readTree(TrustDocumentJson.toJson(doc)); + + assertThat(tree.path("schemaVersion").asText()).isEqualTo("doctruth.trust-document.v1"); + assertThat(tree.path("source").path("sha256").asText()).hasSize(64); + assertThat(tree.path("parserRun").path("backend").asText()).isEqualTo("opendataloader"); + var unit = tree.path("units").get(0); + assertThat(unit.path("location").path("pageStart").asInt()).isEqualTo(1); + assertThat(unit.has("boundingBox")).isTrue(); + } + + @Test + void fromParsedMapsTextTableAndFigureUnits() { + var location = new SourceLocation(1, 1, 1, 1, 0); + var metadata = new DocumentMetadata("contract.pdf", 1, Optional.empty()); + var parsed = new ParsedDocument( + "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + List.of( + new TextSection("Intro", location), + new TableSection(List.of(List.of("A", "B")), location), + new FigureSection("Chart", location)), + metadata); + + TrustDocument doc = TrustDocument.fromParsed(parsed, Path.of("contract.pdf"), PdfParserBackend.OPENDATALOADER); + var json = TrustDocumentJson.toJson(doc); + + assertThat(doc.units()).extracting(TrustUnit::type).containsExactly("text", "table", "figure"); + assertThat(json).contains("\"rows\"", "\"text\" : \"Chart\""); + } + + @Test + void jsonCanStreamDirectlyFromParsedDocument() throws Exception { + var location = new SourceLocation(1, 1, 1, 1, 0); + var metadata = new DocumentMetadata("contract.pdf", 1, Optional.empty()); + var parsed = new ParsedDocument( + "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + List.of( + new TextSection( + "Intro", + location, + BlockKind.BODY, + Optional.of(new BoundingBox(10, 20, 30, 40))), + new TableSection(List.of(List.of("A", "B")), location), + new FigureSection("Chart", location)), + metadata); + + String materialized = TrustDocumentJson.toJson( + TrustDocument.fromParsed(parsed, Path.of("contract.pdf"), PdfParserBackend.OPENDATALOADER)); + String streamed = TrustDocumentJson.toJson(parsed, Path.of("contract.pdf"), PdfParserBackend.OPENDATALOADER); + + assertThat(MAPPER.readTree(streamed)).isEqualTo(MAPPER.readTree(materialized)); + } + + @Test + void fromParsedUsesExplicitSourcePathForSourceFilename() throws Exception { + var location = new SourceLocation(1, 1, 1, 1, 0); + var metadata = new DocumentMetadata("internal-temp-name.pdf", 1, Optional.empty()); + var parsed = new ParsedDocument( + "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + List.of(new TextSection("Intro", location)), + metadata); + + TrustDocument doc = + TrustDocument.fromParsed(parsed, Path.of("/documents/customer-contract.pdf"), PdfParserBackend.OPENDATALOADER); + String streamed = TrustDocumentJson.toJson( + parsed, Path.of("/documents/customer-contract.pdf"), PdfParserBackend.OPENDATALOADER); + + assertThat(doc.source().filename()).isEqualTo("customer-contract.pdf"); + assertThat(MAPPER.readTree(streamed).path("source").path("filename").asText()) + .isEqualTo("customer-contract.pdf"); + } + + @Test + void writerOverloadsMatchStringJson() throws Exception { + var location = new SourceLocation(1, 1, 1, 1, 0); + var metadata = new DocumentMetadata("contract.pdf", 1, Optional.empty()); + var parsed = new ParsedDocument( + "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + List.of(new TextSection("Intro", location)), + metadata); + var doc = TrustDocument.fromParsed(parsed, Path.of("contract.pdf"), PdfParserBackend.OPENDATALOADER); + var materializedDoc = TrustDocumentJson.toJson(doc); + var materializedParsed = TrustDocumentJson.toJson(parsed, Path.of("contract.pdf"), PdfParserBackend.OPENDATALOADER); + var docWriter = new StringWriter(); + var parsedWriter = new StringWriter(); + + TrustDocumentJson.writeJson(doc, docWriter); + TrustDocumentJson.writeJson(parsed, Path.of("contract.pdf"), PdfParserBackend.OPENDATALOADER, parsedWriter); + + assertThat(MAPPER.readTree(docWriter.toString())).isEqualTo(MAPPER.readTree(materializedDoc)); + assertThat(MAPPER.readTree(parsedWriter.toString())).isEqualTo(MAPPER.readTree(materializedParsed)); + } + + @Test + void sourceValidationRejectsInvalidIdentity() { + assertThatThrownBy(() -> new TrustDocumentSource("", "0".repeat(64), 1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("filename"); + assertThatThrownBy(() -> new TrustDocumentSource("x.pdf", "not-sha", 1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("sha256"); + assertThatThrownBy(() -> new TrustDocumentSource("x.pdf", "0".repeat(64), 0)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("pageCount"); + } + + @Test + void parserCannotBeInstantiated() throws Exception { + var constructor = TrustDocumentParser.class.getDeclaredConstructor(); + constructor.setAccessible(true); + + assertThatThrownBy(constructor::newInstance) + .isInstanceOf(InvocationTargetException.class) + .hasCauseInstanceOf(AssertionError.class); + } + + private Path writePdf(String text) throws Exception { + Path path = tempDir.resolve("sample.pdf"); + try (PDDocument document = new PDDocument()) { + document.addPage(new PDPage()); + try (PDPageContentStream content = new PDPageContentStream(document, document.getPage(0))) { + content.beginText(); + content.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12); + content.newLineAtOffset(72, 720); + content.showText(text); + content.endText(); + } + document.save(path.toFile()); + } + return path; + } +} diff --git a/src/test/java/ai/doctruth/cli/CliSupportTest.java b/src/test/java/ai/doctruth/cli/CliSupportTest.java index 722f2e0a..99d94151 100644 --- a/src/test/java/ai/doctruth/cli/CliSupportTest.java +++ b/src/test/java/ai/doctruth/cli/CliSupportTest.java @@ -50,6 +50,23 @@ void parsedDocumentJsonHandlesAllSectionTypes() throws Exception { assertThat(tree.path("sections").get(2).path("type").asText()).isEqualTo("figure"); } + @Test + void parsedDocumentJsonFileOutputMatchesStringOutput() throws Exception { + var loc = new SourceLocation(1, 1, 1, 1, 0); + var doc = new ParsedDocument( + "doc", + java.util.List.of( + new TextSection("hello", loc, BlockKind.BODY), + new TableSection(java.util.List.of(java.util.List.of("a", "b")), loc), + new FigureSection("chart", loc)), + new DocumentMetadata("sample.pdf", 1, Optional.empty())); + Path output = tempDir.resolve("parsed.json"); + + ParsedDocumentJson.writeJson(doc, output); + + assertThat(MAPPER.readTree(output.toFile())).isEqualTo(MAPPER.readTree(ParsedDocumentJson.toJson(doc))); + } + @Test void providerFactoryCreatesSupportedProviders() throws Exception { var env = Map.of( diff --git a/src/test/java/ai/doctruth/cli/DocTruthCliExtractContractTest.java b/src/test/java/ai/doctruth/cli/DocTruthCliExtractContractTest.java new file mode 100644 index 00000000..1cfe3db9 --- /dev/null +++ b/src/test/java/ai/doctruth/cli/DocTruthCliExtractContractTest.java @@ -0,0 +1,206 @@ +package ai.doctruth.cli; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +import ai.doctruth.LlmProvider; +import ai.doctruth.OpenAiProvider; +import ai.doctruth.ProviderRequest; +import ai.doctruth.ProviderResponse; +import ai.doctruth.ProviderUsage; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.font.PDType1Font; +import org.apache.pdfbox.pdmodel.font.Standard14Fonts; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class DocTruthCliExtractContractTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @TempDir + Path tempDir; + + @Test + void extractWritesTrustDocumentLinkedRunManifest() throws Exception { + Path pdf = samplePdf(); + Path schema = schemaFile(); + Path out = tempDir.resolve("run"); + var cli = cliWithProvider(cannedProvider()); + + int code = cli.run(new String[] {"extract", pdf.toString(), "-s", schema.toString(), "-o", out.toString()}); + + assertThat(code).isZero(); + var trust = MAPPER.readTree(Files.readString(out.resolve("trust-document.json"))); + var audit = MAPPER.readTree(Files.readString(out.resolve("audit.json"))); + var firstDerivation = audit.path("prov:wasDerivedFrom").get(0); + assertThat(firstDerivation.path("doctruth:sourceDocId").asText()).isEqualTo(trust.path("docId").asText()); + assertThat(firstDerivation.path("doctruth:sourceUnitId").asText()).isNotBlank(); + assertThat(trust.path("units")) + .anySatisfy(unit -> assertThat(unit.path("id").asText()) + .isEqualTo(firstDerivation.path("doctruth:sourceUnitId").asText())); + var manifest = MAPPER.readTree(Files.readString(out.resolve("manifest.json"))); + assertThat(manifest.path("schemaVersion").asText()).isEqualTo("doctruth.extract-run.v1"); + assertThat(manifest.path("artifacts").path("trustDocument").asText()).isEqualTo("trust-document.json"); + } + + @Test + void extractRequiresNestedSchemaLeafCitationsByDefault() throws Exception { + Path pdf = samplePdf(); + Path schema = nestedSchemaFile(); + Path out = tempDir.resolve("nested-run"); + var cli = cliWithProvider(providerReturning("{\"party\":{\"name\":\"Acme Industrial Materials Pty Ltd\"}}")); + + int code = cli.run(new String[] {"extract", pdf.toString(), "-s", schema.toString(), "-o", out.toString()}); + + assertThat(code).isZero(); + var audit = MAPPER.readTree(Files.readString(out.resolve("audit.json"))); + assertThat(audit.path("prov:wasDerivedFrom")) + .anySatisfy(entry -> assertThat(entry.path("doctruth:fieldPath").asText()).isEqualTo("party.name")); + } + + @Test + void extractEvidenceFirstUnwrapsValuesAndCitesExactQuotes() throws Exception { + Path pdf = samplePdf(); + Path schema = schemaFile(); + Path out = tempDir.resolve("evidence-first-run"); + var cli = cliWithProvider(evidenceFirstProvider()); + + int code = cli.run(new String[] { + "extract", pdf.toString(), "-s", schema.toString(), "-o", out.toString(), "--evidence-first" + }); + + assertThat(code).isZero(); + var result = MAPPER.readTree(Files.readString(out.resolve("result.json"))); + assertThat(result.path("partyA").asText()).isEqualTo("Acme Industrial Materials Pty Ltd"); + var audit = MAPPER.readTree(Files.readString(out.resolve("audit.json"))); + assertThat(audit.path("prov:wasDerivedFrom")) + .anySatisfy(entry -> assertThat(entry.path("prov:value").asText()) + .isEqualTo("Party A: Acme Industrial Materials Pty Ltd")); + } + + private TestCli cliWithProvider(LlmProvider provider) { + var out = new ByteArrayOutputStream(); + var err = new ByteArrayOutputStream(); + var cli = new DocTruthCli( + Map.of("OPENAI_API_KEY", "test"), + new PrintStream(out, true, StandardCharsets.UTF_8), + new PrintStream(err, true, StandardCharsets.UTF_8), + spec -> "{}", + opts -> provider); + return new TestCli(cli, out); + } + + private static LlmProvider cannedProvider() { + return providerReturning("{\"partyA\":\"Acme Industrial Materials Pty Ltd\",\"totalValue\":\"AUD 2,450,000\"}"); + } + + private static LlmProvider providerReturning(String json) { + return new OpenAiProvider("test", URI.create("http://localhost"), "test-model") { + @Override + public ProviderResponse complete(ProviderRequest request) { + return new ProviderResponse(json, new ProviderUsage(1, 1, "test-model")); + } + }; + } + + private static LlmProvider evidenceFirstProvider() { + return new OpenAiProvider("test", URI.create("http://localhost"), "test-model") { + @Override + public ProviderResponse complete(ProviderRequest request) { + assertThat(request.responseSchema().path("properties").path("partyA").path("properties").has("value")) + .isTrue(); + assertThat(request.responseSchema().path("properties").path("partyA").path("properties").has("exactQuote")) + .isTrue(); + return new ProviderResponse( + """ + { + "partyA": { + "value": "Acme Industrial Materials Pty Ltd", + "exactQuote": "Party A: Acme Industrial Materials Pty Ltd" + }, + "totalValue": { + "value": "AUD 2,450,000", + "exactQuote": "Total Value: AUD 2,450,000" + } + } + """, + new ProviderUsage(1, 1, "test-model")); + } + }; + } + + private Path schemaFile() throws IOException { + Path schema = tempDir.resolve("contract.schema.json"); + Files.writeString(schema, """ + { + "type": "object", + "properties": { + "partyA": { "type": "string" }, + "totalValue": { "type": "string" } + }, + "required": ["partyA", "totalValue"], + "additionalProperties": false + } + """); + return schema; + } + + private Path nestedSchemaFile() throws IOException { + Path schema = tempDir.resolve("nested.schema.json"); + Files.writeString(schema, """ + { + "type": "object", + "properties": { + "party": { + "type": "object", + "properties": { + "name": { "type": "string" } + }, + "required": ["name"] + } + }, + "required": ["party"], + "additionalProperties": false + } + """); + return schema; + } + + private Path samplePdf() throws IOException { + Path path = tempDir.resolve("contract.pdf"); + try (var pdf = new PDDocument()) { + var page = new PDPage(); + pdf.addPage(page); + try (var cs = new PDPageContentStream(pdf, page)) { + cs.beginText(); + cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12); + cs.newLineAtOffset(50, 720); + cs.showText("Party A: Acme Industrial Materials Pty Ltd"); + cs.newLineAtOffset(0, -18); + cs.showText("Total Value: AUD 2,450,000"); + cs.endText(); + } + pdf.save(path.toFile()); + } + return path; + } + + private record TestCli(DocTruthCli delegate, ByteArrayOutputStream outBytes) { + int run(String[] args) { + return delegate.run(args); + } + } +} diff --git a/src/test/java/ai/doctruth/cli/DocTruthCliTrustDocumentTest.java b/src/test/java/ai/doctruth/cli/DocTruthCliTrustDocumentTest.java new file mode 100644 index 00000000..923f4331 --- /dev/null +++ b/src/test/java/ai/doctruth/cli/DocTruthCliTrustDocumentTest.java @@ -0,0 +1,319 @@ +package ai.doctruth.cli; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +import ai.doctruth.OpenAiProvider; +import ai.doctruth.ProviderRequest; +import ai.doctruth.ProviderResponse; +import ai.doctruth.ProviderUsage; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.font.PDType1Font; +import org.apache.pdfbox.pdmodel.font.Standard14Fonts; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class DocTruthCliTrustDocumentTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @TempDir + Path tempDir; + + @Test + void parseFormatJsonWritesTrustDocument() throws Exception { + Path pdf = samplePdf(); + Path out = tempDir.resolve("trust-document.json"); + var cli = cli(); + + int code = cli.run(new String[] {"parse", pdf.toString(), "--format", "json", "-o", out.toString()}); + + assertThat(code).isZero(); + var tree = MAPPER.readTree(Files.readString(out)); + assertThat(tree.path("schemaVersion").asText()).isEqualTo("doctruth.trust-document.v1"); + assertThat(tree.path("source").path("filename").asText()).isEqualTo(pdf.getFileName().toString()); + assertThat(tree.path("source").path("sha256").asText()).hasSize(64); + assertThat(tree.path("parserRun").path("backend").asText()).isEqualTo("opendataloader"); + assertThat(tree.path("units")).isNotEmpty(); + assertThat(tree.path("units").get(0).path("location").path("pageStart").asInt()).isEqualTo(1); + } + + @Test + void parseFormatJsonPrintsTrustDocument() throws Exception { + Path pdf = samplePdf(); + var cli = cli(); + + int code = cli.run(new String[] {"parse", pdf.toString(), "--format", "json"}); + + assertThat(code).isZero(); + var tree = MAPPER.readTree(cli.out()); + assertThat(tree.path("schemaVersion").asText()).isEqualTo("doctruth.trust-document.v1"); + assertThat(tree.path("parserRun").path("backend").asText()).isEqualTo("opendataloader"); + } + + @Test + void parseFormatParsedJsonPrintsCompatibilityJson() throws Exception { + Path pdf = samplePdf(); + var cli = cli(); + + int code = cli.run(new String[] {"parse", pdf.toString(), "--format", "parsed-json", "--json"}); + + assertThat(code).isZero(); + var tree = MAPPER.readTree(cli.out()); + assertThat(tree.path("metadata").path("sourceFilename").asText()).isEqualTo(pdf.getFileName().toString()); + assertThat(tree.path("sections")).isNotEmpty(); + } + + @Test + void profileJsonReportsParserLatencyAndMemory() throws Exception { + Path pdf = samplePdf(); + var cli = cli(); + + int code = cli.run(new String[] {"profile", pdf.toString(), "--iterations", "1", "--json"}); + + assertThat(code).isZero(); + var tree = MAPPER.readTree(cli.out()); + assertThat(tree.path("parser").asText()).isEqualTo("opendataloader"); + assertThat(tree.path("iterations").asInt()).isEqualTo(1); + assertThat(tree.path("coldLatencyMillis").asLong()).isGreaterThanOrEqualTo(0); + assertThat(tree.path("fileSizeBytes").asLong()).isEqualTo(Files.size(pdf)); + assertThat(tree.path("sectionCount").asInt()).isGreaterThan(0); + assertThat(tree.path("includeOutput").asText()).isEqualTo("parser-only"); + assertThat(tree.path("parseLatencyMillis")).hasSize(1); + assertThat(tree.path("heapUsedBeforeBytes").asLong()).isGreaterThanOrEqualTo(0); + assertThat(tree.path("heapUsedAfterBytes").asLong()).isGreaterThanOrEqualTo(0); + } + + @Test + void profileCanIncludeTrustJsonOutputCost() throws Exception { + Path pdf = samplePdf(); + var cli = cli(); + + int code = cli.run(new String[] { + "profile", pdf.toString(), "--iterations", "1", "--include-output", "trust-json", "--json" + }); + + assertThat(code).isZero(); + var tree = MAPPER.readTree(cli.out()); + assertThat(tree.path("includeOutput").asText()).isEqualTo("trust-json"); + assertThat(tree.path("outputLatencyMillis")).hasSize(1); + assertThat(tree.path("coldOutputLatencyMillis").asLong()).isGreaterThanOrEqualTo(0); + } + + @Test + void profileCanIncludeParsedJsonOutputCost() throws Exception { + Path pdf = samplePdf(); + var cli = cli(); + + int code = cli.run(new String[] { + "profile", pdf.toString(), "--iterations", "1", "--include-output", "parsed-json", "--json" + }); + + assertThat(code).isZero(); + var tree = MAPPER.readTree(cli.out()); + assertThat(tree.path("includeOutput").asText()).isEqualTo("parsed-json"); + assertThat(tree.path("outputLatencyMillis")).hasSize(1); + } + + @Test + void profileCanIncludeTrustJsonFileOutputCost() throws Exception { + Path pdf = samplePdf(); + var cli = cli(); + + int code = cli.run(new String[] { + "profile", pdf.toString(), "--iterations", "1", "--include-output", "trust-json-file", "--json" + }); + + assertThat(code).isZero(); + var tree = MAPPER.readTree(cli.out()); + assertThat(tree.path("includeOutput").asText()).isEqualTo("trust-json-file"); + assertThat(tree.path("outputLatencyMillis")).hasSize(1); + assertThat(tree.path("profiledOutputBytes").asLong()).isGreaterThan(0); + assertThat(tree.path("profiledOutputChars").asLong()).isZero(); + } + + @Test + void profileCanIncludeParsedJsonFileOutputCost() throws Exception { + Path pdf = samplePdf(); + var cli = cli(); + + int code = cli.run(new String[] { + "profile", pdf.toString(), "--iterations", "1", "--include-output", "parsed-json-file", "--json" + }); + + assertThat(code).isZero(); + var tree = MAPPER.readTree(cli.out()); + assertThat(tree.path("includeOutput").asText()).isEqualTo("parsed-json-file"); + assertThat(tree.path("outputLatencyMillis")).hasSize(1); + assertThat(tree.path("profiledOutputBytes").asLong()).isGreaterThan(0); + assertThat(tree.path("profiledOutputChars").asLong()).isZero(); + } + + @Test + void profilePrintsReadableSummaryWithWarmRun() throws Exception { + Path pdf = samplePdf(); + var cli = cli(); + + int code = cli.run(new String[] {"profile", pdf.toString(), "--iterations", "2"}); + + assertThat(code).isZero(); + assertThat(cli.out()) + .contains("parser: opendataloader") + .contains("iterations: 2") + .contains("file size bytes:") + .contains("sections:") + .contains("warm avg latency ms:"); + } + + @Test + void profilePrintsOutputTimingSummaryWhenIncludingOutput() throws Exception { + Path pdf = samplePdf(); + var cli = cli(); + + int code = cli.run(new String[] { + "profile", pdf.toString(), "--iterations", "1", "--include-output", "trust-json" + }); + + assertThat(code).isZero(); + assertThat(cli.out()) + .contains("include output: trust-json") + .contains("cold output latency ms:") + .contains("warm avg output latency ms:"); + } + + @Test + void profileRejectsUnknownOutputMode() throws Exception { + Path pdf = samplePdf(); + var cli = cli(); + + int code = cli.run(new String[] {"profile", pdf.toString(), "--include-output", "xml"}); + + assertThat(code).isEqualTo(2); + assertThat(cli.err()).contains("unsupported profile output mode"); + } + + @Test + void profileRejectsBadUsage() { + var cli = cli(); + + int code = cli.run(new String[] {"profile"}); + + assertThat(code).isEqualTo(2); + assertThat(cli.err()).contains("usage: doctruth profile"); + } + + @Test + void profileRejectsUnknownOption() throws Exception { + Path pdf = samplePdf(); + var cli = cli(); + + int code = cli.run(new String[] {"profile", pdf.toString(), "--wat"}); + + assertThat(code).isEqualTo(2); + assertThat(cli.err()).contains("unknown profile option"); + } + + @Test + void profileRejectsUnknownParser() throws Exception { + Path pdf = samplePdf(); + var cli = cli(); + + int code = cli.run(new String[] {"profile", pdf.toString(), "--parser", "wat"}); + + assertThat(code).isEqualTo(2); + assertThat(cli.err()).contains("unsupported parser"); + } + + @Test + void profileRejectsInvalidIterations() throws Exception { + Path pdf = samplePdf(); + var cli = cli(); + + int code = cli.run(new String[] {"profile", pdf.toString(), "--iterations", "0"}); + + assertThat(code).isEqualTo(2); + assertThat(cli.err()).contains("--iterations must be >= 1"); + } + + @Test + void profileRejectsNonNumericIterations() throws Exception { + Path pdf = samplePdf(); + var cli = cli(); + + int code = cli.run(new String[] {"profile", pdf.toString(), "--iterations", "many"}); + + assertThat(code).isEqualTo(2); + assertThat(cli.err()).contains("--iterations must be an integer"); + } + + @Test + void profileRejectsMissingOptionValue() throws Exception { + Path pdf = samplePdf(); + var cli = cli(); + + int code = cli.run(new String[] {"profile", pdf.toString(), "--parser"}); + + assertThat(code).isEqualTo(2); + assertThat(cli.err()).contains("--parser requires a value"); + } + + private static TestCli cli() { + var out = new ByteArrayOutputStream(); + var err = new ByteArrayOutputStream(); + var provider = new OpenAiProvider("test", URI.create("http://localhost"), "test-model") { + @Override + public ProviderResponse complete(ProviderRequest request) { + return new ProviderResponse("{}", new ProviderUsage(1, 1, "test-model")); + } + }; + var cli = new DocTruthCli( + Map.of(), + new PrintStream(out, true, StandardCharsets.UTF_8), + new PrintStream(err, true, StandardCharsets.UTF_8), + spec -> "{}", + opts -> provider); + return new TestCli(cli, out, err); + } + + private Path samplePdf() throws Exception { + Path path = tempDir.resolve("contract.pdf"); + try (var pdf = new PDDocument()) { + var page = new PDPage(); + pdf.addPage(page); + try (var cs = new PDPageContentStream(pdf, page)) { + cs.beginText(); + cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12); + cs.newLineAtOffset(50, 720); + cs.showText("Party A: Acme Industrial Materials Pty Ltd"); + cs.endText(); + } + pdf.save(path.toFile()); + } + return path; + } + + private record TestCli(DocTruthCli delegate, ByteArrayOutputStream outBytes, ByteArrayOutputStream errBytes) { + int run(String[] args) { + return delegate.run(args); + } + + String out() { + return outBytes.toString(StandardCharsets.UTF_8); + } + + String err() { + return errBytes.toString(StandardCharsets.UTF_8); + } + } +} diff --git a/src/test/java/ai/doctruth/cli/ParseCommandPerformanceTest.java b/src/test/java/ai/doctruth/cli/ParseCommandPerformanceTest.java new file mode 100644 index 00000000..547a2790 --- /dev/null +++ b/src/test/java/ai/doctruth/cli/ParseCommandPerformanceTest.java @@ -0,0 +1,136 @@ +package ai.doctruth.cli; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import ai.doctruth.DocumentMetadata; +import ai.doctruth.ParsedDocument; +import ai.doctruth.PdfParserBackend; + +import org.junit.jupiter.api.Test; + +class ParseCommandPerformanceTest { + + @Test + void summaryPathDoesNotSerializeJson() throws Exception { + var serializerCalls = new AtomicInteger(); + var out = new ByteArrayOutputStream(); + var context = new CliContext( + Map.of(), + new PrintStream(out, true, StandardCharsets.UTF_8), + new PrintStream(new ByteArrayOutputStream(), true, StandardCharsets.UTF_8), + spec -> "{}", + opts -> { + throw new AssertionError("provider is not used by parse"); + }); + var command = new ParseCommand( + context, (path, backend) -> parsed(path), countingRenderer(serializerCalls, new AtomicInteger())); + + command.run(new String[] {"parse", "contract.pdf"}); + + assertThat(serializerCalls).hasValue(0); + assertThat(out.toString(StandardCharsets.UTF_8)).contains("parser: opendataloader"); + } + + @Test + void jsonOutputSerializesOnce() throws Exception { + var serializerCalls = new AtomicInteger(); + var context = new CliContext( + Map.of(), + new PrintStream(new ByteArrayOutputStream(), true, StandardCharsets.UTF_8), + new PrintStream(new ByteArrayOutputStream(), true, StandardCharsets.UTF_8), + spec -> "{}", + opts -> { + throw new AssertionError("provider is not used by parse"); + }); + var command = new ParseCommand( + context, (path, backend) -> parsed(path), countingRenderer(serializerCalls, new AtomicInteger())); + + command.run(new String[] {"parse", "contract.pdf", "--format", "json"}); + + assertThat(serializerCalls).hasValue(1); + } + + @Test + void jsonFileOutputWritesWithoutStringMaterialization() throws Exception { + var renderCalls = new AtomicInteger(); + var writeCalls = new AtomicInteger(); + Path output = Files.createTempFile("doctruth-trust-document", ".json"); + var context = new CliContext( + Map.of(), + new PrintStream(new ByteArrayOutputStream(), true, StandardCharsets.UTF_8), + new PrintStream(new ByteArrayOutputStream(), true, StandardCharsets.UTF_8), + spec -> "{}", + opts -> { + throw new AssertionError("provider is not used by parse"); + }); + var command = new ParseCommand( + context, + (path, backend) -> parsed(path), + new ParseCommand.JsonRenderer() { + @Override + public String render(ParsedDocument doc, Path source, PdfParserBackend parser, boolean trustDocument) { + renderCalls.incrementAndGet(); + return "{}"; + } + + @Override + public void write( + ParsedDocument doc, + Path source, + PdfParserBackend parser, + boolean trustDocument, + Path out) + throws CliException { + writeCalls.incrementAndGet(); + try { + Files.writeString(out, "{}"); + } catch (java.io.IOException e) { + throw new CliException("write failed", e); + } + } + }); + + command.run(new String[] {"parse", "contract.pdf", "--format", "json", "-o", output.toString()}); + + assertThat(renderCalls).hasValue(0); + assertThat(writeCalls).hasValue(1); + } + + private static ParsedDocument parsed(Path path) { + return new ParsedDocument( + "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + List.of(), + new DocumentMetadata(path.getFileName().toString(), 1, java.util.Optional.empty())); + } + + private static ParseCommand.JsonRenderer countingRenderer(AtomicInteger renderCalls, AtomicInteger writeCalls) { + return new ParseCommand.JsonRenderer() { + @Override + public String render(ParsedDocument doc, Path source, PdfParserBackend parser, boolean trustDocument) { + renderCalls.incrementAndGet(); + return "{}"; + } + + @Override + public void write( + ParsedDocument doc, Path source, PdfParserBackend parser, boolean trustDocument, Path out) + throws CliException { + writeCalls.incrementAndGet(); + try { + Files.writeString(out, "{}"); + } catch (java.io.IOException e) { + throw new CliException("write failed", e); + } + } + }; + } +} diff --git a/src/test/resources/ai/doctruth/public-api-snapshot.txt b/src/test/resources/ai/doctruth/public-api-snapshot.txt index e9882aa1..acf1cad6 100644 --- a/src/test/resources/ai/doctruth/public-api-snapshot.txt +++ b/src/test/resources/ai/doctruth/public-api-snapshot.txt @@ -22,9 +22,10 @@ TYPE record ai.doctruth.BoundingBox [public final] method java.lang.String toString() [public final] TYPE record ai.doctruth.Citation [public final] - record-components ai.doctruth.SourceLocation location, java.lang.String exactQuote, double matchScore, java.util.Optional boundingBox + record-components ai.doctruth.SourceLocation location, java.lang.String exactQuote, double matchScore, java.util.Optional boundingBox, java.util.Optional source ctor Citation(ai.doctruth.SourceLocation, java.lang.String, double) ctor Citation(ai.doctruth.SourceLocation, java.lang.String, double, java.util.Optional) + ctor Citation(ai.doctruth.SourceLocation, java.lang.String, double, java.util.Optional, java.util.Optional) method ai.doctruth.SourceLocation location() [public] method boolean equals(java.lang.Object) [public final] method double matchScore() [public] @@ -32,6 +33,16 @@ TYPE record ai.doctruth.Citation [public final] method java.lang.String exactQuote() [public] method java.lang.String toString() [public final] method java.util.Optional boundingBox() [public] + method java.util.Optional source() [public] + +TYPE record ai.doctruth.CitationSource [public final] + record-components java.lang.String docId, java.lang.String unitId + ctor CitationSource(java.lang.String, java.lang.String) + method boolean equals(java.lang.Object) [public final] + method int hashCode() [public final] + method java.lang.String docId() [public] + method java.lang.String toString() [public final] + method java.lang.String unitId() [public] TYPE record ai.doctruth.Confidence [public final] record-components double score, java.lang.String rationale @@ -87,6 +98,7 @@ TYPE class ai.doctruth.DocumentJsonExtractionBuilder [public final] method ai.doctruth.DocumentJsonExtractionBuilder requireCitation(java.lang.String) [public] method ai.doctruth.DocumentJsonExtractionBuilder withContextStrategy(ai.doctruth.ContextStrategy) [public] method ai.doctruth.DocumentJsonExtractionBuilder withEvidence() [public] + method ai.doctruth.DocumentJsonExtractionBuilder withEvidenceFirst() [public] method ai.doctruth.DocumentJsonExtractionBuilder withMaxRetries(int) [public] method ai.doctruth.DocumentJsonExtractionBuilder withSourcePublishedAt(java.time.Instant) [public] method ai.doctruth.ExtractionResult runJson() [public] @@ -173,6 +185,7 @@ TYPE class ai.doctruth.JsonExtractionBuilder [public final] method ai.doctruth.JsonExtractionBuilder withBitemporal() [public] method ai.doctruth.JsonExtractionBuilder withConfidence() [public] method ai.doctruth.JsonExtractionBuilder withContextStrategy(ai.doctruth.ContextStrategy) [public] + method ai.doctruth.JsonExtractionBuilder withEvidenceFirst() [public] method ai.doctruth.JsonExtractionBuilder withMaxRetries(int) [public] method ai.doctruth.JsonExtractionBuilder withProvenance() [public] method ai.doctruth.JsonExtractionBuilder withSourcePublishedAt(java.time.Instant) [public] @@ -229,6 +242,14 @@ TYPE record ai.doctruth.ParsedDocument [public final] TYPE interface ai.doctruth.ParsedSection [public abstract interface] permits ai.doctruth.TextSection, ai.doctruth.TableSection, ai.doctruth.FigureSection +TYPE record ai.doctruth.ParserRun [public final] + record-components java.lang.String backend + ctor ParserRun(java.lang.String) + method boolean equals(java.lang.Object) [public final] + method int hashCode() [public final] + method java.lang.String backend() [public] + method java.lang.String toString() [public final] + TYPE class ai.doctruth.PdfDocumentParser [public final] method ai.doctruth.ParsedDocument parse(java.nio.file.Path) [public static] method ai.doctruth.ParsedDocument parse(java.nio.file.Path, ai.doctruth.PdfParserBackend) [public static] @@ -368,6 +389,66 @@ TYPE record ai.doctruth.TextSection [public final] method java.lang.String toString() [public final] method java.util.Optional boundingBox() [public] +TYPE record ai.doctruth.TrustDocument [public final] + record-components java.lang.String schemaVersion, java.lang.String docId, ai.doctruth.TrustDocumentSource source, ai.doctruth.ParserRun parserRun, java.util.List units + ctor TrustDocument(java.lang.String, java.lang.String, ai.doctruth.TrustDocumentSource, ai.doctruth.ParserRun, java.util.List) + method ai.doctruth.ParserRun parserRun() [public] + method ai.doctruth.TrustDocument fromParsed(ai.doctruth.ParsedDocument, java.nio.file.Path, ai.doctruth.PdfParserBackend) [public static] + method ai.doctruth.TrustDocument fromParsed(ai.doctruth.ParsedDocument, java.nio.file.Path, java.lang.String) [public static] + method ai.doctruth.TrustDocumentSource source() [public] + method boolean equals(java.lang.Object) [public final] + method int hashCode() [public final] + method java.lang.String docId() [public] + method java.lang.String schemaVersion() [public] + method java.lang.String toString() [public final] + method java.util.List units() [public] + +TYPE class ai.doctruth.TrustDocumentJson [public final] + method java.lang.String toJson(ai.doctruth.ParsedDocument, java.nio.file.Path, ai.doctruth.PdfParserBackend) [public static] + method java.lang.String toJson(ai.doctruth.ParsedDocument, java.nio.file.Path, java.lang.String) [public static] + method java.lang.String toJson(ai.doctruth.TrustDocument) [public static] + method void writeJson(ai.doctruth.ParsedDocument, java.nio.file.Path, ai.doctruth.PdfParserBackend, java.io.Writer) [public static] + method void writeJson(ai.doctruth.ParsedDocument, java.nio.file.Path, ai.doctruth.PdfParserBackend, java.nio.file.Path) [public static] + method void writeJson(ai.doctruth.ParsedDocument, java.nio.file.Path, java.lang.String, java.io.Writer) [public static] + method void writeJson(ai.doctruth.ParsedDocument, java.nio.file.Path, java.lang.String, java.nio.file.Path) [public static] + method void writeJson(ai.doctruth.TrustDocument, java.io.Writer) [public static] + +TYPE class ai.doctruth.TrustDocumentParser [public final] + method ai.doctruth.TrustDocument parse(java.nio.file.Path) [public static] + method ai.doctruth.TrustDocument parse(java.nio.file.Path, ai.doctruth.PdfParserBackend) [public static] + +TYPE record ai.doctruth.TrustDocumentSource [public final] + record-components java.lang.String filename, java.lang.String sha256, int pageCount + ctor TrustDocumentSource(java.lang.String, java.lang.String, int) + method boolean equals(java.lang.Object) [public final] + method int hashCode() [public final] + method int pageCount() [public] + method java.lang.String filename() [public] + method java.lang.String sha256() [public] + method java.lang.String toString() [public final] + +TYPE record ai.doctruth.TrustUnit [public final] + record-components java.lang.String id, java.lang.String type, java.lang.String text, java.util.List rows, ai.doctruth.TrustUnitEvidence evidence + ctor TrustUnit(java.lang.String, java.lang.String, java.lang.String, java.util.List, ai.doctruth.TrustUnitEvidence) + method ai.doctruth.TrustUnitEvidence evidence() [public] + method boolean equals(java.lang.Object) [public final] + method int hashCode() [public final] + method java.lang.String id() [public] + method java.lang.String text() [public] + method java.lang.String toString() [public final] + method java.lang.String type() [public] + method java.util.List rows() [public] + +TYPE record ai.doctruth.TrustUnitEvidence [public final] + record-components ai.doctruth.SourceLocation location, java.util.Optional boundingBox, java.util.Optional blockKind + ctor TrustUnitEvidence(ai.doctruth.SourceLocation, java.util.Optional, java.util.Optional) + method ai.doctruth.SourceLocation location() [public] + method boolean equals(java.lang.Object) [public final] + method int hashCode() [public final] + method java.lang.String toString() [public final] + method java.util.Optional blockKind() [public] + method java.util.Optional boundingBox() [public] + TYPE class ai.doctruth.XlsxDocumentParser [public final] method ai.doctruth.ParsedDocument parse(java.nio.file.Path) [public static] From 3871a69b3cdf51e7cebd970adbafc6370db04eec Mon Sep 17 00:00:00 2001 From: JamesbbBriz Date: Sun, 5 Jul 2026 13:19:58 +0800 Subject: [PATCH 2/2] fix: harden evidence extraction contracts --- .../java/ai/doctruth/EvidenceFirstJson.java | 108 +++++++++- .../ai/doctruth/JsonExtractionBuilder.java | 63 +++++- .../OpenDataLoaderPdfDocumentParser.java | 66 +----- src/main/java/ai/doctruth/TrustDocument.java | 49 ++--- .../java/ai/doctruth/TrustDocumentJson.java | 8 +- src/main/java/ai/doctruth/TrustUnit.java | 3 +- .../java/ai/doctruth/cli/ExtractCommand.java | 46 ++-- .../java/ai/doctruth/cli/IncludeOutput.java | 30 +++ .../java/ai/doctruth/cli/ParseCommand.java | 3 +- .../ai/doctruth/cli/ParsedDocumentJson.java | 4 +- .../java/ai/doctruth/cli/ProfileCommand.java | 114 +--------- .../java/ai/doctruth/cli/ProfileResult.java | 72 +++++++ .../internal/citation/CitationMatcher.java | 19 +- .../citation/RenderedCitationSection.java | 13 ++ .../internal/schema/JsonSchemaResolver.java | 44 ++++ ...idenceFirstJsonExtractionContractTest.java | 202 ++++++++++++++++++ .../ai/doctruth/EvidenceFirstJsonTest.java | 25 ++- .../JsonSchemaCitationContractTest.java | 118 ++++++++++ .../JsonSchemaExtractionContractTest.java | 124 ----------- .../OpenDataLoaderPdfDocumentParserTest.java | 49 ++++- .../OpenDataLoaderSectionMapperTest.java | 3 +- .../ai/doctruth/TrustDocumentParserTest.java | 14 +- .../cli/DocTruthCliExtractContractTest.java | 71 +++++- .../cli/DocTruthCliTrustDocumentTest.java | 20 +- .../cli/ParseCommandPerformanceTest.java | 47 ++-- .../schema/JsonSchemaResolverTest.java | 67 ++++++ 26 files changed, 956 insertions(+), 426 deletions(-) create mode 100644 src/main/java/ai/doctruth/cli/IncludeOutput.java create mode 100644 src/main/java/ai/doctruth/cli/ProfileResult.java create mode 100644 src/main/java/ai/doctruth/internal/citation/RenderedCitationSection.java create mode 100644 src/main/java/ai/doctruth/internal/schema/JsonSchemaResolver.java create mode 100644 src/test/java/ai/doctruth/EvidenceFirstJsonExtractionContractTest.java create mode 100644 src/test/java/ai/doctruth/JsonSchemaCitationContractTest.java create mode 100644 src/test/java/ai/doctruth/internal/schema/JsonSchemaResolverTest.java diff --git a/src/main/java/ai/doctruth/EvidenceFirstJson.java b/src/main/java/ai/doctruth/EvidenceFirstJson.java index 2cca0745..d5a6aaa9 100644 --- a/src/main/java/ai/doctruth/EvidenceFirstJson.java +++ b/src/main/java/ai/doctruth/EvidenceFirstJson.java @@ -3,6 +3,8 @@ import java.util.LinkedHashMap; import java.util.Map; +import ai.doctruth.internal.schema.JsonSchemaResolver; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -17,22 +19,30 @@ private EvidenceFirstJson() { } static JsonNode responseSchema(JsonNode schema) { + return responseSchema(schema, schema); + } + + private static JsonNode responseSchema(JsonNode schema, JsonNode root) { if (schema == null || schema.isMissingNode() || schema.isNull()) { return leafSchema(schema); } - if ("object".equals(schema.path("type").asText()) && schema.path("properties").isObject()) { - ObjectNode copy = schema.deepCopy(); + JsonNode activeSchema = JsonSchemaResolver.resolveLocal(schema, root); + if (isObjectSchema(activeSchema)) { + ObjectNode copy = activeSchema.deepCopy(); ObjectNode properties = MAPPER.createObjectNode(); - schema.path("properties").fields().forEachRemaining(e -> properties.set(e.getKey(), responseSchema(e.getValue()))); + activeSchema + .path("properties") + .fields() + .forEachRemaining(e -> properties.set(e.getKey(), responseSchema(e.getValue(), root))); copy.set("properties", properties); return copy; } - if ("array".equals(schema.path("type").asText()) && schema.has("items")) { - ObjectNode copy = schema.deepCopy(); - copy.set("items", responseSchema(schema.path("items"))); + if (isArraySchema(activeSchema)) { + ObjectNode copy = activeSchema.deepCopy(); + copy.set("items", responseSchema(activeSchema.path("items"), root)); return copy; } - return leafSchema(schema); + return leafSchema(activeSchema); } static JsonNode unwrap(JsonNode evidenceNode) { @@ -52,17 +62,57 @@ static JsonNode unwrap(JsonNode evidenceNode) { return evidenceNode == null ? MAPPER.nullNode() : evidenceNode.deepCopy(); } + static JsonNode unwrap(JsonNode evidenceNode, JsonNode schema) { + return unwrap(evidenceNode, schema, schema); + } + + private static JsonNode unwrap(JsonNode evidenceNode, JsonNode schema, JsonNode root) { + if (evidenceNode == null || evidenceNode.isNull() || evidenceNode.isMissingNode()) { + return MAPPER.nullNode(); + } + JsonNode activeSchema = JsonSchemaResolver.resolveLocal(schema, root); + if (isObjectSchema(activeSchema) && evidenceNode.isObject()) { + ObjectNode out = MAPPER.createObjectNode(); + evidenceNode.fields().forEachRemaining(e -> { + JsonNode childSchema = activeSchema.path("properties").path(e.getKey()); + out.set(e.getKey(), unwrap(e.getValue(), childSchema, root)); + }); + return out; + } + if (isArraySchema(activeSchema) && evidenceNode.isArray()) { + ArrayNode out = MAPPER.createArrayNode(); + for (JsonNode item : evidenceNode) { + out.add(unwrap(item, activeSchema.path("items"), root)); + } + return out; + } + if (isEvidenceLeaf(evidenceNode)) { + return evidenceNode.path("value").deepCopy(); + } + return evidenceNode.deepCopy(); + } + static Map quoteMap(JsonNode evidenceNode) { var out = new LinkedHashMap(); collectQuotes("", evidenceNode, out); return Map.copyOf(out); } + static Map quoteMap(JsonNode evidenceNode, JsonNode schema) { + var out = new LinkedHashMap(); + collectQuotes("", evidenceNode, schema, schema, out); + return Map.copyOf(out); + } + private static ObjectNode leafSchema(JsonNode valueSchema) { ObjectNode wrapper = MAPPER.createObjectNode(); wrapper.put("type", "object"); ObjectNode properties = MAPPER.createObjectNode(); - properties.set("value", valueSchema == null || valueSchema.isMissingNode() ? MAPPER.createObjectNode() : valueSchema.deepCopy()); + properties.set( + "value", + valueSchema == null || valueSchema.isMissingNode() + ? MAPPER.createObjectNode() + : valueSchema.deepCopy()); ObjectNode quote = MAPPER.createObjectNode(); quote.put("type", "string"); quote.put("minLength", 1); @@ -80,6 +130,18 @@ private static boolean isEvidenceLeaf(JsonNode node) { return node != null && node.isObject() && node.has("value") && node.has("exactQuote"); } + private static boolean isObjectSchema(JsonNode schema) { + return schema != null + && schema.path("properties").isObject() + && ("object".equals(schema.path("type").asText()) || schema.has("properties")); + } + + private static boolean isArraySchema(JsonNode schema) { + return schema != null + && schema.has("items") + && ("array".equals(schema.path("type").asText()) || schema.has("items")); + } + private static void collectQuotes(String path, JsonNode node, Map out) { if (node == null || node.isNull() || node.isMissingNode()) { return; @@ -102,6 +164,36 @@ private static void collectQuotes(String path, JsonNode node, Map out) { + if (node == null || node.isNull() || node.isMissingNode()) { + return; + } + JsonNode activeSchema = JsonSchemaResolver.resolveLocal(schema, root); + if (isObjectSchema(activeSchema) && node.isObject()) { + node.fields() + .forEachRemaining(e -> collectQuotes( + joinPath(path, e.getKey()), + e.getValue(), + activeSchema.path("properties").path(e.getKey()), + root, + out)); + return; + } + if (isArraySchema(activeSchema) && node.isArray()) { + for (int i = 0; i < node.size(); i++) { + collectQuotes(path + "[" + i + "]", node.get(i), activeSchema.path("items"), root, out); + } + return; + } + if (isEvidenceLeaf(node)) { + String quote = node.path("exactQuote").asText(); + if (!quote.isBlank()) { + out.put(path, quote); + } + } + } + private static String joinPath(String parent, String child) { return parent.isEmpty() ? child : parent + "." + child; } diff --git a/src/main/java/ai/doctruth/JsonExtractionBuilder.java b/src/main/java/ai/doctruth/JsonExtractionBuilder.java index 0d18b1e7..5817a43a 100644 --- a/src/main/java/ai/doctruth/JsonExtractionBuilder.java +++ b/src/main/java/ai/doctruth/JsonExtractionBuilder.java @@ -89,9 +89,10 @@ public ExtractionResult runJson(ParsedDocument doc) throws ExtractionE if (state.evidenceFirst) { JsonSchemaValidator.validate(rawValue, EvidenceFirstJson.responseSchema(schema.node()), retry); } - JsonNode value = state.evidenceFirst ? EvidenceFirstJson.unwrap(rawValue) : rawValue; + JsonNode value = state.evidenceFirst ? EvidenceFirstJson.unwrap(rawValue, schema.node()) : rawValue; JsonSchemaValidator.validate(value, schema.node(), retry); - Object citationSource = state.evidenceFirst ? EvidenceFirstJson.quoteMap(rawValue) : value; + Object citationSource = + state.evidenceFirst ? EvidenceFirstJson.quoteMap(rawValue, schema.node()) : value; Map citations = citations(citationSource, doc, retry); return result(response, value, citations, retry); } catch (ExtractionException e) { @@ -166,13 +167,13 @@ private Map citations(Object citationSource, ParsedDocument do return matched; } return matched.entrySet().stream() - .filter(e -> state.requiredCitations.contains(e.getKey())) + .filter(e -> isRequiredCitation(e.getKey())) .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue)); } private void requireStrongCitations(Map matched, int retries) throws ExtractionException { for (String fieldPath : state.requiredCitations) { - Citation citation = matched.get(fieldPath); + Citation citation = bestRequiredCitation(matched, fieldPath); if (citation == null || citation.matchScore() < CitationMatcher.DEFAULT_MIN_SCORE) { throw new ExtractionException( "EXTRACTION_EVIDENCE_MISSING", @@ -182,6 +183,57 @@ private void requireStrongCitations(Map matched, int retries) } } + private boolean isRequiredCitation(String actualPath) { + return state.requiredCitations.stream().anyMatch(requiredPath -> citationPathMatches(requiredPath, actualPath)); + } + + private static Citation bestRequiredCitation(Map matched, String requiredPath) { + return matched.entrySet().stream() + .filter(entry -> citationPathMatches(requiredPath, entry.getKey())) + .map(Map.Entry::getValue) + .max((left, right) -> Double.compare(left.matchScore(), right.matchScore())) + .orElse(null); + } + + private static boolean citationPathMatches(String requiredPath, String actualPath) { + if (requiredPath.equals(actualPath)) { + return true; + } + if (!requiredPath.contains("[]")) { + return false; + } + int requiredIndex = 0; + int actualIndex = 0; + while (requiredIndex < requiredPath.length()) { + if (requiredIndex + 1 < requiredPath.length() + && requiredPath.charAt(requiredIndex) == '[' + && requiredPath.charAt(requiredIndex + 1) == ']') { + if (actualIndex >= actualPath.length() || actualPath.charAt(actualIndex) != '[') { + return false; + } + int close = actualPath.indexOf(']', actualIndex); + if (close <= actualIndex + 1) { + return false; + } + for (int i = actualIndex + 1; i < close; i++) { + if (!Character.isDigit(actualPath.charAt(i))) { + return false; + } + } + actualIndex = close + 1; + requiredIndex += 2; + continue; + } + if (actualIndex >= actualPath.length() + || requiredPath.charAt(requiredIndex) != actualPath.charAt(actualIndex)) { + return false; + } + requiredIndex++; + actualIndex++; + } + return actualIndex == actualPath.length(); + } + private ExtractionResult result( ProviderResponse response, JsonNode value, Map citations, int retries) { Map confidence = state.recordConfidence ? confidenceFrom(citations) : Map.of(); @@ -204,7 +256,8 @@ private static Map confidenceFrom(Map cita return citations.entrySet().stream() .collect(Collectors.toUnmodifiableMap( Map.Entry::getKey, - e -> new Confidence(e.getValue().matchScore(), "source evidence quote matched for JSON field"))); + e -> new Confidence( + e.getValue().matchScore(), "source evidence quote matched for JSON field"))); } private static String renderUserPrompt(ParsedDocument doc) { diff --git a/src/main/java/ai/doctruth/OpenDataLoaderPdfDocumentParser.java b/src/main/java/ai/doctruth/OpenDataLoaderPdfDocumentParser.java index 8df5e0fa..01c3a79b 100644 --- a/src/main/java/ai/doctruth/OpenDataLoaderPdfDocumentParser.java +++ b/src/main/java/ai/doctruth/OpenDataLoaderPdfDocumentParser.java @@ -29,28 +29,10 @@ final class OpenDataLoaderPdfDocumentParser { private static OpenDataLoaderRunner openDataLoaderRunner = (pdfPath, outputDir, config) -> OpenDataLoaderPDF.processFile(pdfPath.toString(), config); - static { - suppressOpenDataLoaderInfoLogging(); - } - private OpenDataLoaderPdfDocumentParser() { throw new AssertionError("no instances"); } - private static void suppressOpenDataLoaderInfoLogging() { - for (String name : List.of( - "org.opendataloader", - "org.opendataloader.pdf", - "org.opendataloader.pdf.processors", - "org.opendataloader.pdf.processors.DocumentProcessor", - "org.opendataloader.pdf.json", - "org.opendataloader.pdf.json.JsonWriter")) { - java.util.logging.Logger logger = java.util.logging.Logger.getLogger(name); - logger.setLevel(java.util.logging.Level.WARNING); - logger.setUseParentHandlers(false); - } - } - static ParsedDocument parse(Path pdfPath) throws ParseException { Objects.requireNonNull(pdfPath, "pdfPath"); requireRegularFile(pdfPath); @@ -61,10 +43,7 @@ static ParsedDocument parse(Path pdfPath) throws ParseException { Config config = config(outputDir); synchronized (OPENDATALOADER_LOCK) { try { - try (var suppression = JulInfoSuppression.open()) { - suppression.keepAlive(); - openDataLoaderRunner.process(pdfPath, outputDir, config); - } + openDataLoaderRunner.process(pdfPath, outputDir, config); } finally { OpenDataLoaderPDF.shutdown(); } @@ -240,47 +219,4 @@ private static void deleteRecursively(Path dir) { LOG.debug("failed to inspect temporary OpenDataLoader directory {}", dir, e); } } - - private static final class JulInfoSuppression implements AutoCloseable { - private final java.util.logging.Logger root; - private final java.util.logging.Level rootLevel; - private final java.util.logging.Handler[] handlers; - private final java.util.logging.Level[] handlerLevels; - - private JulInfoSuppression( - java.util.logging.Logger root, - java.util.logging.Level rootLevel, - java.util.logging.Handler[] handlers, - java.util.logging.Level[] handlerLevels) { - this.root = root; - this.rootLevel = rootLevel; - this.handlers = handlers; - this.handlerLevels = handlerLevels; - } - - static JulInfoSuppression open() { - java.util.logging.Logger root = java.util.logging.Logger.getLogger(""); - java.util.logging.Handler[] handlers = root.getHandlers(); - java.util.logging.Level[] handlerLevels = new java.util.logging.Level[handlers.length]; - for (int i = 0; i < handlers.length; i++) { - handlerLevels[i] = handlers[i].getLevel(); - handlers[i].setLevel(java.util.logging.Level.WARNING); - } - java.util.logging.Level rootLevel = root.getLevel(); - root.setLevel(java.util.logging.Level.WARNING); - return new JulInfoSuppression(root, rootLevel, handlers, handlerLevels); - } - - void keepAlive() { - // Referenced to keep javac -Xlint:try satisfied for this scoped guard. - } - - @Override - public void close() { - root.setLevel(rootLevel); - for (int i = 0; i < handlers.length; i++) { - handlers[i].setLevel(handlerLevels[i]); - } - } - } } diff --git a/src/main/java/ai/doctruth/TrustDocument.java b/src/main/java/ai/doctruth/TrustDocument.java index 38b659b9..8244f9a2 100644 --- a/src/main/java/ai/doctruth/TrustDocument.java +++ b/src/main/java/ai/doctruth/TrustDocument.java @@ -17,11 +17,7 @@ * @since 0.2.0 */ public record TrustDocument( - String schemaVersion, - String docId, - TrustDocumentSource source, - ParserRun parserRun, - List units) { + String schemaVersion, String docId, TrustDocumentSource source, ParserRun parserRun, List units) { public static final String SCHEMA_VERSION = "doctruth.trust-document.v1"; @@ -52,7 +48,9 @@ public static TrustDocument fromParsed(ParsedDocument parsed, Path sourcePath, S Objects.requireNonNull(sourcePath, "sourcePath"); Objects.requireNonNull(backend, "backend"); var source = new TrustDocumentSource( - sourceFilename(sourcePath), sha256FromDocId(parsed.docId()), parsed.metadata().pageCount()); + sourceFilename(sourcePath), + sha256FromDocId(parsed.docId()), + parsed.metadata().pageCount()); return new TrustDocument(SCHEMA_VERSION, parsed.docId(), source, new ParserRun(backend), units(parsed)); } @@ -81,24 +79,27 @@ private static List units(ParsedDocument parsed) { private static TrustUnit unit(String id, ParsedSection section) { return switch (section) { - case TextSection text -> new TrustUnit( - id, - "text", - text.text(), - List.of(), - new TrustUnitEvidence(text.location(), text.boundingBox(), Optional.of(text.kind()))); - case TableSection table -> new TrustUnit( - id, - "table", - "", - table.rows(), - new TrustUnitEvidence(table.location(), Optional.empty(), Optional.empty())); - case FigureSection figure -> new TrustUnit( - id, - "figure", - figure.caption(), - List.of(), - new TrustUnitEvidence(figure.location(), Optional.empty(), Optional.empty())); + case TextSection text -> + new TrustUnit( + id, + "text", + text.text(), + List.of(), + new TrustUnitEvidence(text.location(), text.boundingBox(), Optional.of(text.kind()))); + case TableSection table -> + new TrustUnit( + id, + "table", + "", + table.rows(), + new TrustUnitEvidence(table.location(), Optional.empty(), Optional.empty())); + case FigureSection figure -> + new TrustUnit( + id, + "figure", + figure.caption(), + List.of(), + new TrustUnitEvidence(figure.location(), Optional.empty(), Optional.empty())); }; } } diff --git a/src/main/java/ai/doctruth/TrustDocumentJson.java b/src/main/java/ai/doctruth/TrustDocumentJson.java index d31532c6..0885679d 100644 --- a/src/main/java/ai/doctruth/TrustDocumentJson.java +++ b/src/main/java/ai/doctruth/TrustDocumentJson.java @@ -113,8 +113,12 @@ private static void writeParsedDocument(JsonGenerator json, ParsedDocument doc, json.writeStartObject(); json.writeStringField("schemaVersion", TrustDocument.SCHEMA_VERSION); json.writeStringField("docId", doc.docId()); - writeSource(json, new TrustDocumentSource( - sourceFilename(source), sha256FromDocId(doc.docId()), doc.metadata().pageCount())); + writeSource( + json, + new TrustDocumentSource( + sourceFilename(source), + sha256FromDocId(doc.docId()), + doc.metadata().pageCount())); writeParserRun(json, backend); json.writeArrayFieldStart("units"); int id = 1; diff --git a/src/main/java/ai/doctruth/TrustUnit.java b/src/main/java/ai/doctruth/TrustUnit.java index 8976ca14..f8f8e807 100644 --- a/src/main/java/ai/doctruth/TrustUnit.java +++ b/src/main/java/ai/doctruth/TrustUnit.java @@ -13,8 +13,7 @@ * @param evidence source anchor and visual evidence. * @since 0.2.0 */ -public record TrustUnit( - String id, String type, String text, List> rows, TrustUnitEvidence evidence) { +public record TrustUnit(String id, String type, String text, List> rows, TrustUnitEvidence evidence) { public TrustUnit { Objects.requireNonNull(id, "id"); diff --git a/src/main/java/ai/doctruth/cli/ExtractCommand.java b/src/main/java/ai/doctruth/cli/ExtractCommand.java index 25a5935c..fdc27700 100644 --- a/src/main/java/ai/doctruth/cli/ExtractCommand.java +++ b/src/main/java/ai/doctruth/cli/ExtractCommand.java @@ -15,6 +15,7 @@ import ai.doctruth.JsonSchema; import ai.doctruth.ParsedDocument; import ai.doctruth.TrustDocumentJson; +import ai.doctruth.internal.schema.JsonSchemaResolver; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -68,14 +69,16 @@ private static ExtractionResult runExtraction( } } - private static void writeOutputs( - Path dir, Path source, ParsedDocument doc, ExtractionResult result) throws CliException { + private static void writeOutputs(Path dir, Path source, ParsedDocument doc, ExtractionResult result) + throws CliException { try { Files.createDirectories(dir); - TrustDocumentJson.writeJson(doc, source, DocumentParsers.parserId(source), dir.resolve("trust-document.json")); + TrustDocumentJson.writeJson( + doc, source, DocumentParsers.parserId(source), dir.resolve("trust-document.json")); Files.writeString(dir.resolve("result.json"), result.value().toPrettyString()); result.toAuditJson(dir.resolve("audit.json")); - Files.writeString(dir.resolve("manifest.json"), manifest(source, doc).toPrettyString()); + Files.writeString( + dir.resolve("manifest.json"), manifest(source, doc).toPrettyString()); } catch (IOException e) { throw new CliException("failed to write extraction outputs: " + e.getMessage(), e); } @@ -84,7 +87,8 @@ private static void writeOutputs( private static ObjectNode manifest(Path source, ParsedDocument doc) { ObjectNode root = MAPPER.createObjectNode(); root.put("schemaVersion", RUN_SCHEMA_VERSION); - root.put("source", source.toString()); + root.put("sourceFilename", source.getFileName().toString()); + root.put("sourceSha256", sourceSha256(doc.docId())); root.put("docId", doc.docId()); root.put("parser", DocumentParsers.parserId(source)); ObjectNode artifacts = MAPPER.createObjectNode(); @@ -95,6 +99,10 @@ private static ObjectNode manifest(Path source, ParsedDocument doc) { return root; } + private static String sourceSha256(String docId) { + return docId.startsWith("sha256:") ? docId.substring("sha256:".length()) : docId; + } + private void printSummary(Path dir, ExtractionResult result) { int fields = result.value().isObject() ? result.value().size() : 1; long weak = result.citations().values().stream() @@ -180,18 +188,23 @@ Set requiredFields(JsonSchema schema) { return require; } var fields = new LinkedHashSet(); - collectLeafFields("", schema.node(), fields); + JsonNode root = schema.node(); + collectLeafFields("", root, root, fields); return Set.copyOf(fields); } - private static void collectLeafFields(String path, JsonNode schema, Set fields) { - var properties = schema.path("properties"); - if ("object".equals(schema.path("type").asText()) && properties.isObject()) { - properties.fields().forEachRemaining(e -> collectLeafFields(joinPath(path, e.getKey()), e.getValue(), fields)); + private static void collectLeafFields(String path, JsonNode schema, JsonNode root, Set fields) { + JsonNode activeSchema = JsonSchemaResolver.resolveLocal(schema, root); + var properties = activeSchema.path("properties"); + if (isObjectSchema(activeSchema)) { + properties + .fields() + .forEachRemaining( + e -> collectLeafFields(joinPath(path, e.getKey()), e.getValue(), root, fields)); return; } - if ("array".equals(schema.path("type").asText()) && schema.has("items")) { - collectLeafFields(path, schema.path("items"), fields); + if (isArraySchema(activeSchema)) { + collectLeafFields(path + "[]", activeSchema.path("items"), root, fields); return; } if (!path.isBlank()) { @@ -199,6 +212,15 @@ private static void collectLeafFields(String path, JsonNode schema, Set } } + private static boolean isObjectSchema(JsonNode schema) { + return schema.path("properties").isObject() + && ("object".equals(schema.path("type").asText()) || schema.has("properties")); + } + + private static boolean isArraySchema(JsonNode schema) { + return schema.has("items") && ("array".equals(schema.path("type").asText()) || schema.has("items")); + } + private static String joinPath(String parent, String child) { return parent.isEmpty() ? child : parent + "." + child; } diff --git a/src/main/java/ai/doctruth/cli/IncludeOutput.java b/src/main/java/ai/doctruth/cli/IncludeOutput.java new file mode 100644 index 00000000..4eadcd63 --- /dev/null +++ b/src/main/java/ai/doctruth/cli/IncludeOutput.java @@ -0,0 +1,30 @@ +package ai.doctruth.cli; + +enum IncludeOutput { + PARSER_ONLY("parser-only"), + TRUST_JSON("trust-json"), + PARSED_JSON("parsed-json"), + TRUST_JSON_FILE("trust-json-file"), + PARSED_JSON_FILE("parsed-json-file"); + + private final String id; + + IncludeOutput(String id) { + this.id = id; + } + + String id() { + return id; + } + + static IncludeOutput parse(String raw) { + return switch (raw) { + case "parser-only", "none" -> PARSER_ONLY; + case "trust-json", "json", "trust-document-json" -> TRUST_JSON; + case "parsed-json" -> PARSED_JSON; + case "trust-json-file", "trust-document-json-file", "json-file" -> TRUST_JSON_FILE; + case "parsed-json-file" -> PARSED_JSON_FILE; + default -> throw new UsageException("unsupported profile output mode: " + raw); + }; + } +} diff --git a/src/main/java/ai/doctruth/cli/ParseCommand.java b/src/main/java/ai/doctruth/cli/ParseCommand.java index 3688023b..81546234 100644 --- a/src/main/java/ai/doctruth/cli/ParseCommand.java +++ b/src/main/java/ai/doctruth/cli/ParseCommand.java @@ -21,8 +21,7 @@ public String render(ParsedDocument doc, Path source, PdfParserBackend parser, b } @Override - public void write( - ParsedDocument doc, Path source, PdfParserBackend parser, boolean trustDocument, Path out) + public void write(ParsedDocument doc, Path source, PdfParserBackend parser, boolean trustDocument, Path out) throws CliException { writeJson(doc, source, parser, trustDocument, out); } diff --git a/src/main/java/ai/doctruth/cli/ParsedDocumentJson.java b/src/main/java/ai/doctruth/cli/ParsedDocumentJson.java index df5d75fb..bee6d685 100644 --- a/src/main/java/ai/doctruth/cli/ParsedDocumentJson.java +++ b/src/main/java/ai/doctruth/cli/ParsedDocumentJson.java @@ -63,7 +63,9 @@ private static void writeDocument(JsonGenerator json, ParsedDocument doc) throws json.writeStringField("sourceFilename", doc.metadata().sourceFilename()); json.writeNumberField("pageCount", doc.metadata().pageCount()); if (doc.metadata().sourcePublishedAt().isPresent()) { - json.writeStringField("sourcePublishedAt", doc.metadata().sourcePublishedAt().get().toString()); + json.writeStringField( + "sourcePublishedAt", + doc.metadata().sourcePublishedAt().get().toString()); } json.writeEndObject(); json.writeArrayFieldStart("sections"); diff --git a/src/main/java/ai/doctruth/cli/ProfileCommand.java b/src/main/java/ai/doctruth/cli/ProfileCommand.java index 6e86a224..727407b1 100644 --- a/src/main/java/ai/doctruth/cli/ProfileCommand.java +++ b/src/main/java/ai/doctruth/cli/ProfileCommand.java @@ -9,13 +9,8 @@ import ai.doctruth.PdfParserBackend; import ai.doctruth.TrustDocumentJson; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; - final class ProfileCommand { - private static final ObjectMapper MAPPER = new ObjectMapper(); - private final CliContext context; ProfileCommand(CliContext context) { @@ -84,11 +79,16 @@ private static ProfileResult measure(ProfileOptions options) throws CliException private static OutputMeasurement renderOutput(ParsedDocument doc, ProfileOptions options) throws CliException { return switch (options.includeOutput()) { case PARSER_ONLY -> new OutputMeasurement(0, 0); - case TRUST_JSON -> new OutputMeasurement( - TrustDocumentJson.toJson(doc, options.document(), options.parser()).length(), 0); - case PARSED_JSON -> new OutputMeasurement(ParsedDocumentJson.toJson(doc).length(), 0); - case TRUST_JSON_FILE -> renderFileOutput(output -> TrustDocumentJson.writeJson( - doc, options.document(), options.parser(), output)); + case TRUST_JSON -> + new OutputMeasurement( + TrustDocumentJson.toJson(doc, options.document(), options.parser()) + .length(), + 0); + case PARSED_JSON -> + new OutputMeasurement(ParsedDocumentJson.toJson(doc).length(), 0); + case TRUST_JSON_FILE -> + renderFileOutput( + output -> TrustDocumentJson.writeJson(doc, options.document(), options.parser(), output)); case PARSED_JSON_FILE -> renderFileOutput(output -> ParsedDocumentJson.writeJson(doc, output)); }; } @@ -146,35 +146,6 @@ private interface FileOutputWriter { private record OutputMeasurement(long chars, long bytes) {} - private enum IncludeOutput { - PARSER_ONLY("parser-only"), - TRUST_JSON("trust-json"), - PARSED_JSON("parsed-json"), - TRUST_JSON_FILE("trust-json-file"), - PARSED_JSON_FILE("parsed-json-file"); - - private final String id; - - IncludeOutput(String id) { - this.id = id; - } - - String id() { - return id; - } - - static IncludeOutput parse(String raw) { - return switch (raw) { - case "parser-only", "none" -> PARSER_ONLY; - case "trust-json", "json", "trust-document-json" -> TRUST_JSON; - case "parsed-json" -> PARSED_JSON; - case "trust-json-file", "trust-document-json-file", "json-file" -> TRUST_JSON_FILE; - case "parsed-json-file" -> PARSED_JSON_FILE; - default -> throw new UsageException("unsupported profile output mode: " + raw); - }; - } - } - private record ProfileOptions( Path document, PdfParserBackend parser, int iterations, boolean json, IncludeOutput includeOutput) { static ProfileOptions parse(String[] args) { @@ -228,69 +199,4 @@ private static String nextValue(ArgCursor cursor, String option) { return cursor.next(); } } - - private record ProfileResult( - String parser, - int iterations, - long fileSizeBytes, - int sectionCount, - IncludeOutput includeOutput, - long[] parseLatencyMillis, - long[] outputLatencyMillis, - long profiledOutputChars, - long profiledOutputBytes, - long heapUsedBeforeBytes, - long heapUsedAfterBytes) { - long coldLatencyMillis() { - return parseLatencyMillis.length == 0 ? -1 : parseLatencyMillis[0]; - } - - long warmAverageLatencyMillis() { - return warmAverage(parseLatencyMillis); - } - - long coldOutputLatencyMillis() { - return outputLatencyMillis.length == 0 ? -1 : outputLatencyMillis[0]; - } - - long warmAverageOutputLatencyMillis() { - return warmAverage(outputLatencyMillis); - } - - String toJson() throws CliException { - try { - ObjectNode node = MAPPER.createObjectNode(); - node.put("parser", parser); - node.put("iterations", iterations); - node.put("fileSizeBytes", fileSizeBytes); - node.put("sectionCount", sectionCount); - node.put("includeOutput", includeOutput.id()); - node.set("parseLatencyMillis", MAPPER.valueToTree(parseLatencyMillis)); - node.set("outputLatencyMillis", MAPPER.valueToTree(outputLatencyMillis)); - node.put("coldLatencyMillis", coldLatencyMillis()); - node.put("warmAverageLatencyMillis", warmAverageLatencyMillis()); - node.put("coldOutputLatencyMillis", coldOutputLatencyMillis()); - node.put("warmAverageOutputLatencyMillis", warmAverageOutputLatencyMillis()); - node.put("profiledOutputChars", profiledOutputChars); - node.put("profiledOutputBytes", profiledOutputBytes); - node.put("heapUsedBeforeBytes", heapUsedBeforeBytes); - node.put("heapUsedAfterBytes", heapUsedAfterBytes); - node.put("heapDeltaBytes", heapUsedAfterBytes - heapUsedBeforeBytes); - return MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(node); - } catch (com.fasterxml.jackson.core.JsonProcessingException e) { - throw new CliException("failed to serialize profile JSON", e); - } - } - - private static long warmAverage(long[] values) { - if (values.length <= 1) { - return -1; - } - long total = 0; - for (int i = 1; i < values.length; i++) { - total += values[i]; - } - return total / (values.length - 1); - } - } } diff --git a/src/main/java/ai/doctruth/cli/ProfileResult.java b/src/main/java/ai/doctruth/cli/ProfileResult.java new file mode 100644 index 00000000..842ddf7f --- /dev/null +++ b/src/main/java/ai/doctruth/cli/ProfileResult.java @@ -0,0 +1,72 @@ +package ai.doctruth.cli; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +record ProfileResult( + String parser, + int iterations, + long fileSizeBytes, + int sectionCount, + IncludeOutput includeOutput, + long[] parseLatencyMillis, + long[] outputLatencyMillis, + long profiledOutputChars, + long profiledOutputBytes, + long heapUsedBeforeBytes, + long heapUsedAfterBytes) { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + long coldLatencyMillis() { + return parseLatencyMillis.length == 0 ? -1 : parseLatencyMillis[0]; + } + + long warmAverageLatencyMillis() { + return warmAverage(parseLatencyMillis); + } + + long coldOutputLatencyMillis() { + return outputLatencyMillis.length == 0 ? -1 : outputLatencyMillis[0]; + } + + long warmAverageOutputLatencyMillis() { + return warmAverage(outputLatencyMillis); + } + + String toJson() throws CliException { + try { + ObjectNode node = MAPPER.createObjectNode(); + node.put("parser", parser); + node.put("iterations", iterations); + node.put("fileSizeBytes", fileSizeBytes); + node.put("sectionCount", sectionCount); + node.put("includeOutput", includeOutput.id()); + node.set("parseLatencyMillis", MAPPER.valueToTree(parseLatencyMillis)); + node.set("outputLatencyMillis", MAPPER.valueToTree(outputLatencyMillis)); + node.put("coldLatencyMillis", coldLatencyMillis()); + node.put("warmAverageLatencyMillis", warmAverageLatencyMillis()); + node.put("coldOutputLatencyMillis", coldOutputLatencyMillis()); + node.put("warmAverageOutputLatencyMillis", warmAverageOutputLatencyMillis()); + node.put("profiledOutputChars", profiledOutputChars); + node.put("profiledOutputBytes", profiledOutputBytes); + node.put("heapUsedBeforeBytes", heapUsedBeforeBytes); + node.put("heapUsedAfterBytes", heapUsedAfterBytes); + node.put("heapDeltaBytes", heapUsedAfterBytes - heapUsedBeforeBytes); + return MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(node); + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { + throw new CliException("failed to serialize profile JSON", e); + } + } + + private static long warmAverage(long[] values) { + if (values.length <= 1) { + return -1; + } + long total = 0; + for (int i = 1; i < values.length; i++) { + total += values[i]; + } + return total / (values.length - 1); + } +} diff --git a/src/main/java/ai/doctruth/internal/citation/CitationMatcher.java b/src/main/java/ai/doctruth/internal/citation/CitationMatcher.java index 51c66bbc..0544c620 100644 --- a/src/main/java/ai/doctruth/internal/citation/CitationMatcher.java +++ b/src/main/java/ai/doctruth/internal/citation/CitationMatcher.java @@ -72,7 +72,8 @@ public Map matchAll(Object value, ParsedDocument doc) { return Map.copyOf(out); } - private Citation matchOne(String needle, String path, List sections, SourceLocation fallback) { + private Citation matchOne( + String needle, String path, List sections, SourceLocation fallback) { for (var sec : sections) { int idx = sec.text().indexOf(needle); if (idx >= 0) { @@ -94,7 +95,7 @@ private Citation matchOne(String needle, String path, List sections, S return best; } - private static Citation bestFuzzy(String needle, List sections) { + private static Citation bestFuzzy(String needle, List sections) { Citation best = null; for (var sec : sections) { var c = bestFuzzyWindow(needle, sec); @@ -108,7 +109,7 @@ private static Citation bestFuzzy(String needle, List sections) { return best; } - private static Citation bestFuzzyWindow(String needle, Rendered sec) { + private static Citation bestFuzzyWindow(String needle, RenderedCitationSection sec) { String haystack = sec.text(); if (haystack.isEmpty() || needle.isEmpty()) { return null; @@ -139,7 +140,7 @@ private static Citation bestFuzzyWindow(String needle, Rendered sec) { return citation(sec, bestQuote, Math.max(0.0, Math.min(1.0, bestScore))); } - private static Citation citation(Rendered sec, String exactQuote, double matchScore) { + private static Citation citation(RenderedCitationSection sec, String exactQuote, double matchScore) { return new Citation( sec.location(), exactQuote, @@ -173,11 +174,12 @@ private static List candidatePositions(String needle, String haystack) return out; } - private static List renderedSections(ParsedDocument doc) { - var out = new ArrayList(doc.sections().size()); + private static List renderedSections(ParsedDocument doc) { + var out = new ArrayList(doc.sections().size()); for (int i = 0; i < doc.sections().size(); i++) { var s = doc.sections().get(i); - out.add(new Rendered(textOf(s), locationOf(s), boundingBoxOf(s), doc.docId(), "u" + (i + 1))); + out.add(new RenderedCitationSection( + textOf(s), locationOf(s), boundingBoxOf(s), doc.docId(), "u" + (i + 1))); } return out; } @@ -292,7 +294,4 @@ private static String joinPath(String parent, String child) { } private record Leaf(String path, String value) {} - - private record Rendered(String text, SourceLocation location, Optional boundingBox, String sourceDocId, - String sourceUnitId) {} } diff --git a/src/main/java/ai/doctruth/internal/citation/RenderedCitationSection.java b/src/main/java/ai/doctruth/internal/citation/RenderedCitationSection.java new file mode 100644 index 00000000..61777b49 --- /dev/null +++ b/src/main/java/ai/doctruth/internal/citation/RenderedCitationSection.java @@ -0,0 +1,13 @@ +package ai.doctruth.internal.citation; + +import java.util.Optional; + +import ai.doctruth.BoundingBox; +import ai.doctruth.SourceLocation; + +record RenderedCitationSection( + String text, + SourceLocation location, + Optional boundingBox, + String sourceDocId, + String sourceUnitId) {} diff --git a/src/main/java/ai/doctruth/internal/schema/JsonSchemaResolver.java b/src/main/java/ai/doctruth/internal/schema/JsonSchemaResolver.java new file mode 100644 index 00000000..55ece1d8 --- /dev/null +++ b/src/main/java/ai/doctruth/internal/schema/JsonSchemaResolver.java @@ -0,0 +1,44 @@ +package ai.doctruth.internal.schema; + +import java.util.LinkedHashSet; +import java.util.Objects; +import java.util.Set; + +import com.fasterxml.jackson.databind.JsonNode; + +/** + * Resolves the local JSON Schema reference subset DocTruth accepts at extraction + * boundaries. + * + * @hidden + */ +public final class JsonSchemaResolver { + + private JsonSchemaResolver() { + throw new AssertionError("no instances"); + } + + public static JsonNode resolveLocal(JsonNode schema, JsonNode root) { + Objects.requireNonNull(schema, "schema"); + Objects.requireNonNull(root, "root"); + return resolveLocal(schema, root, new LinkedHashSet<>()); + } + + private static JsonNode resolveLocal(JsonNode schema, JsonNode root, Set seenRefs) { + if (!schema.has("$ref")) { + return schema; + } + String ref = schema.path("$ref").asText(); + if (!ref.startsWith("#/")) { + return schema; + } + if (!seenRefs.add(ref)) { + return schema; + } + JsonNode resolved = root.at(ref.substring(1)); + if (resolved.isMissingNode()) { + return schema; + } + return resolveLocal(resolved, root, seenRefs); + } +} diff --git a/src/test/java/ai/doctruth/EvidenceFirstJsonExtractionContractTest.java b/src/test/java/ai/doctruth/EvidenceFirstJsonExtractionContractTest.java new file mode 100644 index 00000000..4a2b0f25 --- /dev/null +++ b/src/test/java/ai/doctruth/EvidenceFirstJsonExtractionContractTest.java @@ -0,0 +1,202 @@ +package ai.doctruth; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; + +import com.fasterxml.jackson.databind.JsonNode; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +class EvidenceFirstJsonExtractionContractTest { + + private static final String CONTRACT_SCHEMA = """ + { + "title": "Contract", + "type": "object", + "properties": { + "partyA": { "type": "string" }, + "status": { "type": "string", "enum": ["draft", "signed"] }, + "totalValue": { "type": "number" } + }, + "required": ["partyA", "status"], + "additionalProperties": false + } + """; + + @Test + @DisplayName("evidence-first JSON wraps provider schema, unwraps values, and cites exact quotes") + void evidenceFirstJsonUsesProviderQuotesAsCitations() throws ExtractionException { + var captured = new AtomicReference(); + LlmProvider fake = new AnthropicProvider("test-key") { + @Override + public ProviderResponse complete(ProviderRequest req) { + captured.set(req.responseSchema()); + return response(""" + { + "partyA": { + "value": "Acme", + "exactQuote": "Party A: Acme" + }, + "status": { + "value": "signed", + "exactQuote": "status signed" + }, + "totalValue": { + "value": 1000, + "exactQuote": "value 1000" + } + } + """); + } + }; + + var result = DocTruth.from(fake) + .extractJson("extract contract", JsonSchema.from(CONTRACT_SCHEMA)) + .withEvidenceFirst() + .withProvenance() + .withConfidence() + .runJson(doc("Party A: Acme status signed value 1000")); + + assertThat(captured.get() + .path("properties") + .path("partyA") + .path("properties") + .has("value")) + .isTrue(); + assertThat(captured.get() + .path("properties") + .path("partyA") + .path("properties") + .has("exactQuote")) + .isTrue(); + assertThat(result.value().path("partyA").asText()).isEqualTo("Acme"); + assertThat(result.value().path("totalValue").asInt()).isEqualTo(1000); + assertThat(result.citations().get("partyA").exactQuote()).isEqualTo("Party A: Acme"); + assertThat(result.confidence().get("partyA").rationale()).contains("evidence quote"); + } + + @Test + @DisplayName("evidence-first JSON is schema-aware when user fields are named value and exactQuote") + void evidenceFirstJsonDoesNotCollapseBusinessFieldsNamedLikeEvidenceWrapper() throws ExtractionException { + String schema = """ + { + "type": "object", + "properties": { + "claim": { + "type": "object", + "properties": { + "value": { "type": "string" }, + "exactQuote": { "type": "string" } + }, + "required": ["value", "exactQuote"], + "additionalProperties": false + } + }, + "required": ["claim"], + "additionalProperties": false + } + """; + LlmProvider fake = new AnthropicProvider("test-key") { + @Override + public ProviderResponse complete(ProviderRequest req) { + return response(""" + { + "claim": { + "value": { + "value": "covered", + "exactQuote": "Claim value: covered" + }, + "exactQuote": { + "value": "Loss stated in policy", + "exactQuote": "Quoted policy sentence" + } + } + } + """); + } + }; + + var result = DocTruth.from(fake) + .extractJson("extract claim", JsonSchema.from(schema)) + .withEvidenceFirst() + .withProvenance() + .runJson(doc("Claim value: covered. Quoted policy sentence.")); + + assertThat(result.value().path("claim").path("value").asText()).isEqualTo("covered"); + assertThat(result.value().path("claim").path("exactQuote").asText()).isEqualTo("Loss stated in policy"); + assertThat(result.citations()).containsKeys("claim.value", "claim.exactQuote"); + } + + @Test + @DisplayName("evidence-first JSON expands local $defs refs before wrapping leaves") + void evidenceFirstJsonWrapsLocalRefLeaves() throws ExtractionException { + String schema = """ + { + "type": "object", + "$defs": { + "party": { + "type": "object", + "properties": { + "name": { "type": "string" } + }, + "required": ["name"], + "additionalProperties": false + } + }, + "properties": { + "party": { "$ref": "#/$defs/party" } + }, + "required": ["party"], + "additionalProperties": false + } + """; + var captured = new AtomicReference(); + LlmProvider fake = new AnthropicProvider("test-key") { + @Override + public ProviderResponse complete(ProviderRequest req) { + captured.set(req.responseSchema()); + return response(""" + { + "party": { + "name": { + "value": "Acme", + "exactQuote": "Party: Acme" + } + } + } + """); + } + }; + + var result = DocTruth.from(fake) + .extractJson("extract party", JsonSchema.from(schema)) + .withEvidenceFirst() + .withProvenance() + .runJson(doc("Party: Acme")); + + assertThat(captured.get() + .path("properties") + .path("party") + .path("properties") + .path("name") + .path("properties") + .has("value")) + .isTrue(); + assertThat(result.value().path("party").path("name").asText()).isEqualTo("Acme"); + assertThat(result.citations()).containsKey("party.name"); + } + + private static ParsedDocument doc(String text) { + var loc = new SourceLocation(1, 1, 1, 1, 0); + var section = new TextSection(text, loc); + var meta = new DocumentMetadata("contract.pdf", 1, Optional.empty()); + return new ParsedDocument("doc-json-schema", List.of(section), meta); + } + + private static ProviderResponse response(String rawJson) { + return new ProviderResponse(rawJson, new ProviderUsage(120, 18, "claude-sonnet-4-7-test")); + } +} diff --git a/src/test/java/ai/doctruth/EvidenceFirstJsonTest.java b/src/test/java/ai/doctruth/EvidenceFirstJsonTest.java index e5fecbb1..609e2a38 100644 --- a/src/test/java/ai/doctruth/EvidenceFirstJsonTest.java +++ b/src/test/java/ai/doctruth/EvidenceFirstJsonTest.java @@ -11,8 +11,7 @@ class EvidenceFirstJsonTest { @Test void responseSchemaWrapsObjectLeavesAndArrayItems() throws Exception { - var schema = MAPPER.readTree( - """ + var schema = MAPPER.readTree(""" { "type": "object", "properties": { @@ -33,8 +32,10 @@ void responseSchemaWrapsObjectLeavesAndArrayItems() throws Exception { var wrapped = EvidenceFirstJson.responseSchema(schema); - assertThat(wrapped.path("properties").path("party").path("properties").has("value")).isTrue(); - assertThat(wrapped.path("properties").path("party").path("properties").has("exactQuote")).isTrue(); + assertThat(wrapped.path("properties").path("party").path("properties").has("value")) + .isTrue(); + assertThat(wrapped.path("properties").path("party").path("properties").has("exactQuote")) + .isTrue(); assertThat(wrapped.path("properties") .path("items") .path("items") @@ -55,13 +56,17 @@ void responseSchemaWrapsNullMissingAndPrimitiveSchemasAsLeaves() throws Exceptio assertThat(nullWrapped.path("properties").path("value").isObject()).isTrue(); assertThat(missingWrapped.path("properties").path("value").isObject()).isTrue(); - assertThat(primitiveWrapped.path("properties").path("value").path("type").asText()).isEqualTo("boolean"); + assertThat(primitiveWrapped + .path("properties") + .path("value") + .path("type") + .asText()) + .isEqualTo("boolean"); } @Test void unwrapConvertsEvidenceLeavesInsideObjectsAndArrays() throws Exception { - var raw = MAPPER.readTree( - """ + var raw = MAPPER.readTree(""" { "party": { "value": "Acme", "exactQuote": "Party: Acme" }, "lines": [ @@ -81,13 +86,13 @@ void unwrapConvertsEvidenceLeavesInsideObjectsAndArrays() throws Exception { @Test void unwrapNullReturnsJsonNullAndNonEvidenceScalarIsCopied() throws Exception { assertThat(EvidenceFirstJson.unwrap(null).isNull()).isTrue(); - assertThat(EvidenceFirstJson.unwrap(MAPPER.readTree("\"plain\"")).asText()).isEqualTo("plain"); + assertThat(EvidenceFirstJson.unwrap(MAPPER.readTree("\"plain\"")).asText()) + .isEqualTo("plain"); } @Test void quoteMapCollectsNestedQuotesAndSkipsBlankQuotes() throws Exception { - var raw = MAPPER.readTree( - """ + var raw = MAPPER.readTree(""" { "party": { "value": "Acme", "exactQuote": "Party: Acme" }, "lines": [ diff --git a/src/test/java/ai/doctruth/JsonSchemaCitationContractTest.java b/src/test/java/ai/doctruth/JsonSchemaCitationContractTest.java new file mode 100644 index 00000000..888abe12 --- /dev/null +++ b/src/test/java/ai/doctruth/JsonSchemaCitationContractTest.java @@ -0,0 +1,118 @@ +package ai.doctruth; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +class JsonSchemaCitationContractTest { + + private static final String CONTRACT_SCHEMA = """ + { + "title": "Contract", + "type": "object", + "properties": { + "partyA": { "type": "string" }, + "status": { "type": "string", "enum": ["draft", "signed"] }, + "totalValue": { "type": "number" } + }, + "required": ["partyA", "status"], + "additionalProperties": false + } + """; + + private static final String VALID_JSON = """ + {"partyA":"Acme","status":"signed","totalValue":1000} + """; + + @Test + @DisplayName("requireCitation(field) retries when that JSON field cannot cite the source") + void requiredJsonFieldCitationIsRetried() throws ExtractionException { + var calls = new AtomicInteger(); + LlmProvider fake = new AnthropicProvider("test-key") { + @Override + public ProviderResponse complete(ProviderRequest req) { + int attempt = calls.incrementAndGet(); + if (attempt == 1) { + return response("{\"partyA\":\"Missing Co\",\"status\":\"signed\",\"totalValue\":1000}"); + } + return response(VALID_JSON); + } + }; + + var result = DocTruth.from(fake) + .extractJson("extract contract", JsonSchema.from(CONTRACT_SCHEMA)) + .requireCitation("partyA") + .withMaxRetries(1) + .runJson(doc("Acme signed contract value 1000")); + + assertThat(calls.get()).isEqualTo(2); + assertThat(result.citations()).containsKey("partyA"); + assertThat(result.citations()).doesNotContainKey("status"); + assertThat(result.citations().get("partyA").matchScore()).isEqualTo(1.0); + } + + @Test + @DisplayName("withProvenance and withConfidence return full JSON-field citations and confidence") + void provenanceAndConfidenceReturnAllJsonFieldEvidence() throws ExtractionException { + Instant publishedAt = Instant.parse("2026-01-01T00:00:00Z"); + LlmProvider fake = new AnthropicProvider("test-key") { + @Override + public ProviderResponse complete(ProviderRequest req) { + return response(VALID_JSON); + } + }; + + var result = DocTruth.from(fake) + .extractJson("extract contract", JsonSchema.from(CONTRACT_SCHEMA)) + .withProvenance() + .withBitemporal() + .withConfidence() + .withSourcePublishedAt(publishedAt) + .runJson(doc("Acme signed contract value 1000")); + + assertThat(result.citations()).containsKeys("partyA", "status", "totalValue"); + assertThat(result.confidence()).containsKeys("partyA", "status", "totalValue"); + assertThat(result.provenance().sourcePublishedAt()).contains(publishedAt); + } + + @Test + @DisplayName("builder invariants reject invalid retry and citation arguments") + void builderInvariants() { + var builder = DocTruth.from(new AnthropicProvider("test-key")) + .extractJson("extract contract", JsonSchema.from(CONTRACT_SCHEMA)); + + assertThatThrownBy(() -> builder.withMaxRetries(-1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("maxRetries"); + assertThatThrownBy(() -> builder.requireCitation(null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("fieldPath"); + assertThatThrownBy(() -> builder.requireCitation(" ")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("fieldPath"); + assertThatThrownBy(() -> builder.withSourcePublishedAt(null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("sourcePublishedAt"); + assertThatThrownBy(() -> builder.withContextStrategy(null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("contextStrategy"); + } + + private static ParsedDocument doc(String text) { + var loc = new SourceLocation(1, 1, 1, 1, 0); + var section = new TextSection(text, loc); + var meta = new DocumentMetadata("contract.pdf", 1, Optional.empty()); + return new ParsedDocument("doc-json-schema", List.of(section), meta); + } + + private static ProviderResponse response(String rawJson) { + return new ProviderResponse(rawJson, new ProviderUsage(120, 18, "claude-sonnet-4-7-test")); + } +} diff --git a/src/test/java/ai/doctruth/JsonSchemaExtractionContractTest.java b/src/test/java/ai/doctruth/JsonSchemaExtractionContractTest.java index 8e4e6beb..6a841c57 100644 --- a/src/test/java/ai/doctruth/JsonSchemaExtractionContractTest.java +++ b/src/test/java/ai/doctruth/JsonSchemaExtractionContractTest.java @@ -6,7 +6,6 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -276,127 +275,4 @@ public ProviderResponse complete(ProviderRequest req) { .contains("Return corrected JSON only"); } } - - @Nested - @DisplayName("citation requirements") - class CitationRequirements { - - @Test - @DisplayName("requireCitation(field) retries when that JSON field cannot cite the source") - void requiredJsonFieldCitationIsRetried() throws ExtractionException { - var calls = new AtomicInteger(); - LlmProvider fake = new AnthropicProvider("test-key") { - @Override - public ProviderResponse complete(ProviderRequest req) { - int attempt = calls.incrementAndGet(); - if (attempt == 1) { - return response("{\"partyA\":\"Missing Co\",\"status\":\"signed\",\"totalValue\":1000}"); - } - return response(VALID_JSON); - } - }; - - var result = DocTruth.from(fake) - .extractJson("extract contract", JsonSchema.from(CONTRACT_SCHEMA)) - .requireCitation("partyA") - .withMaxRetries(1) - .runJson(doc("Acme signed contract value 1000")); - - assertThat(calls.get()).isEqualTo(2); - assertThat(result.citations()).containsKey("partyA"); - assertThat(result.citations()).doesNotContainKey("status"); - assertThat(result.citations().get("partyA").matchScore()).isEqualTo(1.0); - } - - @Test - @DisplayName("withProvenance and withConfidence return full JSON-field citations and confidence") - void provenanceAndConfidenceReturnAllJsonFieldEvidence() throws ExtractionException { - Instant publishedAt = Instant.parse("2026-01-01T00:00:00Z"); - LlmProvider fake = new AnthropicProvider("test-key") { - @Override - public ProviderResponse complete(ProviderRequest req) { - return response(VALID_JSON); - } - }; - - var result = DocTruth.from(fake) - .extractJson("extract contract", JsonSchema.from(CONTRACT_SCHEMA)) - .withProvenance() - .withBitemporal() - .withConfidence() - .withSourcePublishedAt(publishedAt) - .runJson(doc("Acme signed contract value 1000")); - - assertThat(result.citations()).containsKeys("partyA", "status", "totalValue"); - assertThat(result.confidence()).containsKeys("partyA", "status", "totalValue"); - assertThat(result.provenance().sourcePublishedAt()).contains(publishedAt); - } - - @Test - @DisplayName("evidence-first JSON wraps provider schema, unwraps values, and cites exact quotes") - void evidenceFirstJsonUsesProviderQuotesAsCitations() throws ExtractionException { - var captured = new AtomicReference(); - LlmProvider fake = new AnthropicProvider("test-key") { - @Override - public ProviderResponse complete(ProviderRequest req) { - captured.set(req.responseSchema()); - return response(""" - { - "partyA": { - "value": "Acme", - "exactQuote": "Party A: Acme" - }, - "status": { - "value": "signed", - "exactQuote": "status signed" - }, - "totalValue": { - "value": 1000, - "exactQuote": "value 1000" - } - } - """); - } - }; - - var result = DocTruth.from(fake) - .extractJson("extract contract", JsonSchema.from(CONTRACT_SCHEMA)) - .withEvidenceFirst() - .withProvenance() - .withConfidence() - .runJson(doc("Party A: Acme status signed value 1000")); - - assertThat(captured.get().path("properties").path("partyA").path("properties").has("value")) - .isTrue(); - assertThat(captured.get().path("properties").path("partyA").path("properties").has("exactQuote")) - .isTrue(); - assertThat(result.value().path("partyA").asText()).isEqualTo("Acme"); - assertThat(result.value().path("totalValue").asInt()).isEqualTo(1000); - assertThat(result.citations().get("partyA").exactQuote()).isEqualTo("Party A: Acme"); - assertThat(result.confidence().get("partyA").rationale()).contains("evidence quote"); - } - - @Test - @DisplayName("builder invariants reject invalid retry and citation arguments") - void builderInvariants() { - var builder = DocTruth.from(new AnthropicProvider("test-key")) - .extractJson("extract contract", JsonSchema.from(CONTRACT_SCHEMA)); - - assertThatThrownBy(() -> builder.withMaxRetries(-1)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("maxRetries"); - assertThatThrownBy(() -> builder.requireCitation(null)) - .isInstanceOf(NullPointerException.class) - .hasMessageContaining("fieldPath"); - assertThatThrownBy(() -> builder.requireCitation(" ")) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("fieldPath"); - assertThatThrownBy(() -> builder.withSourcePublishedAt(null)) - .isInstanceOf(NullPointerException.class) - .hasMessageContaining("sourcePublishedAt"); - assertThatThrownBy(() -> builder.withContextStrategy(null)) - .isInstanceOf(NullPointerException.class) - .hasMessageContaining("contextStrategy"); - } - } } diff --git a/src/test/java/ai/doctruth/OpenDataLoaderPdfDocumentParserTest.java b/src/test/java/ai/doctruth/OpenDataLoaderPdfDocumentParserTest.java index 0bfb6092..fe266581 100644 --- a/src/test/java/ai/doctruth/OpenDataLoaderPdfDocumentParserTest.java +++ b/src/test/java/ai/doctruth/OpenDataLoaderPdfDocumentParserTest.java @@ -9,6 +9,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDPage; @@ -83,8 +85,8 @@ void streamsOpenDataLoaderJsonIntoParsedSections() throws Exception { ] } """); - var geometry = new OpenDataLoaderPdfGeometry( - Map.of(1, new OpenDataLoaderPdfGeometry.PageGeometry(200.0, 400.0))); + var geometry = + new OpenDataLoaderPdfGeometry(Map.of(1, new OpenDataLoaderPdfGeometry.PageGeometry(200.0, 400.0))); var parsed = OpenDataLoaderPdfDocumentParser.readOpenDataLoaderJson(json, geometry); @@ -190,6 +192,49 @@ void serializesOpenDataLoaderProcessLifecycle() throws Exception { assertThat(maxActiveRuns).hasValue(1); } + @Test + void parseDoesNotMutateRootJulLogger() throws Exception { + Path pdf = tempDir.resolve("logger.pdf"); + Files.writeString(pdf, "fake pdf bytes"); + Logger root = Logger.getLogger(""); + Level previousRootLevel = root.getLevel(); + var handlers = root.getHandlers(); + var previousHandlerLevels = new Level[handlers.length]; + for (int i = 0; i < handlers.length; i++) { + previousHandlerLevels[i] = handlers[i].getLevel(); + } + OpenDataLoaderPdfDocumentParser.OpenDataLoaderRunner runner = (pdfPath, outputDir, config) -> { + assertThat(root.getLevel()).isEqualTo(previousRootLevel); + for (int i = 0; i < handlers.length; i++) { + assertThat(handlers[i].getLevel()).isEqualTo(previousHandlerLevels[i]); + } + String outputName = pdfPath.getFileName().toString().replaceFirst("\\.[^.]+$", "") + ".json"; + Files.writeString(outputDir.resolve(outputName), """ + { + "number of pages": 1, + "kids": [ + { "type": "heading", "page number": 1, "content": "logger safe" } + ] + } + """); + }; + + try (var runnerOverride = OpenDataLoaderPdfDocumentParser.useOpenDataLoaderRunnerForTesting(runner)) { + assertThat(runnerOverride).isNotNull(); + OpenDataLoaderPdfDocumentParser.parse(pdf); + } finally { + root.setLevel(previousRootLevel); + for (int i = 0; i < handlers.length; i++) { + handlers[i].setLevel(previousHandlerLevels[i]); + } + } + + assertThat(root.getLevel()).isEqualTo(previousRootLevel); + for (int i = 0; i < handlers.length; i++) { + assertThat(handlers[i].getLevel()).isEqualTo(previousHandlerLevels[i]); + } + } + private static void sleepBriefly() throws java.io.IOException { try { Thread.sleep(100); diff --git a/src/test/java/ai/doctruth/OpenDataLoaderSectionMapperTest.java b/src/test/java/ai/doctruth/OpenDataLoaderSectionMapperTest.java index 347bebee..2c52d20d 100644 --- a/src/test/java/ai/doctruth/OpenDataLoaderSectionMapperTest.java +++ b/src/test/java/ai/doctruth/OpenDataLoaderSectionMapperTest.java @@ -169,8 +169,7 @@ void loadsGeometryOnlyWhenBoundingBoxesArePresent() throws Exception { var loads = new AtomicInteger(); var geometry = OpenDataLoaderPdfGeometry.lazy(() -> { loads.incrementAndGet(); - return new OpenDataLoaderPdfGeometry( - Map.of(1, new OpenDataLoaderPdfGeometry.PageGeometry(200.0, 400.0))); + return new OpenDataLoaderPdfGeometry(Map.of(1, new OpenDataLoaderPdfGeometry.PageGeometry(200.0, 400.0))); }); new OpenDataLoaderSectionMapper(geometry).map(MAPPER.readTree(""" diff --git a/src/test/java/ai/doctruth/TrustDocumentParserTest.java b/src/test/java/ai/doctruth/TrustDocumentParserTest.java index c0639c0e..d8335964 100644 --- a/src/test/java/ai/doctruth/TrustDocumentParserTest.java +++ b/src/test/java/ai/doctruth/TrustDocumentParserTest.java @@ -3,8 +3,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import java.lang.reflect.InvocationTargetException; import java.io.StringWriter; +import java.lang.reflect.InvocationTargetException; import java.nio.file.Path; import java.util.List; import java.util.Optional; @@ -80,10 +80,7 @@ void jsonCanStreamDirectlyFromParsedDocument() throws Exception { "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", List.of( new TextSection( - "Intro", - location, - BlockKind.BODY, - Optional.of(new BoundingBox(10, 20, 30, 40))), + "Intro", location, BlockKind.BODY, Optional.of(new BoundingBox(10, 20, 30, 40))), new TableSection(List.of(List.of("A", "B")), location), new FigureSection("Chart", location)), metadata); @@ -104,8 +101,8 @@ void fromParsedUsesExplicitSourcePathForSourceFilename() throws Exception { List.of(new TextSection("Intro", location)), metadata); - TrustDocument doc = - TrustDocument.fromParsed(parsed, Path.of("/documents/customer-contract.pdf"), PdfParserBackend.OPENDATALOADER); + TrustDocument doc = TrustDocument.fromParsed( + parsed, Path.of("/documents/customer-contract.pdf"), PdfParserBackend.OPENDATALOADER); String streamed = TrustDocumentJson.toJson( parsed, Path.of("/documents/customer-contract.pdf"), PdfParserBackend.OPENDATALOADER); @@ -124,7 +121,8 @@ void writerOverloadsMatchStringJson() throws Exception { metadata); var doc = TrustDocument.fromParsed(parsed, Path.of("contract.pdf"), PdfParserBackend.OPENDATALOADER); var materializedDoc = TrustDocumentJson.toJson(doc); - var materializedParsed = TrustDocumentJson.toJson(parsed, Path.of("contract.pdf"), PdfParserBackend.OPENDATALOADER); + var materializedParsed = + TrustDocumentJson.toJson(parsed, Path.of("contract.pdf"), PdfParserBackend.OPENDATALOADER); var docWriter = new StringWriter(); var parsedWriter = new StringWriter(); diff --git a/src/test/java/ai/doctruth/cli/DocTruthCliExtractContractTest.java b/src/test/java/ai/doctruth/cli/DocTruthCliExtractContractTest.java index 1cfe3db9..ff81a73c 100644 --- a/src/test/java/ai/doctruth/cli/DocTruthCliExtractContractTest.java +++ b/src/test/java/ai/doctruth/cli/DocTruthCliExtractContractTest.java @@ -46,13 +46,18 @@ void extractWritesTrustDocumentLinkedRunManifest() throws Exception { var trust = MAPPER.readTree(Files.readString(out.resolve("trust-document.json"))); var audit = MAPPER.readTree(Files.readString(out.resolve("audit.json"))); var firstDerivation = audit.path("prov:wasDerivedFrom").get(0); - assertThat(firstDerivation.path("doctruth:sourceDocId").asText()).isEqualTo(trust.path("docId").asText()); + assertThat(firstDerivation.path("doctruth:sourceDocId").asText()) + .isEqualTo(trust.path("docId").asText()); assertThat(firstDerivation.path("doctruth:sourceUnitId").asText()).isNotBlank(); assertThat(trust.path("units")) .anySatisfy(unit -> assertThat(unit.path("id").asText()) .isEqualTo(firstDerivation.path("doctruth:sourceUnitId").asText())); var manifest = MAPPER.readTree(Files.readString(out.resolve("manifest.json"))); assertThat(manifest.path("schemaVersion").asText()).isEqualTo("doctruth.extract-run.v1"); + assertThat(manifest.has("source")).isFalse(); + assertThat(manifest.path("sourceFilename").asText()) + .isEqualTo(pdf.getFileName().toString()); + assertThat(manifest.path("sourceSha256").asText()).isNotBlank(); assertThat(manifest.path("artifacts").path("trustDocument").asText()).isEqualTo("trust-document.json"); } @@ -68,7 +73,24 @@ void extractRequiresNestedSchemaLeafCitationsByDefault() throws Exception { assertThat(code).isZero(); var audit = MAPPER.readTree(Files.readString(out.resolve("audit.json"))); assertThat(audit.path("prov:wasDerivedFrom")) - .anySatisfy(entry -> assertThat(entry.path("doctruth:fieldPath").asText()).isEqualTo("party.name")); + .anySatisfy(entry -> + assertThat(entry.path("doctruth:fieldPath").asText()).isEqualTo("party.name")); + } + + @Test + void extractRequiresArrayItemSchemaLeafCitationsByDefault() throws Exception { + Path pdf = samplePdf(); + Path schema = arraySchemaFile(); + Path out = tempDir.resolve("array-run"); + var cli = cliWithProvider(providerReturning("{\"items\":[{\"name\":\"Premium Support Plan\"}]}")); + + int code = cli.run(new String[] {"extract", pdf.toString(), "-s", schema.toString(), "-o", out.toString()}); + + assertThat(code).isZero(); + var audit = MAPPER.readTree(Files.readString(out.resolve("audit.json"))); + assertThat(audit.path("prov:wasDerivedFrom")) + .anySatisfy(entry -> + assertThat(entry.path("doctruth:fieldPath").asText()).isEqualTo("items[0].name")); } @Test @@ -120,12 +142,19 @@ private static LlmProvider evidenceFirstProvider() { return new OpenAiProvider("test", URI.create("http://localhost"), "test-model") { @Override public ProviderResponse complete(ProviderRequest request) { - assertThat(request.responseSchema().path("properties").path("partyA").path("properties").has("value")) + assertThat(request.responseSchema() + .path("properties") + .path("partyA") + .path("properties") + .has("value")) .isTrue(); - assertThat(request.responseSchema().path("properties").path("partyA").path("properties").has("exactQuote")) + assertThat(request.responseSchema() + .path("properties") + .path("partyA") + .path("properties") + .has("exactQuote")) .isTrue(); - return new ProviderResponse( - """ + return new ProviderResponse(""" { "partyA": { "value": "Acme Industrial Materials Pty Ltd", @@ -136,8 +165,7 @@ public ProviderResponse complete(ProviderRequest request) { "exactQuote": "Total Value: AUD 2,450,000" } } - """, - new ProviderUsage(1, 1, "test-model")); + """, new ProviderUsage(1, 1, "test-model")); } }; } @@ -179,6 +207,31 @@ private Path nestedSchemaFile() throws IOException { return schema; } + private Path arraySchemaFile() throws IOException { + Path schema = tempDir.resolve("array.schema.json"); + Files.writeString(schema, """ + { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { "type": "string" } + }, + "required": ["name"], + "additionalProperties": false + } + } + }, + "required": ["items"], + "additionalProperties": false + } + """); + return schema; + } + private Path samplePdf() throws IOException { Path path = tempDir.resolve("contract.pdf"); try (var pdf = new PDDocument()) { @@ -191,6 +244,8 @@ private Path samplePdf() throws IOException { cs.showText("Party A: Acme Industrial Materials Pty Ltd"); cs.newLineAtOffset(0, -18); cs.showText("Total Value: AUD 2,450,000"); + cs.newLineAtOffset(0, -18); + cs.showText("Line item: Premium Support Plan"); cs.endText(); } pdf.save(path.toFile()); diff --git a/src/test/java/ai/doctruth/cli/DocTruthCliTrustDocumentTest.java b/src/test/java/ai/doctruth/cli/DocTruthCliTrustDocumentTest.java index 923f4331..ca6ee454 100644 --- a/src/test/java/ai/doctruth/cli/DocTruthCliTrustDocumentTest.java +++ b/src/test/java/ai/doctruth/cli/DocTruthCliTrustDocumentTest.java @@ -42,11 +42,13 @@ void parseFormatJsonWritesTrustDocument() throws Exception { assertThat(code).isZero(); var tree = MAPPER.readTree(Files.readString(out)); assertThat(tree.path("schemaVersion").asText()).isEqualTo("doctruth.trust-document.v1"); - assertThat(tree.path("source").path("filename").asText()).isEqualTo(pdf.getFileName().toString()); + assertThat(tree.path("source").path("filename").asText()) + .isEqualTo(pdf.getFileName().toString()); assertThat(tree.path("source").path("sha256").asText()).hasSize(64); assertThat(tree.path("parserRun").path("backend").asText()).isEqualTo("opendataloader"); assertThat(tree.path("units")).isNotEmpty(); - assertThat(tree.path("units").get(0).path("location").path("pageStart").asInt()).isEqualTo(1); + assertThat(tree.path("units").get(0).path("location").path("pageStart").asInt()) + .isEqualTo(1); } @Test @@ -71,7 +73,8 @@ void parseFormatParsedJsonPrintsCompatibilityJson() throws Exception { assertThat(code).isZero(); var tree = MAPPER.readTree(cli.out()); - assertThat(tree.path("metadata").path("sourceFilename").asText()).isEqualTo(pdf.getFileName().toString()); + assertThat(tree.path("metadata").path("sourceFilename").asText()) + .isEqualTo(pdf.getFileName().toString()); assertThat(tree.path("sections")).isNotEmpty(); } @@ -100,9 +103,9 @@ void profileCanIncludeTrustJsonOutputCost() throws Exception { Path pdf = samplePdf(); var cli = cli(); - int code = cli.run(new String[] { - "profile", pdf.toString(), "--iterations", "1", "--include-output", "trust-json", "--json" - }); + int code = cli.run( + new String[] {"profile", pdf.toString(), "--iterations", "1", "--include-output", "trust-json", "--json" + }); assertThat(code).isZero(); var tree = MAPPER.readTree(cli.out()); @@ -181,9 +184,8 @@ void profilePrintsOutputTimingSummaryWhenIncludingOutput() throws Exception { Path pdf = samplePdf(); var cli = cli(); - int code = cli.run(new String[] { - "profile", pdf.toString(), "--iterations", "1", "--include-output", "trust-json" - }); + int code = cli.run( + new String[] {"profile", pdf.toString(), "--iterations", "1", "--include-output", "trust-json"}); assertThat(code).isZero(); assertThat(cli.out()) diff --git a/src/test/java/ai/doctruth/cli/ParseCommandPerformanceTest.java b/src/test/java/ai/doctruth/cli/ParseCommandPerformanceTest.java index 547a2790..4853a434 100644 --- a/src/test/java/ai/doctruth/cli/ParseCommandPerformanceTest.java +++ b/src/test/java/ai/doctruth/cli/ParseCommandPerformanceTest.java @@ -72,32 +72,24 @@ void jsonFileOutputWritesWithoutStringMaterialization() throws Exception { opts -> { throw new AssertionError("provider is not used by parse"); }); - var command = new ParseCommand( - context, - (path, backend) -> parsed(path), - new ParseCommand.JsonRenderer() { - @Override - public String render(ParsedDocument doc, Path source, PdfParserBackend parser, boolean trustDocument) { - renderCalls.incrementAndGet(); - return "{}"; - } - - @Override - public void write( - ParsedDocument doc, - Path source, - PdfParserBackend parser, - boolean trustDocument, - Path out) - throws CliException { - writeCalls.incrementAndGet(); - try { - Files.writeString(out, "{}"); - } catch (java.io.IOException e) { - throw new CliException("write failed", e); - } - } - }); + var command = new ParseCommand(context, (path, backend) -> parsed(path), new ParseCommand.JsonRenderer() { + @Override + public String render(ParsedDocument doc, Path source, PdfParserBackend parser, boolean trustDocument) { + renderCalls.incrementAndGet(); + return "{}"; + } + + @Override + public void write(ParsedDocument doc, Path source, PdfParserBackend parser, boolean trustDocument, Path out) + throws CliException { + writeCalls.incrementAndGet(); + try { + Files.writeString(out, "{}"); + } catch (java.io.IOException e) { + throw new CliException("write failed", e); + } + } + }); command.run(new String[] {"parse", "contract.pdf", "--format", "json", "-o", output.toString()}); @@ -121,8 +113,7 @@ public String render(ParsedDocument doc, Path source, PdfParserBackend parser, b } @Override - public void write( - ParsedDocument doc, Path source, PdfParserBackend parser, boolean trustDocument, Path out) + public void write(ParsedDocument doc, Path source, PdfParserBackend parser, boolean trustDocument, Path out) throws CliException { writeCalls.incrementAndGet(); try { diff --git a/src/test/java/ai/doctruth/internal/schema/JsonSchemaResolverTest.java b/src/test/java/ai/doctruth/internal/schema/JsonSchemaResolverTest.java new file mode 100644 index 00000000..781f1235 --- /dev/null +++ b/src/test/java/ai/doctruth/internal/schema/JsonSchemaResolverTest.java @@ -0,0 +1,67 @@ +package ai.doctruth.internal.schema; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +class JsonSchemaResolverTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Test + void resolvesLocalRefsRecursively() throws Exception { + var root = MAPPER.readTree(""" + { + "$defs": { + "alias": { "$ref": "#/$defs/party" }, + "party": { + "type": "object", + "properties": { + "name": { "type": "string" } + } + } + }, + "properties": { + "party": { "$ref": "#/$defs/alias" } + } + } + """); + + var resolved = JsonSchemaResolver.resolveLocal(root.at("/properties/party"), root); + + assertThat(resolved.path("type").asText()).isEqualTo("object"); + assertThat(resolved.path("properties").has("name")).isTrue(); + } + + @Test + void returnsOriginalSchemaForRefsOutsideSupportedLocalSubset() throws Exception { + var root = MAPPER.readTree(""" + { + "$defs": { + "cycle": { "$ref": "#/$defs/cycle" } + }, + "remote": { "$ref": "https://example.invalid/schema.json" }, + "missing": { "$ref": "#/$defs/missing" } + } + """); + + assertThat(JsonSchemaResolver.resolveLocal(root.path("remote"), root)).isSameAs(root.path("remote")); + assertThat(JsonSchemaResolver.resolveLocal(root.path("missing"), root)).isSameAs(root.path("missing")); + assertThat(JsonSchemaResolver.resolveLocal(root.at("/$defs/cycle"), root)) + .isSameAs(root.at("/$defs/cycle")); + } + + @Test + void rejectsNullInputs() throws Exception { + var schema = MAPPER.readTree("{}"); + + assertThatThrownBy(() -> JsonSchemaResolver.resolveLocal(null, schema)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("schema"); + assertThatThrownBy(() -> JsonSchemaResolver.resolveLocal(schema, null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("root"); + } +}