From 9749955f7c18a6c3c28c71c317a0b557fe440fc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Sou=C5=A1ek?= <15769039+sousekd@users.noreply.github.com> Date: Thu, 11 Jun 2026 19:59:50 +0200 Subject: [PATCH 1/2] Add content transformers extension family: mdream, transform step, clean-deterministic pipeline - New ContentTransformer contract (supports/transform) with registry and engine wiring, as a sixth built-in category - Built-in mdream transformer using @mdream/js (pure-JS HTML -> markdown) with minimal and clean knobs - Generic transform pipeline step with target gate and diagnostics passthrough - New firecrawl-html source provider (Firecrawl rawHtml, no onlyMainContent or stripBase64Images -- those are no-ops with rawHtml) - New clean-deterministic pipeline: load-source(firecrawl-html) -> transform(mdream) -> truncate (no LLM) - Naming overhaul: all providers and LLM registries use {type}-{variant} convention (http-default, firecrawl-markdown, firecrawl-html, docling-default, mdream-default, llm-default) - Updated inspect-suspects.ps1 to fetch rawHtml from Firecrawl as the universal comparison baseline - Updated all docs for new naming and features - .env.example and compose files: added MDREAM_MINIMAL/MDREAM_CLEAN passthrough with true defaults; updated SOURCE_PROVIDER defaults --- .env.example | 21 ++- README.md | 5 +- compose.deploy.yaml | 6 +- compose.yaml | 6 +- config/llm-context-loader.yaml | 55 +++++- docs/ARCHITECTURE.md | 11 +- docs/CONFIGURATION.md | 29 +-- docs/CUSTOMIZATION.md | 75 +++++++- docs/ROADMAP.md | 2 +- docs/agents/architecture-rules.md | 2 +- docs/agents/extension-authoring.md | 8 +- package-lock.json | 29 +++ package.json | 1 + scripts/inspect-suspects.ps1 | 8 +- .../mdream/mdream-transformer-config.ts | 27 +++ .../mdream/mdream-transformer-descriptor.ts | 20 +++ .../mdream/mdream-transformer.ts | 73 ++++++++ .../transform/transform-step-config.ts | 35 ++++ .../transform/transform-step-descriptor.ts | 32 ++++ .../transform/transform-step.ts | 76 ++++++++ src/bundles/default-engine-descriptors.ts | 6 + src/config/yaml/yaml-app-config.ts | 1 + src/config/yaml/yaml-config.ts | 1 + .../extensions/content-transformer.ts | 79 +++++++++ .../extensions/resolved-extension.ts | 8 + src/engine/create-engine.ts | 17 +- src/engine/engine-config.ts | 1 + src/engine/engine-descriptors.ts | 2 + src/engine/engine-runtime.ts | 2 + .../builders/build-content-transformers.ts | 44 +++++ src/shared/media-types.ts | 6 + tests/app/compose.test.ts | 22 +-- tests/architecture/import-boundaries.test.ts | 21 ++- .../mdream/mdream-transformer-config.test.ts | 18 ++ .../mdream/mdream-transformer.test.ts | 127 ++++++++++++++ .../llm-pass/llm-pass-step-config.test.ts | 2 +- .../transform/transform-step-config.test.ts | 33 ++++ .../transform/transform-step.test.ts | 165 ++++++++++++++++++ .../default-engine-descriptors.test.ts | 57 +++++- tests/engine/activation-lazy.test.ts | 4 + tests/engine/activation-pipeline.test.ts | 4 + tests/engine/create-engine.test.ts | 12 +- 42 files changed, 1085 insertions(+), 68 deletions(-) create mode 100644 src/builtins/content-transformers/mdream/mdream-transformer-config.ts create mode 100644 src/builtins/content-transformers/mdream/mdream-transformer-descriptor.ts create mode 100644 src/builtins/content-transformers/mdream/mdream-transformer.ts create mode 100644 src/builtins/pipeline-steps/transform/transform-step-config.ts create mode 100644 src/builtins/pipeline-steps/transform/transform-step-descriptor.ts create mode 100644 src/builtins/pipeline-steps/transform/transform-step.ts create mode 100644 src/contracts/extensions/content-transformer.ts create mode 100644 src/engine/internal/builders/build-content-transformers.ts create mode 100644 tests/builtins/content-transformers/mdream/mdream-transformer-config.test.ts create mode 100644 tests/builtins/content-transformers/mdream/mdream-transformer.test.ts create mode 100644 tests/builtins/pipeline-steps/transform/transform-step-config.test.ts create mode 100644 tests/builtins/pipeline-steps/transform/transform-step.test.ts diff --git a/.env.example b/.env.example index 8931826..99aa713 100644 --- a/.env.example +++ b/.env.example @@ -22,7 +22,7 @@ LOG_LEVEL=info LOG_PRETTY=auto # --- Pipeline ------------------------------------------------------------ -# Active pipeline selected by HTTP adapters. Default: truncate (load-source + truncate) +# Active pipeline selected by HTTP adapters: truncate, clean-deterministic, or clean-llm. Default: truncate (load-source + truncate) DEFAULT_PIPELINE=truncate # Maximum concurrent source-loading groups. Default: 1 @@ -41,29 +41,36 @@ DEBUG_XML_INCLUDE_SKIPPED=false OUTPUT_TARGET_CHARS=35000 # --- Source provider ----------------------------------------------------- -# Active source provider name used by pipelines (truncate and clean-llm). Default: default-http (native HTTP fetch, no external service needed) -SOURCE_PROVIDER=default-http +# Active source provider name used by pipelines (truncate, clean-deterministic, clean-llm). Default: http-default (native HTTP fetch, no external service needed) +SOURCE_PROVIDER=http-default -# Firecrawl base URL. Required only when a referenced provider uses it (e.g. default-firecrawl). +# Firecrawl base URL. Required only when a referenced provider uses it (e.g. firecrawl-markdown, firecrawl-html). FIRECRAWL_BASE_URL=https://firecrawl.example # Optional Firecrawl bearer token. Default: empty (no token) FIRECRAWL_API_KEY= -# Docling Serve base URL. Required only when a referenced provider uses it (e.g. default-docling). +# Docling Serve base URL. Required only when a referenced provider uses it (e.g. docling-default). DOCLING_BASE_URL=https://docling.example # Optional Docling API key (X-Api-Key header). Default: empty (no token) DOCLING_API_KEY= +# --- Content transformer ------------------------------------------------- +# Isolate main content and filter boilerplate before conversion. Default: true +MDREAM_MINIMAL=true + +# Post-conversion link and whitespace cleanup. Default: true +MDREAM_CLEAN=true + # --- LLM provider -------------------------------------------------------- -# OpenAI-compatible Chat Completions /v1 base URL. Required only when a referenced provider uses it (e.g. default-llm). +# OpenAI-compatible Chat Completions /v1 base URL. Required only when a referenced provider uses it (e.g. llm-default). LLM_BASE_URL=https://openai-compatible.example/v1 # Optional LLM bearer token. Default: empty (no token) LLM_API_KEY= -# Model identifier sent to the chat-completions endpoint. Required only when a referenced provider uses it (e.g. default-llm). +# Model identifier sent to the chat-completions endpoint. Required only when a referenced provider uses it (e.g. llm-default). LLM_MODEL=model-name # Model context window in tokens. Default: empty (disables the context-fit gate) diff --git a/README.md b/README.md index ebf15cb..f6b7bfe 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,10 @@ LLM Context Loader is a small HTTP service that turns URLs into markdown for LLM ## Current Shape -- **Pipelines:** `truncate` (default, no LLM) and `clean-llm` (Firecrawl + LLM), selected via `DEFAULT_PIPELINE`. +- **Pipelines:** `truncate` (default, no LLM), `clean-deterministic` (Firecrawl HTML + mdream, no LLM), and `clean-llm` (Firecrawl + LLM), selected via `DEFAULT_PIPELINE`. - **Source providers:** native HTTP fetch, Firecrawl, Docling. -- **LLM provider:** OpenAI-compatible `/chat/completions`. +- **Content transformers:** `mdream` (HTML to markdown). +- **LLM providers:** OpenAI-compatible `/chat/completions`. - **Output renderers:** `debug-xml` and `passthrough`, selected per pipeline in YAML. ## Quick Start diff --git a/compose.deploy.yaml b/compose.deploy.yaml index 8ea88b6..48e8d5f 100644 --- a/compose.deploy.yaml +++ b/compose.deploy.yaml @@ -21,12 +21,16 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l OUTPUT_TARGET_CHARS: "${OUTPUT_TARGET_CHARS:-25000}" # Source provider. - SOURCE_PROVIDER: "${SOURCE_PROVIDER:-default-http}" + SOURCE_PROVIDER: "${SOURCE_PROVIDER:-http-default}" FIRECRAWL_BASE_URL: "${FIRECRAWL_BASE_URL:-}" FIRECRAWL_API_KEY: "${FIRECRAWL_API_KEY:-}" DOCLING_BASE_URL: "${DOCLING_BASE_URL:-}" DOCLING_API_KEY: "${DOCLING_API_KEY:-}" + # Content transformer. + MDREAM_MINIMAL: "${MDREAM_MINIMAL:-true}" + MDREAM_CLEAN: "${MDREAM_CLEAN:-true}" + # LLM provider. LLM_BASE_URL: "${LLM_BASE_URL:-}" LLM_API_KEY: "${LLM_API_KEY:-}" diff --git a/compose.yaml b/compose.yaml index 7dbb7c6..b98c586 100644 --- a/compose.yaml +++ b/compose.yaml @@ -21,12 +21,16 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l OUTPUT_TARGET_CHARS: "${OUTPUT_TARGET_CHARS:-25000}" # Source provider. - SOURCE_PROVIDER: "${SOURCE_PROVIDER:-default-http}" + SOURCE_PROVIDER: "${SOURCE_PROVIDER:-http-default}" FIRECRAWL_BASE_URL: "${FIRECRAWL_BASE_URL:-}" FIRECRAWL_API_KEY: "${FIRECRAWL_API_KEY:-}" DOCLING_BASE_URL: "${DOCLING_BASE_URL:-}" DOCLING_API_KEY: "${DOCLING_API_KEY:-}" + # Content transformer. + MDREAM_MINIMAL: "${MDREAM_MINIMAL:-true}" + MDREAM_CLEAN: "${MDREAM_CLEAN:-true}" + # LLM provider. LLM_BASE_URL: "${LLM_BASE_URL:-}" LLM_API_KEY: "${LLM_API_KEY:-}" diff --git a/config/llm-context-loader.yaml b/config/llm-context-loader.yaml index 2de37f9..58b1a65 100644 --- a/config/llm-context-loader.yaml +++ b/config/llm-context-loader.yaml @@ -28,13 +28,20 @@ outputRenderers: # Source providers turn an input URL into source markdown. sourceProviders: - default-http: + http-default: type: http config: userAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" maxBytes: 5000000 titleFromHtml: true - default-firecrawl: + firecrawl-html: + type: firecrawl + config: + baseUrl: ${FIRECRAWL_BASE_URL:-} + apiKey: ${FIRECRAWL_API_KEY:-} + output: rawHtml + parsePdf: true + firecrawl-markdown: type: firecrawl config: baseUrl: ${FIRECRAWL_BASE_URL:-} @@ -43,7 +50,7 @@ sourceProviders: onlyMainContent: true stripBase64Images: true parsePdf: true - default-docling: + docling-default: type: docling config: baseUrl: ${DOCLING_BASE_URL:-} @@ -52,9 +59,17 @@ sourceProviders: doOcr: true tableMode: accurate +# Content transformers rewrite the body between representations (e.g. HTML to markdown). +contentTransformers: + mdream-default: + type: mdream + config: + minimal: ${MDREAM_MINIMAL:-true} + clean: ${MDREAM_CLEAN:-true} + # LLM providers run prompt-rendered chat-completions passes. llmProviders: - default-llm: + llm-default: type: openai-chat config: baseUrl: ${LLM_BASE_URL:-} @@ -77,7 +92,31 @@ pipelines: concurrencyGroup: source timeoutSeconds: ${LOAD_SOURCE_TIMEOUT_SECONDS:-20} config: - provider: ${SOURCE_PROVIDER:-default-http} + provider: ${SOURCE_PROVIDER:-http-default} + - type: truncate + name: truncate + config: + targetChars: ${OUTPUT_TARGET_CHARS:-25000} + + # Deterministic HTML-to-markdown path: fetch HTML, convert with mdream, then truncate. + clean-deterministic: + outputRenderer: ${DEFAULT_OUTPUT_RENDERER:-debug-xml} + limiters: + source: ${SOURCE_CONCURRENCY:-1} + steps: + - type: load-source + name: fetch + concurrencyGroup: source + timeoutSeconds: ${LOAD_SOURCE_TIMEOUT_SECONDS:-20} + config: + provider: ${SOURCE_PROVIDER:-firecrawl-html} + - type: transform + name: html_to_markdown + config: + transformer: mdream-default + target: text/markdown + onUnsupported: skip + emitDiagnostics: true - type: truncate name: truncate config: @@ -94,7 +133,7 @@ pipelines: concurrencyGroup: source timeoutSeconds: ${LOAD_SOURCE_TIMEOUT_SECONDS:-20} config: - provider: ${SOURCE_PROVIDER:-default-firecrawl} + provider: ${SOURCE_PROVIDER:-firecrawl-markdown} - type: capture-urls name: capture_source_urls concurrencyGroup: source @@ -105,7 +144,7 @@ pipelines: concurrencyGroup: llm timeoutSeconds: ${CLEAN_TIMEOUT_SECONDS:-60} config: - provider: default-llm + provider: llm-default minInputChars: ${CLEAN_MIN_INPUT_CHARS:-1000} outputReserveRatio: 1.0 templates: @@ -123,7 +162,7 @@ pipelines: concurrencyGroup: llm timeoutSeconds: ${SUMMARIZE_TIMEOUT_SECONDS:-60} config: - provider: default-llm + provider: llm-default minInputChars: ${OUTPUT_TARGET_CHARS:-25000} outputReserveChars: ${OUTPUT_TARGET_CHARS:-25000} templates: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b9fc5e1..fdeb4c8 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -30,7 +30,7 @@ HTTP adapter `src/main.ts` parses bootstrap environment, creates the root logger, loads app assembly, builds Fastify, and installs shutdown handling. `src/adapters/http/http-app.ts` owns Fastify setup, request correlation, shared error handling, the silent `/health` route, and adapter registration. -`src/engine/create-engine.ts` is the programmatic engine boundary. It accepts `EngineConfig`, `EngineDescriptors`, `HostTools`, and a logger, then builds provider and output renderer registries, registers construction-time extension services, builds compiled pipelines internally, binds `PipelineHandle`s, and exposes `EngineRuntime`. +`src/engine/create-engine.ts` is the programmatic engine boundary. It accepts `EngineConfig`, `EngineDescriptors`, `HostTools`, and a logger, then builds provider, content transformer, and output renderer registries, registers construction-time extension services, builds compiled pipelines internally, binds `PipelineHandle`s, and exposes `EngineRuntime`. `src/app/compose.ts` is the service assembly point. It loads `AppConfig` from YAML, builds the host-tools bag, calls `createEngine(...)`, and creates HTTP adapter plugins over engine-provided pipeline handles. The composed result type is `ComposedApp` with two top-level keys: `adapters` (the `http` adapter set) and `registries` (`sourceProviders`, `llmProviders`, `outputRenderers`, and pipeline discovery info). @@ -44,11 +44,12 @@ src/ core/ framework-free pipeline engine (orchestrator, runner, effects, body store); `core/pipeline/` mirrors `contracts/pipeline/` contracts/ framework-free contracts grouped by intent pipeline/ pipeline ports and types: step, diagnostics, context, report, and handle definitions - extensions/ source provider, LLM provider, output renderer, registry, and resolved wrapper contracts + extensions/ source provider, content transformer, LLM provider, output renderer, registry, and resolved wrapper contracts host/ HostTools and ExtensionServices construction-time service contracts engine/ programmatic engine API, EngineConfig, EngineRuntime, createEngine, and engine-local builders builtins/ individual engine built-ins collected by descriptor bundles source-providers/ + content-transformers/ llm-providers/ pipeline-steps/ output-renderers/ @@ -67,8 +68,9 @@ The current built-ins are: - HTTP adapters: `open-webui`, `jina`. - Source providers: `http`, `firecrawl`, `docling`. +- Content transformers: `mdream`. - LLM providers: `openai-chat`. -- Pipeline steps: `load-source`, `llm-pass`, `truncate`, `capture-urls`, `verify-urls`. +- Pipeline steps: `load-source`, `llm-pass`, `transform`, `truncate`, `capture-urls`, `verify-urls`. - Output renderers: `debug-xml`, `passthrough`. ## Dependency Boundaries @@ -82,6 +84,7 @@ The import graph is enforced by [tests/architecture/import-boundaries.test.ts](. - `src/adapters/http/` owns Fastify integration, HTTP adapter contracts, HTTP adapter construction, the HTTP descriptor bundle, and HTTP built-ins. Fastify imports are allowed only in this layer. - `src/adapters/http/builtins//` may import its own files, the shared adapter auth helper, HTTP adapter contracts, contracts, shared code, or external packages. - `src/builtins/source-providers//` and `src/builtins/llm-providers//` may import their own files, contracts, shared code, or external packages. +- `src/builtins/content-transformers//` may import their own files, contracts, shared code, or external packages. - `src/builtins/pipeline-steps//` may import their own files, contracts, shared code, or external packages. - `src/builtins/output-renderers//` may import their own files, contracts, shared code, or external packages. - `src/bundles/` collects engine built-in descriptors. It may import concrete engine built-ins, contracts, and shared helpers, but not HTTP adapter descriptors. @@ -166,7 +169,7 @@ Bootstrap environment is intentionally small and parsed by `src/config/env-confi The YAML document is translated to `AppConfig` before runtime construction: -1. `src/config/yaml/yaml-config.ts` validates the coarse YAML shape: `httpAdapters`, `outputRenderers`, `sourceProviders`, `llmProviders`, and `pipelines`. +1. `src/config/yaml/yaml-config.ts` validates the coarse YAML shape: `httpAdapters`, `outputRenderers`, `sourceProviders`, `contentTransformers`, `llmProviders`, and `pipelines`. 2. `src/config/yaml/yaml-app-config.ts` maps engine-owned sections to `AppConfig.engineConfig`, HTTP adapter declarations to `AppConfig.adapters.http`, and `schemaVersion` to app metadata. Pipeline activation follows a tri-state `enabled`: `true` compiles, `false` parks it, **omitted** activates only when an HTTP adapter references it. 3. Each built-in descriptor parses its own `config` block with a local schema during engine or adapter construction. diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 77947f8..46c3098 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -47,7 +47,7 @@ These are consumed by helper scripts and ignored by the Node app. The default YAML file uses environment substitution for provider URLs, tokens, concurrency, timeouts, and renderer selection. Most of these values are shown in [.env.example](../.env.example) and passed through the Compose files. -The default (`truncate`) pipeline needs no provider configuration — it fetches and truncates. The `clean-llm` pipeline uses an LLM and Firecrawl; those env vars are +The default (`truncate`) pipeline needs no provider configuration — it fetches and truncates. The `clean-deterministic` pipeline loads HTML from `firecrawl-html` and converts it to markdown with the `mdream` transformer (no LLM); its Firecrawl env vars are required only when it is active (set `DEFAULT_PIPELINE=clean-deterministic`). The `clean-llm` pipeline uses an LLM and Firecrawl; those env vars are required only when `clean-llm` is active (set `DEFAULT_PIPELINE=clean-llm`). Validation is lazy: the service validates and constructs only the providers and pipelines that @@ -60,31 +60,40 @@ The placeholders below are grouped by the part of the pipeline they configure. | Variable | Default / behavior | Purpose | | ----------------- | ------------------ | -------------------------------------------------------------- | -| `SOURCE_PROVIDER` | `default-http` | Named source provider used by the `load-source` pipeline step. | +| `SOURCE_PROVIDER` | `http-default` | Named source provider used by the `load-source` pipeline step. | -The built-in `http` provider has no environment variables — it fetches the input URL directly. +The built-in `http` provider has no environment variables — it fetches the input URL directly. Each pipeline supplies its own `SOURCE_PROVIDER` fallback: `http-default` for `truncate`, `firecrawl-html` for `clean-deterministic`, and `firecrawl-markdown` for `clean-llm`. ### Source provider (Firecrawl) -| Variable | Default / behavior | Purpose | -| -------------------- | ------------------ | -------------------------------------------------------------------------------------------- | -| `FIRECRAWL_BASE_URL` | empty | Firecrawl base URL. Required only when a pipeline referencing `default-firecrawl` is active. | -| `FIRECRAWL_API_KEY` | empty | Optional Firecrawl bearer token. | +| Variable | Default / behavior | Purpose | +| -------------------- | ------------------ | ------------------------------------------------------------------------------------------------------------------ | +| `FIRECRAWL_BASE_URL` | empty | Firecrawl base URL. Required when any Firecrawl-based provider is active (`firecrawl-markdown`, `firecrawl-html`). | +| `FIRECRAWL_API_KEY` | empty | Optional Firecrawl bearer token. | ### Source provider (Docling) | Variable | Default / behavior | Purpose | | ------------------ | ------------------ | ---------------------------------------------------------------------------------------------- | -| `DOCLING_BASE_URL` | empty | Docling Serve base URL. Required only when a pipeline referencing `default-docling` is active. | +| `DOCLING_BASE_URL` | empty | Docling Serve base URL. Required only when a pipeline referencing `docling-default` is active. | | `DOCLING_API_KEY` | empty | Optional Docling API key. | +### Content transformer (mdream) + +| Variable | Default / behavior | Purpose | +| ---------------- | ------------------ | ---------------------------------------------------------------- | +| `MDREAM_MINIMAL` | `true` | Isolate main content and filter boilerplate in `mdream-default`. | +| `MDREAM_CLEAN` | `true` | Post-conversion link and whitespace cleanup. | + +These are read only when a pipeline using the `transform` step (such as `clean-deterministic`) is active. + ### LLM provider (OpenAI-compatible) | Variable | Default / behavior | Purpose | | -------------------------- | ------------------ | --------------------------------------------------------------------------------------------------------------------------- | -| `LLM_BASE_URL` | empty | OpenAI-compatible API base URL, usually ending in `/v1`. Required only when a pipeline referencing `default-llm` is active. | +| `LLM_BASE_URL` | empty | OpenAI-compatible API base URL, usually ending in `/v1`. Required only when a pipeline referencing `llm-default` is active. | | `LLM_API_KEY` | empty | Optional LLM bearer token. | -| `LLM_MODEL` | empty | Model identifier sent to chat completions. Required only when a pipeline referencing `default-llm` is active. | +| `LLM_MODEL` | empty | Model identifier sent to chat completions. Required only when a pipeline referencing `llm-default` is active. | | `LLM_CONTEXT_TOKENS` | empty in YAML | Optional model context window in tokens. Empty disables the context-fit gate. | | `LLM_CHARS_PER_TOKEN` | `3.5` | Conservative chars-per-token estimator used for the context-fit gate. | | `LLM_SAFETY_MARGIN_TOKENS` | `128` | Extra tokens reserved for chat-template framing and estimator drift. | diff --git a/docs/CUSTOMIZATION.md b/docs/CUSTOMIZATION.md index 6dcb386..71b1685 100644 --- a/docs/CUSTOMIZATION.md +++ b/docs/CUSTOMIZATION.md @@ -15,6 +15,7 @@ schemaVersion: 1 httpAdapters: {} outputRenderers: {} sourceProviders: {} +contentTransformers: {} llmProviders: {} pipelines: {} ``` @@ -171,6 +172,27 @@ config: The provider calls `POST /v1/convert/source`. Title is read from `document.json_content.name` (the JSON format is always requested internally regardless of the `output` setting). Upstream HTTP, parse, empty, and network failures are converted to degradable upstream errors. `output: markdown` returns `text/markdown`; `output: html` returns `text/html`. +## Content Transformers + +Content transformers convert a loaded body from one representation to another in-process, with no upstream call. The `transform` pipeline step selects a transformer by name and applies it to the current body. + +### `mdream` + +Converts HTML bodies to markdown using the [`@mdream/js`](https://www.npmjs.com/package/@mdream/js) library. Pure-JS, no native dependencies. + +```yaml +config: + minimal: ${MDREAM_MINIMAL:-true} + clean: ${MDREAM_CLEAN:-true} +``` + +| Knob | Values | Default | Purpose | +| --------- | ------ | ------- | -------------------------------------------------------------------------------------------------------- | +| `minimal` | bool | `true` | Apply mdream's minimal preset (isolate main content, filter boilerplate). Takes precedence over `clean`. | +| `clean` | bool | `true` | Clean up the markdown output: drop tracking params, redundant and empty links, and collapse blank lines. | + +The transformer supports text bodies whose media type is `text/html` or `application/xhtml+xml` and a requested target of `text/markdown`. The input URL is passed to mdream as the conversion origin, so relative links and images resolve against it. The original body title is preserved. + ## LLM Providers ### `openai-chat` @@ -214,7 +236,7 @@ pipelines: `outputRenderer` references a named output renderer instance. `limiters` declares concurrency group names and their maximum concurrency. A step can opt into a group with `concurrencyGroup`. -Adjacent steps that share the same `concurrencyGroup` share one limiter acquisition. In the default pipeline, `llm-pass(clean)`, `verify_after_clean`, `llm-pass(summarize)`, and `verify_after_summarize` all use the `llm` group, so a URL holds one LLM workflow slot across both passes and their URL checks. Inserting a step with a different (or no) group between them splits that acquisition. +Adjacent steps that share the same `concurrencyGroup` share one limiter acquisition. In the `clean-llm` pipeline, `llm-pass(clean)`, `verify_after_clean`, `llm-pass(summarize)`, and `verify_after_summarize` all use the `llm` group, so a URL holds one LLM workflow slot across both passes and their URL checks. Inserting a step with a different (or no) group between them splits that acquisition. `timeoutSeconds` applies to both limiter waiting and step execution. For one adjacent concurrency block, limiter waiting uses the longest `timeoutSeconds` value in that block. @@ -224,7 +246,7 @@ Adjacent steps that share the same `concurrencyGroup` share one limiter acquisit ```yaml config: - provider: ${SOURCE_PROVIDER:-default-http} + provider: ${SOURCE_PROVIDER:-http-default} ``` Loads the initial body from a named source provider. If a body already exists, the step skips with `body_present`. @@ -233,7 +255,7 @@ Loads the initial body from a named source provider. If a body already exists, t ```yaml config: - provider: default-llm + provider: llm-default minInputChars: 1500 maxInputChars: 80000 outputReserveRatio: 1.0 @@ -271,6 +293,27 @@ Both templates are required. `templates.vars` is an optional map of literal valu Mustache escaping is disabled for prompt templates so markdown is passed through as-is. +### `transform` + +```yaml +config: + transformer: mdream-default + target: text/markdown + onUnsupported: skip + emitDiagnostics: false +``` + +Applies a named content transformer to the current body. The step resolves `transformer` from the `contentTransformers` registry and asks it to produce `target`. + +| Knob | Values | Default | Purpose | +| ----------------- | ---------------- | ------- | -------------------------------------------------------------------------------------------------------------- | +| `transformer` | string | — | Name of a configured `contentTransformers` instance. Required. | +| `target` | string | — | Media type the transformer must produce, for example `text/markdown`. Required. | +| `onUnsupported` | `skip` \| `fail` | `skip` | What to do when the transformer does not support the current body (wrong source media type or representation). | +| `emitDiagnostics` | bool | `false` | When enabled, transformer-reported diagnostics are surfaced as child nodes in the step report. | + +The step skips with `no_body` when there is no body. When the transformer does not support the current body it skips with `unsupported`, or fails with that reason when `onUnsupported: fail`. A transform aborted by the step timeout fails with `timeout`. If the transformer returns a body that does not match `target`, the step fails with `wrong_output_type`. + ### `truncate` ```yaml @@ -313,12 +356,30 @@ In both modes a hallucination makes the pipeline rollup `degraded` (an earlier b Shipped defaults: `verify_after_clean` uses `rollback` with `maxReportedUrls: 0` (silent rollback), `verify_after_summarize` uses `report` with `maxReportedUrls: 50`. -## Default Pipeline +## Shipped Pipelines + +The config ships three pipelines, selected via `DEFAULT_PIPELINE` (defaults to `truncate`). + +### `truncate` (default) + +```text +load-source(http-default) -> truncate +``` + +Fetches the URL directly and truncates to budget. No external services required. + +### `clean-deterministic` + +```text +load-source(firecrawl-html) -> transform(html -> markdown via mdream) -> truncate +``` + +Loads HTML from the `firecrawl-html` source instance (Firecrawl with `output: rawHtml`) and converts it to markdown deterministically with the `mdream` transformer, with no LLM passes. Defaults `SOURCE_PROVIDER` to `firecrawl-html`. Select with `DEFAULT_PIPELINE=clean-deterministic`. -The shipped default pipeline is: +### `clean-llm` ```text -load-source -> capture-urls -> llm-pass(clean) -> verify-urls(rollback) -> llm-pass(summarize) -> verify-urls(report) -> truncate +load-source(firecrawl-markdown) -> capture-urls -> llm-pass(clean) -> verify-urls(rollback) -> llm-pass(summarize) -> verify-urls(report) -> truncate ``` -That shape is a starting point, not a fixed contract. Prefer small YAML changes and verify behavior with a few representative URLs before relying on a customized pipeline. +The original reference pipeline. Loads markdown from a Firecrawl instance, runs clean and summarize LLM passes with URL-hallucination gates, then truncates. Defaults `SOURCE_PROVIDER` to `firecrawl-markdown`. Select with `DEFAULT_PIPELINE=clean-llm`. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 66b24d6..e806798 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -40,7 +40,7 @@ The URL-in / clean-Markdown-within-a-budget contract does not change. What chang ## Coming soon -1. **Deterministic extract and convert steps.** First implementations: main-content extraction with Mozilla Readability (in-process, needs only a DOM such as linkedom) and HTML-to-Markdown with node-html-markdown or Turndown. Composed, these replace the LLM clean pass on the HTML path. +1. **In-process main-content extraction.** A new pipeline step that extracts content from HTML using Readability. This creates a fully deterministic HTML-to-markdown path with no external service dependency for boilerplate removal. 2. **Demote the LLM clean pass.** Once deterministic clean matches or beats it on representative URLs, make the clean stage optional and only execute it for URLs where deterministic clean did not produce desired results. Keep summarize. 3. **Keep summaries honest.** Extend the existing URL quality gate from all-or-nothing rejection into a deterministic repair pass, and implement fenced-code-block verification/repair. diff --git a/docs/agents/architecture-rules.md b/docs/agents/architecture-rules.md index a2a9f7d..5587638 100644 --- a/docs/agents/architecture-rules.md +++ b/docs/agents/architecture-rules.md @@ -34,7 +34,7 @@ Classify the file you are touching before editing it. If the change does not fit ## Placement Rules -- New source provider, LLM provider, pipeline step, or output renderer: put implementation and descriptor under `src/builtins/...`, add its descriptor to `src/bundles/default-engine-descriptors.ts` only when it belongs in the default service bundle, and test under `tests/builtins/...`. +- New source provider, content transformer, LLM provider, pipeline step, or output renderer: put implementation and descriptor under `src/builtins/...`, add its descriptor to `src/bundles/default-engine-descriptors.ts` only when it belongs in the default service bundle, and test under `tests/builtins/...`. - New HTTP adapter: put implementation, config, descriptor, and adapter-local helpers under `src/adapters/http/builtins//`, add it to `src/adapters/http/descriptor-bundle.ts` only when it belongs in the hosted default bundle, and test under `tests/adapters/http/builtins//`. - New adapter surface such as MCP or CLI: create a sibling under `src/adapters//`, consume `EngineRuntime` or `PipelineHandle`, and update the architecture test in the same change. - New host singleton such as clock, cache, telemetry, persistence, or fetch policy: prefer a `HostTools` key in `src/contracts/host/host-tools.ts`, with the concrete host implementation wired from `src/app/`. diff --git a/docs/agents/extension-authoring.md b/docs/agents/extension-authoring.md index 6f70666..dfdeb28 100644 --- a/docs/agents/extension-authoring.md +++ b/docs/agents/extension-authoring.md @@ -1,12 +1,12 @@ # Extension Authoring -This guide is for adding a new built-in implementation (source provider, LLM provider, pipeline step, output renderer, or HTTP adapter) inside this repository. Each category sits behind a small descriptor contract in its own folder, which keeps adding a built-in a localized change. Keep this guide and the live contracts in sync. +This guide is for adding a new built-in implementation (source provider, content transformer, LLM provider, pipeline step, output renderer, or HTTP adapter) inside this repository. Each category sits behind a small descriptor contract in its own folder, which keeps adding a built-in a localized change. Keep this guide and the live contracts in sync. Read [docs/ARCHITECTURE.md](../ARCHITECTURE.md) and [architecture-rules.md](architecture-rules.md) first. This document assumes you already know the runtime flow, dependency graph, and the three host-capability surfaces. ## Common Shape -All five extension categories follow the same shape: +All six extension categories follow the same shape: 1. A small **runtime instance interface** under `src/contracts/extensions/`, `src/contracts/pipeline/`, or the adapter surface describing behavior only. These interfaces do not carry `name` or `type`. 2. A **descriptor** included by the appropriate descriptor bundle. Each descriptor declares: @@ -63,6 +63,10 @@ Note the `.provider` deref — registries hold `Resolved*` wrappers. Implement `SourceProvider` (`src/contracts/extensions/source-provider.ts`). The interface has `load(url, { signal }): Promise`. Throw `UpstreamError` for HTTP, parse, or empty-response failures (use the constructors in `src/shared/errors.ts`); let abort errors propagate. Built-in example: `src/builtins/source-providers/firecrawl/`. +### Content Transformers + +Implement `ContentTransformer` (`src/contracts/extensions/content-transformer.ts`): `supports({ sourceKind, sourceMediaType, request })` and `transform({ url, body, request }, { signal }): Promise`. `supports` gates `transform`: the step only invokes a matching transformer, so `transform` may throw `InternalError` for inputs that bypass the gate. Built-in example: `src/builtins/content-transformers/mdream/`. + ### LLM Providers Implement `LlmProvider` (`src/contracts/extensions/llm-provider.ts`). Same error rules as source providers. Built-in example: `src/builtins/llm-providers/openai-chat/`. diff --git a/package-lock.json b/package-lock.json index a91078c..d601c17 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.2.1", "license": "MIT", "dependencies": { + "@mdream/js": "^1.3.0", "fastify": "^5.2.1", "mustache": "^4.2.0", "p-limit": "^6.2.0", @@ -688,6 +689,34 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@mdream/js": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@mdream/js/-/js-1.3.0.tgz", + "integrity": "sha512-ZRLe+8T3iApvBNVHcOwbfgysYWHDvhAjk8Wonb1PJ48UMse627UmUIXPxf6kG6S9brl1Opnq/DdWRLMCnI0mEA==", + "license": "MIT", + "dependencies": { + "cac": "^7.0.0", + "pathe": "^2.0.3" + }, + "bin": { + "mdream-js": "bin/mdream.mjs" + } + }, + "node_modules/@mdream/js/node_modules/cac": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/cac/-/cac-7.0.0.tgz", + "integrity": "sha512-tixWYgm5ZoOD+3g6UTea91eow5z6AAHaho3g0V9CNSNb45gM8SmflpAc+GRd1InC4AqN/07Unrgp56Y94N9hJQ==", + "license": "MIT", + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/@mdream/js/node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "license": "MIT" + }, "node_modules/@pinojs/redact": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", diff --git a/package.json b/package.json index 8ba2952..a2572be 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "format:check": "prettier --check \"*.ts\" \"src/**/*.ts\" \"tests/**/*.ts\"" }, "dependencies": { + "@mdream/js": "^1.3.0", "fastify": "^5.2.1", "mustache": "^4.2.0", "p-limit": "^6.2.0", diff --git a/scripts/inspect-suspects.ps1 b/scripts/inspect-suspects.ps1 index 44d609b..0a4d0bd 100644 --- a/scripts/inspect-suspects.ps1 +++ b/scripts/inspect-suspects.ps1 @@ -3,7 +3,7 @@ # poor cleanup result. # # For each URL three artifacts are written to scripts/out/: -# .source.md - raw Firecrawl markdown +# .source.html - raw HTML from Firecrawl (universal baseline for any pipeline) # .clean.md - loader's /r/ body (including footer) # .footer.txt - extracted line (or empty) # @@ -95,7 +95,7 @@ foreach ($url in $Urls) { Write-Host "`n=== $slug ===" -ForegroundColor Cyan Write-Host " $url" -ForegroundColor DarkGray - $body = @{ url = $url; formats = @('markdown'); onlyMainContent = $true } | ConvertTo-Json -Compress + $body = @{ url = $url; formats = @('rawHtml') } | ConvertTo-Json -Compress $src = '' try { @@ -103,7 +103,7 @@ foreach ($url in $Urls) { -Uri "$($firecrawl.BaseUrl)/v2/scrape" ` -Headers $firecrawlHeaders -ContentType 'application/json' ` -Body $body -TimeoutSec 120 - $src = $firecrawlResponse.data.markdown + $src = $firecrawlResponse.data.rawHtml if (-not $src) { $src = $firecrawlResponse.data.content } } catch { @@ -111,7 +111,7 @@ foreach ($url in $Urls) { $fail++ continue } - Set-Content -LiteralPath (Join-Path $outDir "$slug.source.md") -Value $src -Encoding utf8 + Set-Content -LiteralPath (Join-Path $outDir "$slug.source.html") -Value $src -Encoding utf8 $clean = '' try { diff --git a/src/builtins/content-transformers/mdream/mdream-transformer-config.ts b/src/builtins/content-transformers/mdream/mdream-transformer-config.ts new file mode 100644 index 0000000..3889cae --- /dev/null +++ b/src/builtins/content-transformers/mdream/mdream-transformer-config.ts @@ -0,0 +1,27 @@ +/** + * Parses YAML configuration for the built-in mdream content transformer. + * + * Environment substitution runs before this schema, so boolean fields use the + * shared preprocessor to treat blank placeholders as omitted values. These knobs + * tune transformer behavior per instance; routing intent (the target media type) + * comes from the requesting step, not this config. + */ + +import { z } from "zod"; + +import { booleanStringAsBooleanOrUndefined } from "../../../shared/config-coercion.js"; + +const mdreamTransformerConfigSchema = z + .object({ + minimal: z.preprocess(booleanStringAsBooleanOrUndefined, z.boolean().default(false)), + clean: z.preprocess(booleanStringAsBooleanOrUndefined, z.boolean().default(true)) + }) + .strict(); + +/** Represents parsed mdream transformer configuration. */ +export type MdreamTransformerConfig = z.infer; + +/** Parses mdream transformer configuration from YAML. */ +export function parseMdreamTransformerConfig(raw: unknown): MdreamTransformerConfig { + return Object.freeze(mdreamTransformerConfigSchema.parse(raw)); +} diff --git a/src/builtins/content-transformers/mdream/mdream-transformer-descriptor.ts b/src/builtins/content-transformers/mdream/mdream-transformer-descriptor.ts new file mode 100644 index 0000000..d832d30 --- /dev/null +++ b/src/builtins/content-transformers/mdream/mdream-transformer-descriptor.ts @@ -0,0 +1,20 @@ +/** + * Exposes the mdream content transformer descriptor to engine bundles. + * + * The transformer is a self-contained leaf with no host tool or service + * dependencies; configured identity is added later by engine registry + * construction. + */ + +import { parseMdreamTransformerConfig } from "./mdream-transformer-config.js"; +import { MdreamTransformer } from "./mdream-transformer.js"; + +import type { ContentTransformerDescriptor } from "../../../contracts/extensions/content-transformer.js"; +import type { MdreamTransformerConfig } from "./mdream-transformer-config.js"; + +/** Defines the built-in mdream content transformer type. */ +export const mdreamTransformerDescriptor = { + type: "mdream", + parseConfig: parseMdreamTransformerConfig, + create: args => new MdreamTransformer(args.config, { logger: args.deps.logger }) +} satisfies ContentTransformerDescriptor; diff --git a/src/builtins/content-transformers/mdream/mdream-transformer.ts b/src/builtins/content-transformers/mdream/mdream-transformer.ts new file mode 100644 index 0000000..b6362ef --- /dev/null +++ b/src/builtins/content-transformers/mdream/mdream-transformer.ts @@ -0,0 +1,73 @@ +/** + * Converts HTML bodies to Markdown using the pure-JS mdream engine. + * + * `htmlToMarkdown` is synchronous and has no native dependencies. The instance + * passes the source URL as the mdream `origin` so relative links and images + * resolve to absolute URLs. The `minimal` preset additionally isolates main + * content and filters boilerplate; `clean` applies link and whitespace cleanup. + */ + +import { htmlToMarkdown, withMinimalPreset, type MdreamOptions } from "@mdream/js"; + +import { InternalError } from "../../../shared/errors.js"; +import { isHtmlMediaType, mediaTypes } from "../../../shared/media-types.js"; + +import type { + ContentTransformDiagnostic, + ContentTransformRequest, + ContentTransformResult, + ContentTransformer +} from "../../../contracts/extensions/content-transformer.js"; +import type { BodyContent } from "../../../contracts/pipeline/context.js"; +import type { Logger } from "../../../shared/logger.js"; +import type { MdreamTransformerConfig } from "./mdream-transformer-config.js"; + +/** Transforms HTML text bodies into Markdown via mdream. */ +export class MdreamTransformer implements ContentTransformer { + /** Creates an mdream transformer instance. */ + constructor( + private readonly config: MdreamTransformerConfig, + private readonly deps: { readonly logger: Logger } + ) {} + + /** Supports HTML or XHTML text bodies targeting Markdown output. */ + supports(input: { + readonly sourceKind: BodyContent["kind"]; + readonly sourceMediaType: string; + readonly request: ContentTransformRequest; + }): boolean { + return ( + input.sourceKind === "text" && + isHtmlMediaType(input.sourceMediaType) && + input.request.targetMediaType === mediaTypes.markdown + ); + } + + /** Renders the HTML body to Markdown, preserving the source title. */ + async transform( + input: { readonly url: string; readonly body: BodyContent; readonly request: ContentTransformRequest }, + opts: { readonly signal: AbortSignal } + ): Promise { + opts.signal.throwIfAborted(); + const body = input.body; + if (body.kind !== "text") + throw new InternalError("mdream transformer requires a text body", "mdream_non_text_body"); + + const markdown = htmlToMarkdown(body.content, this.buildOptions(input.url)).trim(); + const diagnostics: ContentTransformDiagnostic[] = [ + { code: "mdream", message: `${body.content.length} html chars -> ${markdown.length} markdown chars` } + ]; + if (!markdown) diagnostics.push({ code: "empty_output" }); + + return { + body: { kind: "text", mediaType: mediaTypes.markdown, content: markdown, title: body.title }, + diagnostics + }; + } + + /** Builds mdream options from instance config, threading the source URL. */ + private buildOptions(origin: string): Partial { + const base: Partial = { origin }; + return this.config.minimal ? withMinimalPreset(base) : { ...base, clean: this.config.clean }; + } +} diff --git a/src/builtins/pipeline-steps/transform/transform-step-config.ts b/src/builtins/pipeline-steps/transform/transform-step-config.ts new file mode 100644 index 0000000..aef90ab --- /dev/null +++ b/src/builtins/pipeline-steps/transform/transform-step-config.ts @@ -0,0 +1,35 @@ +/** + * Parses YAML configuration for the built-in transform pipeline step. + * + * The step is generic: `transformer` selects a configured content transformer + * instance and `target` is the required output media type the step both requests + * and asserts. `onUnsupported` decides whether an unhandled body skips or fails. + */ + +import { z } from "zod"; + +import { booleanStringAsBooleanOrUndefined } from "../../../shared/config-coercion.js"; + +const transformStepConfigSchema = z + .object({ + transformer: z.string().min(1, "transform step requires a transformer name"), + target: z.string().min(1, "transform step requires a target media type"), + onUnsupported: z.enum(["skip", "fail"]).default("skip"), + emitDiagnostics: z.preprocess(booleanStringAsBooleanOrUndefined, z.boolean().default(false)) + }) + .strict(); + +/** Represents parsed YAML configuration for a transform step. */ +export type TransformStepConfig = z.infer; + +/** Describes constructor configuration for a transform step instance. */ +export interface TransformStepOptions { + readonly target: string; + readonly onUnsupported: "skip" | "fail"; + readonly emitDiagnostics: boolean; +} + +/** Parses transform step configuration from YAML. */ +export function parseTransformStepConfig(raw: unknown): TransformStepConfig { + return Object.freeze(transformStepConfigSchema.parse(raw)); +} diff --git a/src/builtins/pipeline-steps/transform/transform-step-descriptor.ts b/src/builtins/pipeline-steps/transform/transform-step-descriptor.ts new file mode 100644 index 0000000..c11bda4 --- /dev/null +++ b/src/builtins/pipeline-steps/transform/transform-step-descriptor.ts @@ -0,0 +1,32 @@ +/** + * Exposes the transform pipeline step descriptor to engine bundles. + * + * Construction resolves the configured content transformer by name so runtime + * step execution only depends on the transformer port, target, and logger. + */ + +import { contentTransformerRegistryKey } from "../../../contracts/extensions/content-transformer.js"; +import { parseTransformStepConfig } from "./transform-step-config.js"; +import { TransformStep } from "./transform-step.js"; + +import type { PipelineStepDescriptor } from "../../../contracts/pipeline/step.js"; +import type { TransformStepConfig } from "./transform-step-config.js"; + +/** Defines the built-in transform step type. */ +export const transformStepDescriptor = { + type: "transform", + parseConfig: parseTransformStepConfig, + create: args => { + const transformer = args.services + .require(contentTransformerRegistryKey) + .require(args.config.transformer).transformer; + return new TransformStep( + { + target: args.config.target, + onUnsupported: args.config.onUnsupported, + emitDiagnostics: args.config.emitDiagnostics + }, + { transformer, logger: args.deps.logger } + ); + } +} satisfies PipelineStepDescriptor; diff --git a/src/builtins/pipeline-steps/transform/transform-step.ts b/src/builtins/pipeline-steps/transform/transform-step.ts new file mode 100644 index 0000000..a84714b --- /dev/null +++ b/src/builtins/pipeline-steps/transform/transform-step.ts @@ -0,0 +1,76 @@ +/** + * Runs a configured content transformer against the current pipeline body. + * + * The step is transformer-agnostic. It asks the resolved transformer whether it + * supports the current body for the requested target, runs it, then asserts the + * result matches the requested target media type and representation before + * applying it. This intrinsic honesty gate keeps transformers truthful. + */ + +import { isAbortError } from "../../../shared/errors.js"; +import { isTextLike } from "../../../shared/media-types.js"; + +import type { + ContentTransformDiagnostic, + ContentTransformRequest, + ContentTransformer +} from "../../../contracts/extensions/content-transformer.js"; +import type { BodyContent, PipelineContext } from "../../../contracts/pipeline/context.js"; +import type { StepDiagnostics } from "../../../contracts/pipeline/diagnostics.js"; +import type { PipelineStep, StepResult } from "../../../contracts/pipeline/step.js"; +import type { Logger } from "../../../shared/logger.js"; +import type { TransformStepOptions } from "./transform-step-config.js"; + +/** Transforms the current body toward a target representation via a transformer. */ +export class TransformStep implements PipelineStep { + /** Creates a transform step instance. */ + constructor( + private readonly config: TransformStepOptions, + private readonly deps: { readonly transformer: ContentTransformer; readonly logger: Logger } + ) {} + + /** Runs the transformer when it supports the current body for the target. */ + async run(ctx: PipelineContext): Promise { + const body = ctx.body.current(); + if (!body) return { status: "skipped", reason: "no_body" }; + + const request: ContentTransformRequest = { targetMediaType: this.config.target }; + if (!this.deps.transformer.supports({ sourceKind: body.kind, sourceMediaType: body.mediaType, request })) + return { status: this.config.onUnsupported === "fail" ? "failed" : "skipped", reason: "unsupported" }; + + let result; + try { + result = await this.deps.transformer.transform({ url: ctx.input.url, body, request }, { signal: ctx.signal }); + } catch (error) { + if (isAbortError(error)) return { status: "failed", reason: "timeout" }; + throw error; + } + + if (!outputMatchesTarget(result.body, this.config.target)) return { status: "failed", reason: "wrong_output_type" }; + + return { + status: "ok", + effects: { body: result.body }, + diagnostics: this.config.emitDiagnostics ? toStepDiagnostics(result.diagnostics) : undefined + }; + } +} + +/** Returns whether a transformer result matches the requested target type and representation. */ +function outputMatchesTarget(body: BodyContent, target: string): boolean { + if (body.mediaType !== target) return false; + return isTextLike(target) ? body.kind === "text" : body.kind === "binary"; +} + +/** Maps transformer diagnostics into a step diagnostics tree, or undefined when empty. */ +function toStepDiagnostics( + diagnostics: ReadonlyArray | undefined +): StepDiagnostics | undefined { + if (!diagnostics || diagnostics.length === 0) return undefined; + return { + children: diagnostics.map(diagnostic => ({ + name: diagnostic.code, + attributes: diagnostic.message === undefined ? undefined : { message: diagnostic.message } + })) + }; +} diff --git a/src/bundles/default-engine-descriptors.ts b/src/bundles/default-engine-descriptors.ts index d0b48a8..fc9919f 100644 --- a/src/bundles/default-engine-descriptors.ts +++ b/src/bundles/default-engine-descriptors.ts @@ -6,12 +6,14 @@ * engine construction never imports this bundle directly. */ +import { mdreamTransformerDescriptor } from "../builtins/content-transformers/mdream/mdream-transformer-descriptor.js"; import { openAiChatProviderDescriptor } from "../builtins/llm-providers/openai-chat/openai-chat-provider-descriptor.js"; import { debugXmlRendererDescriptor } from "../builtins/output-renderers/debug-xml/debug-xml-renderer-descriptor.js"; import { passthroughRendererDescriptor } from "../builtins/output-renderers/passthrough/passthrough-renderer-descriptor.js"; import { captureUrlsStepDescriptor } from "../builtins/pipeline-steps/capture-urls/capture-urls-step-descriptor.js"; import { llmPassStepDescriptor } from "../builtins/pipeline-steps/llm-pass/llm-pass-step-descriptor.js"; import { loadSourceStepDescriptor } from "../builtins/pipeline-steps/load-source/load-source-step-descriptor.js"; +import { transformStepDescriptor } from "../builtins/pipeline-steps/transform/transform-step-descriptor.js"; import { truncateStepDescriptor } from "../builtins/pipeline-steps/truncate/truncate-step-descriptor.js"; import { verifyUrlsStepDescriptor } from "../builtins/pipeline-steps/verify-urls/verify-urls-step-descriptor.js"; import { httpProviderDescriptor } from "../builtins/source-providers/http/http-provider-descriptor.js"; @@ -19,6 +21,7 @@ import { firecrawlProviderDescriptor } from "../builtins/source-providers/firecr import { doclingProviderDescriptor } from "../builtins/source-providers/docling/docling-provider-descriptor.js"; import { createDescriptorRecord } from "../shared/descriptors.js"; +import type { ContentTransformerDescriptor } from "../contracts/extensions/content-transformer.js"; import type { LlmProviderDescriptor } from "../contracts/extensions/llm-provider.js"; import type { OutputRendererDescriptor } from "../contracts/extensions/output-renderer.js"; import type { SourceProviderDescriptor } from "../contracts/extensions/source-provider.js"; @@ -27,6 +30,7 @@ import type { PipelineStepDescriptor } from "../contracts/pipeline/step.js"; /** Engine-facing descriptor bundle selected by the hosted service by default. */ export interface EngineDescriptorBundle { readonly sourceProviders: Readonly>; + readonly contentTransformers: Readonly>; readonly llmProviders: Readonly>; readonly outputRenderers: Readonly>; readonly pipelineSteps: Readonly>; @@ -38,6 +42,7 @@ export const DEFAULT_ENGINE_DESCRIPTOR_BUNDLE: EngineDescriptorBundle = Object.f [httpProviderDescriptor, firecrawlProviderDescriptor, doclingProviderDescriptor], descriptor => descriptor.type ), + contentTransformers: createDescriptorRecord([mdreamTransformerDescriptor], descriptor => descriptor.type), llmProviders: createDescriptorRecord([openAiChatProviderDescriptor], descriptor => descriptor.type), outputRenderers: createDescriptorRecord( [debugXmlRendererDescriptor, passthroughRendererDescriptor], @@ -48,6 +53,7 @@ export const DEFAULT_ENGINE_DESCRIPTOR_BUNDLE: EngineDescriptorBundle = Object.f captureUrlsStepDescriptor, loadSourceStepDescriptor, llmPassStepDescriptor, + transformStepDescriptor, truncateStepDescriptor, verifyUrlsStepDescriptor ], diff --git a/src/config/yaml/yaml-app-config.ts b/src/config/yaml/yaml-app-config.ts index 921dc98..60d0253 100644 --- a/src/config/yaml/yaml-app-config.ts +++ b/src/config/yaml/yaml-app-config.ts @@ -49,6 +49,7 @@ export function yamlToAppConfig(yamlConfig: RawYamlConfig): AppConfig { return { engineConfig: { sourceProviders: yamlConfig.sourceProviders, + contentTransformers: yamlConfig.contentTransformers, llmProviders: yamlConfig.llmProviders, outputRenderers: yamlConfig.outputRenderers, pipelines diff --git a/src/config/yaml/yaml-config.ts b/src/config/yaml/yaml-config.ts index 4982c99..15cddfd 100644 --- a/src/config/yaml/yaml-config.ts +++ b/src/config/yaml/yaml-config.ts @@ -40,6 +40,7 @@ export const rawYamlConfigSchema = z .object({ schemaVersion: z.literal(1).default(1), sourceProviders: z.record(providerEntrySchema).default({}), + contentTransformers: z.record(providerEntrySchema).default({}), llmProviders: z.record(providerEntrySchema).default({}), outputRenderers: z.record(providerEntrySchema).default({}), pipelines: z.record( diff --git a/src/contracts/extensions/content-transformer.ts b/src/contracts/extensions/content-transformer.ts new file mode 100644 index 0000000..564cbfd --- /dev/null +++ b/src/contracts/extensions/content-transformer.ts @@ -0,0 +1,79 @@ +/** + * Defines the content transformer extension contract and registry service key. + * + * Content transformers rewrite one pipeline body into another representation, + * such as HTML to Markdown today and Markdown cleanup later. A transformer + * instance may handle several source/target combinations; `supports` is its + * capability matrix and `transform` performs the conversion. The requesting + * step owns routing intent (the target media type) and asserts the result. + */ + +import { createExtensionServiceKey } from "../host/extension-services.js"; + +import type { Logger } from "../../shared/logger.js"; +import type { HostTools } from "../host/host-tools.js"; +import type { BodyContent } from "../pipeline/context.js"; +import type { NamedRegistry } from "./named-registry.js"; +import type { ResolvedContentTransformer } from "./resolved-extension.js"; + +/** Describes the routing intent a step passes to a content transformer. */ +export interface ContentTransformRequest { + readonly targetMediaType: string; +} + +/** Carries observability-only detail about one transform outcome. */ +export interface ContentTransformDiagnostic { + readonly code: string; + readonly message?: string; +} + +/** Carries the transformed body and optional diagnostics. */ +export interface ContentTransformResult { + readonly body: BodyContent; + readonly diagnostics?: ReadonlyArray; +} + +/** Transforms one pipeline body representation into another. */ +export interface ContentTransformer { + /** Reports whether this instance can satisfy the requested transform. */ + supports(input: { + readonly sourceKind: BodyContent["kind"]; + readonly sourceMediaType: string; + readonly request: ContentTransformRequest; + }): boolean; + + /** Transforms the body toward the requested target representation. */ + transform( + input: { readonly url: string; readonly body: BodyContent; readonly request: ContentTransformRequest }, + opts: { readonly signal: AbortSignal } + ): Promise; +} + +/** Resolves configured content transformers by name. */ +export type ContentTransformerRegistry = NamedRegistry; + +/** Provides dependencies available while constructing a content transformer. */ +export interface ContentTransformerCreateDeps { + readonly logger: Logger; + readonly tools: HostTools; +} + +/** Provides arguments used to construct one content transformer instance. */ +export interface ContentTransformerCreateArgs { + readonly name: string; + readonly config: TConfig; + readonly deps: ContentTransformerCreateDeps; +} + +/** Defines one content transformer implementation type addressable from YAML. */ +export interface ContentTransformerDescriptor { + readonly type: string; + parseConfig(raw: unknown): TConfig; + create(args: ContentTransformerCreateArgs): ContentTransformer; +} + +/** Identifies the content transformer registry extension service. */ +export const contentTransformerRegistryKey = createExtensionServiceKey({ + id: "llmc.contentTransformerRegistry", + description: "content transformer registry" +}); diff --git a/src/contracts/extensions/resolved-extension.ts b/src/contracts/extensions/resolved-extension.ts index 6760b42..a362e20 100644 --- a/src/contracts/extensions/resolved-extension.ts +++ b/src/contracts/extensions/resolved-extension.ts @@ -7,6 +7,7 @@ * identities without coupling implementations to app configuration. */ +import type { ContentTransformer } from "./content-transformer.js"; import type { LlmProvider } from "./llm-provider.js"; import type { OutputRenderer } from "./output-renderer.js"; import type { SourceProvider } from "./source-provider.js"; @@ -18,6 +19,13 @@ export interface ResolvedSourceProvider { readonly provider: SourceProvider; } +/** Pairs a configured content transformer instance with its identity. */ +export interface ResolvedContentTransformer { + readonly name: string; + readonly type: string; + readonly transformer: ContentTransformer; +} + /** Pairs a configured LLM provider instance with its identity. */ export interface ResolvedLlmProvider { readonly name: string; diff --git a/src/engine/create-engine.ts b/src/engine/create-engine.ts index 35622f9..b89d924 100644 --- a/src/engine/create-engine.ts +++ b/src/engine/create-engine.ts @@ -7,12 +7,14 @@ * bundles, process bootstrap code, or concrete built-ins. */ +import { contentTransformerRegistryKey } from "../contracts/extensions/content-transformer.js"; import { llmProviderRegistryKey } from "../contracts/extensions/llm-provider.js"; import { outputRendererRegistryKey } from "../contracts/extensions/output-renderer.js"; import { sourceProviderRegistryKey } from "../contracts/extensions/source-provider.js"; import { PipelineOrchestrator } from "../core/pipeline/orchestrator.js"; import { PipelineRunner } from "../core/pipeline/runner.js"; import { createEngineRuntime } from "./engine-runtime.js"; +import { buildContentTransformers } from "./internal/builders/build-content-transformers.js"; import { buildLlmProviders } from "./internal/builders/build-llm-providers.js"; import { buildOutputRenderers } from "./internal/builders/build-output-renderers.js"; import { buildPipelines } from "./internal/builders/build-pipelines.js"; @@ -40,6 +42,13 @@ export async function createEngine(args: { args.descriptors.sourceProviders ); serviceBuilder.register(sourceProviderRegistryKey, sourceProviders); + const contentTransformers = buildContentTransformers( + args.config.contentTransformers, + args.tools, + args.logger, + args.descriptors.contentTransformers + ); + serviceBuilder.register(contentTransformerRegistryKey, contentTransformers); const llmProviders = buildLlmProviders( args.config.llmProviders, args.tools, @@ -65,6 +74,12 @@ export async function createEngine(args: { ); logSkipped(args.logger, "source provider", Object.keys(args.config.sourceProviders), sourceProviders.builtNames()); + logSkipped( + args.logger, + "content transformer", + Object.keys(args.config.contentTransformers), + contentTransformers.builtNames() + ); logSkipped(args.logger, "LLM provider", Object.keys(args.config.llmProviders), llmProviders.builtNames()); logSkipped(args.logger, "output renderer", Object.keys(args.config.outputRenderers), outputRenderers.builtNames()); @@ -72,7 +87,7 @@ export async function createEngine(args: { new PipelineOrchestrator({ logger: args.logger.child({ component: "orchestrator" }) }) ); const handles = new Map([...compiledPipelines].map(([name, pipeline]) => [name, runner.bindTo(pipeline)])); - const registries: EngineRegistries = { sourceProviders, llmProviders, outputRenderers }; + const registries: EngineRegistries = { sourceProviders, contentTransformers, llmProviders, outputRenderers }; return createEngineRuntime({ registries, pipelines: pipelineInfos(args.config), handles }); } diff --git a/src/engine/engine-config.ts b/src/engine/engine-config.ts index 0535506..afacd8b 100644 --- a/src/engine/engine-config.ts +++ b/src/engine/engine-config.ts @@ -32,6 +32,7 @@ export interface EnginePipelineConfig { /** Engine-only config. Excludes schema metadata and adapter configuration. */ export interface EngineConfig { readonly sourceProviders: Readonly>; + readonly contentTransformers: Readonly>; readonly llmProviders: Readonly>; readonly outputRenderers: Readonly>; readonly pipelines: Readonly>; diff --git a/src/engine/engine-descriptors.ts b/src/engine/engine-descriptors.ts index 842579b..05f078a 100644 --- a/src/engine/engine-descriptors.ts +++ b/src/engine/engine-descriptors.ts @@ -5,6 +5,7 @@ * itself. Bundle selection belongs to app assembly or another embedding host. */ +import type { ContentTransformerDescriptor } from "../contracts/extensions/content-transformer.js"; import type { LlmProviderDescriptor } from "../contracts/extensions/llm-provider.js"; import type { OutputRendererDescriptor } from "../contracts/extensions/output-renderer.js"; import type { SourceProviderDescriptor } from "../contracts/extensions/source-provider.js"; @@ -13,6 +14,7 @@ import type { PipelineStepDescriptor } from "../contracts/pipeline/step.js"; /** Descriptor records needed to construct an engine runtime. */ export interface EngineDescriptors { readonly sourceProviders: Readonly>; + readonly contentTransformers: Readonly>; readonly llmProviders: Readonly>; readonly outputRenderers: Readonly>; readonly pipelineSteps: Readonly>; diff --git a/src/engine/engine-runtime.ts b/src/engine/engine-runtime.ts index 7c7ae2c..36936dd 100644 --- a/src/engine/engine-runtime.ts +++ b/src/engine/engine-runtime.ts @@ -8,6 +8,7 @@ import { ConfigurationError } from "../shared/errors.js"; +import type { ContentTransformerRegistry } from "../contracts/extensions/content-transformer.js"; import type { LlmProviderRegistry } from "../contracts/extensions/llm-provider.js"; import type { OutputRendererRegistry } from "../contracts/extensions/output-renderer.js"; import type { SourceProviderRegistry } from "../contracts/extensions/source-provider.js"; @@ -17,6 +18,7 @@ import type { PipelineHandle, PipelineRunOutput } from "../contracts/pipeline/ha /** Registries of engine-owned configured runtime instances. */ export interface EngineRegistries { readonly sourceProviders: SourceProviderRegistry; + readonly contentTransformers: ContentTransformerRegistry; readonly llmProviders: LlmProviderRegistry; readonly outputRenderers: OutputRendererRegistry; } diff --git a/src/engine/internal/builders/build-content-transformers.ts b/src/engine/internal/builders/build-content-transformers.ts new file mode 100644 index 0000000..d83e9c9 --- /dev/null +++ b/src/engine/internal/builders/build-content-transformers.ts @@ -0,0 +1,44 @@ +/** + * Constructs configured content transformers for the engine registry. + * + * The generic registry builder handles descriptor lookup, config parsing, and + * identity wrapping. This file supplies content-transformer-specific error codes + * and logger identity fields. + */ + +import { buildNamedRegistry, type BuiltNameTracking } from "./build-named-registry.js"; + +import type { + ContentTransformerDescriptor, + ContentTransformerRegistry +} from "../../../contracts/extensions/content-transformer.js"; +import type { HostTools } from "../../../contracts/host/host-tools.js"; +import type { Logger } from "../../../shared/logger.js"; +import type { EngineConfig } from "../../engine-config.js"; + +/** Constructs content transformers from configured content transformer entries. */ +export function buildContentTransformers( + rawTransformers: EngineConfig["contentTransformers"], + tools: HostTools, + logger: Logger, + descriptors: Readonly> +): ContentTransformerRegistry & BuiltNameTracking { + return buildNamedRegistry({ + rawEntries: rawTransformers, + descriptors, + tools, + logger, + labels: { + logField: "content_transformer", + unknownTypeCode: "unknown_content_transformer_type", + unknownTypeMessage: type => `Unknown content transformer type: ${type}`, + invalidConfigCode: "invalid_content_transformer_config", + invalidConfigMessage: (name, type) => `Invalid config for content transformer '${name}' of type '${type}'`, + createFailedCode: "content_transformer_create_failed", + createFailedMessage: (name, type) => `Failed to create content transformer '${name}' of type '${type}'`, + missingCode: "unknown_content_transformer", + missingMessage: name => `Unknown content transformer: ${name}` + }, + resolve: ({ name, type }, transformer) => ({ name, type, transformer }) + }); +} diff --git a/src/shared/media-types.ts b/src/shared/media-types.ts index b397a54..f14d218 100644 --- a/src/shared/media-types.ts +++ b/src/shared/media-types.ts @@ -27,3 +27,9 @@ export function isTextLike(mediaType: string): boolean { normalized.endsWith("+xml") ); } + +/** Returns whether a media type is HTML or XHTML, ignoring parameters and case. */ +export function isHtmlMediaType(mediaType: string): boolean { + const essence = mediaType.split(";", 1)[0].trim().toLowerCase(); + return essence === mediaTypes.html || essence === mediaTypes.xhtml; +} diff --git a/tests/app/compose.test.ts b/tests/app/compose.test.ts index f6797b2..61b33f3 100644 --- a/tests/app/compose.test.ts +++ b/tests/app/compose.test.ts @@ -27,8 +27,8 @@ describe("composeApp", () => { httpFetch: async () => new Response("{}") }); - expect(loaded.registries.sourceProviders.require("default-firecrawl").name).toBe("default-firecrawl"); - expect(loaded.registries.llmProviders.require("default-llm").name).toBe("default-llm"); + expect(loaded.registries.sourceProviders.require("firecrawl-markdown").name).toBe("firecrawl-markdown"); + expect(loaded.registries.llmProviders.require("llm-default").name).toBe("llm-default"); expect( loaded.registries.pipelines.find(pipeline => pipeline.name === "default")?.steps.map(entry => entry.name) ).toEqual(["fetch", "clean", "truncate"]); @@ -45,7 +45,7 @@ describe("composeApp", () => { logger: createTestLogger(), httpFetch: fetch }) - ).rejects.toThrow("Invalid config for source provider 'default-firecrawl'"); + ).rejects.toThrow("Invalid config for source provider 'firecrawl-markdown'"); const missingTemplate = await writeConfigFixture(defaultYaml(), false); await expect( @@ -110,7 +110,7 @@ describe("composeApp", () => { }); await loaded.registries.llmProviders - .require("default-llm") + .require("llm-default") .provider.chat([], { signal: new AbortController().signal }); expect(requestBody?.model).toBe("${LLM_MODEL}"); @@ -123,8 +123,8 @@ describe("composeApp", () => { .replace("stripBase64Images: true", "stripBase64Images: ${FIRECRAWL_STRIP_IMAGES:-}") .replace("includeSkipped: true", "includeSkipped: ${DEBUG_XML_INCLUDE_SKIPPED:-}") .replace( - " config:\n provider: default-llm", - " timeoutSeconds: ${CLEAN_TIMEOUT_SECONDS:-}\n config:\n provider: default-llm" + " config:\n provider: llm-default", + " timeoutSeconds: ${CLEAN_TIMEOUT_SECONDS:-}\n config:\n provider: llm-default" ) .replace("minInputChars: 1", "minInputChars: ${MIN_INPUT_CHARS:-}") .replace("maxInputChars: 100", "maxInputChars: ${MAX_INPUT_CHARS:-}") @@ -159,7 +159,7 @@ describe("composeApp", () => { ).toBe(60); await loaded.registries.sourceProviders - .require("default-firecrawl") + .require("firecrawl-markdown") .provider.load("https://example.com", { signal: new AbortController().signal }); expect(firecrawlRequestBody).toMatchObject({ onlyMainContent: false, removeBase64Images: true }); @@ -243,7 +243,7 @@ function envValues(): NodeJS.ProcessEnv { function defaultYaml(): string { return `sourceProviders: - default-firecrawl: + firecrawl-markdown: type: firecrawl config: baseUrl: \${FIRECRAWL_BASE_URL} @@ -253,7 +253,7 @@ function defaultYaml(): string { stripBase64Images: true parsePdf: true llmProviders: - default-llm: + llm-default: type: openai-chat config: baseUrl: \${LLM_BASE_URL} @@ -277,12 +277,12 @@ pipelines: name: fetch concurrencyGroup: source config: - provider: default-firecrawl + provider: firecrawl-markdown - type: llm-pass name: clean concurrencyGroup: llm config: - provider: default-llm + provider: llm-default minInputChars: 1 maxInputChars: 100 templates: diff --git a/tests/architecture/import-boundaries.test.ts b/tests/architecture/import-boundaries.test.ts index 186141f..e822a35 100644 --- a/tests/architecture/import-boundaries.test.ts +++ b/tests/architecture/import-boundaries.test.ts @@ -24,6 +24,7 @@ interface ClassifiedSource { type SourceLayer = | "adapter-http" | "bundles" + | "builtins-content-transformer" | "builtins-http-adapter" | "builtins-output-renderer" | "builtins-pipeline-step" @@ -195,6 +196,8 @@ function classifySource(sourcePath: string): ClassifiedSource { if (sourcePath.startsWith("src/adapters/http/")) return { layer: "adapter-http" }; if (sourcePath.startsWith("src/builtins/source-providers/")) return { layer: "builtins-provider", component: providerComponent(sourcePath, "source") }; + if (sourcePath.startsWith("src/builtins/content-transformers/")) + return { layer: "builtins-content-transformer", component: contentTransformerComponent(sourcePath) }; if (sourcePath.startsWith("src/builtins/llm-providers/")) return { layer: "builtins-provider", component: providerComponent(sourcePath, "llm") }; if (sourcePath.startsWith("src/builtins/pipeline-steps/")) @@ -211,6 +214,7 @@ function isAllowedImport(edge: ImportEdge, source: ClassifiedSource, target: Cla switch (source.layer) { case "bundles": return [ + "builtins-content-transformer", "builtins-output-renderer", "builtins-pipeline-step", "builtins-provider", @@ -238,6 +242,8 @@ function isAllowedImport(edge: ImportEdge, source: ClassifiedSource, target: Cla ); case "builtins-provider": return ["contracts", "shared"].includes(target.layer) || isSameComponent(source, target); + case "builtins-content-transformer": + return ["contracts", "shared"].includes(target.layer) || isSameComponent(source, target); case "builtins-pipeline-step": return ["contracts", "shared"].includes(target.layer) || isSameComponent(source, target); case "builtins-output-renderer": @@ -282,9 +288,13 @@ function groupConcreteBuiltinImportsBySource(): Map> { } function isConcreteBuiltinLayer(layer: SourceLayer): boolean { - return ["builtins-http-adapter", "builtins-output-renderer", "builtins-pipeline-step", "builtins-provider"].includes( - layer - ); + return [ + "builtins-content-transformer", + "builtins-http-adapter", + "builtins-output-renderer", + "builtins-pipeline-step", + "builtins-provider" + ].includes(layer); } function isDescriptorBundleSource(sourcePath: string): boolean { @@ -310,6 +320,11 @@ function pipelineStepComponent(sourcePath: string): string | undefined { return segments.length > 4 ? segments[3] : undefined; } +function contentTransformerComponent(sourcePath: string): string | undefined { + const segments = sourcePath.split("/"); + return segments.length > 4 ? segments[3] : undefined; +} + function outputRendererComponent(sourcePath: string): string | undefined { const segments = sourcePath.split("/"); return segments.length > 4 ? segments[3] : undefined; diff --git a/tests/builtins/content-transformers/mdream/mdream-transformer-config.test.ts b/tests/builtins/content-transformers/mdream/mdream-transformer-config.test.ts new file mode 100644 index 0000000..8485060 --- /dev/null +++ b/tests/builtins/content-transformers/mdream/mdream-transformer-config.test.ts @@ -0,0 +1,18 @@ +/** Verifies the mdream content transformer config parsing. */ +import { describe, expect, it } from "vitest"; + +import { parseMdreamTransformerConfig } from "../../../../src/builtins/content-transformers/mdream/mdream-transformer-config.js"; + +describe("parseMdreamTransformerConfig", () => { + it("applies defaults", () => { + expect(parseMdreamTransformerConfig({})).toEqual({ minimal: false, clean: true }); + }); + + it("coerces env-substituted boolean strings", () => { + expect(parseMdreamTransformerConfig({ minimal: "true", clean: "false" })).toEqual({ minimal: true, clean: false }); + }); + + it("rejects unknown keys", () => { + expect(() => parseMdreamTransformerConfig({ unexpected: true })).toThrow(); + }); +}); diff --git a/tests/builtins/content-transformers/mdream/mdream-transformer.test.ts b/tests/builtins/content-transformers/mdream/mdream-transformer.test.ts new file mode 100644 index 0000000..40abe45 --- /dev/null +++ b/tests/builtins/content-transformers/mdream/mdream-transformer.test.ts @@ -0,0 +1,127 @@ +/** Verifies the mdream content transformer support matrix and conversion. */ +import { describe, expect, it } from "vitest"; + +import { MdreamTransformer } from "../../../../src/builtins/content-transformers/mdream/mdream-transformer.js"; +import { InternalError } from "../../../../src/shared/errors.js"; +import { mediaTypes } from "../../../../src/shared/media-types.js"; +import { createTestLogger } from "../../../helpers/logger.js"; + +import type { BodyContent } from "../../../../src/contracts/pipeline/context.js"; + +function makeTransformer(config: { minimal?: boolean; clean?: boolean } = {}) { + return new MdreamTransformer( + { minimal: config.minimal ?? false, clean: config.clean ?? true }, + { + logger: createTestLogger() + } + ); +} + +const markdownRequest = { targetMediaType: mediaTypes.markdown }; + +describe("MdreamTransformer.supports", () => { + it("supports HTML and XHTML text bodies targeting markdown", () => { + const transformer = makeTransformer(); + expect(transformer.supports({ sourceKind: "text", sourceMediaType: "text/html", request: markdownRequest })).toBe( + true + ); + expect( + transformer.supports({ sourceKind: "text", sourceMediaType: "application/xhtml+xml", request: markdownRequest }) + ).toBe(true); + expect( + transformer.supports({ + sourceKind: "text", + sourceMediaType: "text/html; charset=utf-8", + request: markdownRequest + }) + ).toBe(true); + }); + + it("rejects non-HTML sources, non-text kinds, and non-markdown targets", () => { + const transformer = makeTransformer(); + expect( + transformer.supports({ sourceKind: "text", sourceMediaType: "text/markdown", request: markdownRequest }) + ).toBe(false); + expect(transformer.supports({ sourceKind: "binary", sourceMediaType: "text/html", request: markdownRequest })).toBe( + false + ); + expect( + transformer.supports({ + sourceKind: "text", + sourceMediaType: "text/html", + request: { targetMediaType: "text/html" } + }) + ).toBe(false); + }); +}); + +describe("MdreamTransformer.transform", () => { + const signal = new AbortController().signal; + + it("converts HTML to markdown, resolves relative links via origin, and preserves the title", async () => { + const transformer = makeTransformer(); + const body: BodyContent = { + kind: "text", + mediaType: "text/html", + content: '

Hello

World link

', + title: "Doc" + }; + + const result = await transformer.transform( + { url: "https://example.com", body, request: markdownRequest }, + { signal } + ); + + expect(result.body.kind).toBe("text"); + expect(result.body.mediaType).toBe(mediaTypes.markdown); + expect(result.body.title).toBe("Doc"); + if (result.body.kind === "text") { + expect(result.body.content).toContain("# Hello"); + expect(result.body.content).toContain("](https://example.com"); + } + }); + + it("emits a summary diagnostic and flags empty output", async () => { + const transformer = makeTransformer(); + const empty = await transformer.transform( + { + url: "https://example.com/", + body: { kind: "text", mediaType: "text/html", content: "
" }, + request: markdownRequest + }, + { signal } + ); + expect(empty.diagnostics?.some(diagnostic => diagnostic.code === "empty_output")).toBe(true); + expect(empty.diagnostics?.some(diagnostic => diagnostic.code === "mdream")).toBe(true); + }); + + it("throws when given a non-text body", async () => { + const transformer = makeTransformer(); + await expect( + transformer.transform( + { + url: "https://example.com/", + body: { kind: "binary", mediaType: "application/pdf", bytes: new Uint8Array([1]) }, + request: markdownRequest + }, + { signal } + ) + ).rejects.toBeInstanceOf(InternalError); + }); + + it("throws if the signal is already aborted", async () => { + const transformer = makeTransformer(); + const controller = new AbortController(); + controller.abort(); + await expect( + transformer.transform( + { + url: "https://example.com/", + body: { kind: "text", mediaType: "text/html", content: "

hi

" }, + request: markdownRequest + }, + { signal: controller.signal } + ) + ).rejects.toBeDefined(); + }); +}); diff --git a/tests/builtins/pipeline-steps/llm-pass/llm-pass-step-config.test.ts b/tests/builtins/pipeline-steps/llm-pass/llm-pass-step-config.test.ts index 2f45a6c..c96384a 100644 --- a/tests/builtins/pipeline-steps/llm-pass/llm-pass-step-config.test.ts +++ b/tests/builtins/pipeline-steps/llm-pass/llm-pass-step-config.test.ts @@ -4,7 +4,7 @@ import { describe, expect, it } from "vitest"; import { parseLlmPassStepConfig } from "../../../../src/builtins/pipeline-steps/llm-pass/llm-pass-step-config.js"; const baseConfig = { - provider: "default-llm", + provider: "llm-default", templates: { system: "system", user: "user" } }; diff --git a/tests/builtins/pipeline-steps/transform/transform-step-config.test.ts b/tests/builtins/pipeline-steps/transform/transform-step-config.test.ts new file mode 100644 index 0000000..b1ba992 --- /dev/null +++ b/tests/builtins/pipeline-steps/transform/transform-step-config.test.ts @@ -0,0 +1,33 @@ +/** Verifies the transform pipeline step config parsing. */ +import { describe, expect, it } from "vitest"; + +import { parseTransformStepConfig } from "../../../../src/builtins/pipeline-steps/transform/transform-step-config.js"; + +describe("parseTransformStepConfig", () => { + it("applies defaults for optional fields", () => { + expect(parseTransformStepConfig({ transformer: "mdream-default", target: "text/markdown" })).toEqual({ + transformer: "mdream-default", + target: "text/markdown", + onUnsupported: "skip", + emitDiagnostics: false + }); + }); + + it("requires transformer and target", () => { + expect(() => parseTransformStepConfig({ target: "text/markdown" })).toThrow(); + expect(() => parseTransformStepConfig({ transformer: "x" })).toThrow(); + }); + + it("validates the onUnsupported enum and coerces emitDiagnostics", () => { + expect(() => + parseTransformStepConfig({ transformer: "x", target: "text/markdown", onUnsupported: "explode" }) + ).toThrow(); + expect( + parseTransformStepConfig({ transformer: "x", target: "text/markdown", emitDiagnostics: "true" }).emitDiagnostics + ).toBe(true); + }); + + it("rejects unknown keys", () => { + expect(() => parseTransformStepConfig({ transformer: "x", target: "text/markdown", extra: 1 })).toThrow(); + }); +}); diff --git a/tests/builtins/pipeline-steps/transform/transform-step.test.ts b/tests/builtins/pipeline-steps/transform/transform-step.test.ts new file mode 100644 index 0000000..1fb9178 --- /dev/null +++ b/tests/builtins/pipeline-steps/transform/transform-step.test.ts @@ -0,0 +1,165 @@ +/** Verifies the transform pipeline step routing, honesty gate, and diagnostics. */ +import { describe, expect, it } from "vitest"; + +import { TransformStep } from "../../../../src/builtins/pipeline-steps/transform/transform-step.js"; +import { createTestLogger } from "../../../helpers/logger.js"; +import { makeStepContext } from "../utils.js"; + +import type { + ContentTransformer, + ContentTransformResult +} from "../../../../src/contracts/extensions/content-transformer.js"; +import type { TransformStepOptions } from "../../../../src/builtins/pipeline-steps/transform/transform-step-config.js"; + +const baseConfig: TransformStepOptions = { target: "text/markdown", onUnsupported: "skip", emitDiagnostics: false }; +const markdownResult: ContentTransformResult = { body: { kind: "text", mediaType: "text/markdown", content: "# md" } }; + +function fakeTransformer(overrides: Partial = {}): ContentTransformer { + return { + supports: () => true, + transform: async () => markdownResult, + ...overrides + }; +} + +function makeStep(config: TransformStepOptions, transformer: ContentTransformer): TransformStep { + return new TransformStep(config, { transformer, logger: createTestLogger() }); +} + +const htmlBody = { content: "

md

", mediaType: "text/html" }; + +describe("TransformStep", () => { + it("skips when there is no body", async () => { + const step = makeStep( + baseConfig, + fakeTransformer({ + supports: () => { + throw new Error("supports must not be called without a body"); + } + }) + ); + + await expect(step.run(makeStepContext())).resolves.toMatchObject({ status: "skipped", reason: "no_body" }); + }); + + it("skips an unsupported body when onUnsupported is skip", async () => { + const step = makeStep(baseConfig, fakeTransformer({ supports: () => false })); + + await expect(step.run(makeStepContext({ body: htmlBody }))).resolves.toMatchObject({ + status: "skipped", + reason: "unsupported" + }); + }); + + it("fails an unsupported body when onUnsupported is fail", async () => { + const step = makeStep({ ...baseConfig, onUnsupported: "fail" }, fakeTransformer({ supports: () => false })); + + await expect(step.run(makeStepContext({ body: htmlBody }))).resolves.toMatchObject({ + status: "failed", + reason: "unsupported" + }); + }); + + it("applies the transformed body on success without diagnostics by default", async () => { + const step = makeStep(baseConfig, fakeTransformer()); + + const result = await step.run(makeStepContext({ body: htmlBody })); + + expect(result.status).toBe("ok"); + expect(result.effects?.body).toEqual({ kind: "text", mediaType: "text/markdown", content: "# md" }); + expect(result.diagnostics).toBeUndefined(); + }); + + it("passes the url and requested target to the transformer", async () => { + let captured: { url: string; request: { targetMediaType: string } } | undefined; + const step = makeStep( + baseConfig, + fakeTransformer({ + transform: async input => { + captured = { url: input.url, request: input.request }; + return markdownResult; + } + }) + ); + + await step.run(makeStepContext({ body: htmlBody, url: "https://example.com/page" })); + + expect(captured).toEqual({ url: "https://example.com/page", request: { targetMediaType: "text/markdown" } }); + }); + + it("surfaces transformer diagnostics as children when emitDiagnostics is true", async () => { + const step = makeStep( + { ...baseConfig, emitDiagnostics: true }, + fakeTransformer({ + transform: async () => ({ + body: markdownResult.body, + diagnostics: [{ code: "mdream", message: "10 -> 4" }, { code: "empty_output" }] + }) + }) + ); + + const result = await step.run(makeStepContext({ body: htmlBody })); + + expect(result.diagnostics).toEqual({ + children: [ + { name: "mdream", attributes: { message: "10 -> 4" } }, + { name: "empty_output", attributes: undefined } + ] + }); + }); + + it("fails when the transformer returns a different media type", async () => { + const step = makeStep( + baseConfig, + fakeTransformer({ transform: async () => ({ body: { kind: "text", mediaType: "text/plain", content: "x" } }) }) + ); + + await expect(step.run(makeStepContext({ body: htmlBody }))).resolves.toMatchObject({ + status: "failed", + reason: "wrong_output_type" + }); + }); + + it("fails when the transformer returns the wrong representation for a text target", async () => { + const step = makeStep( + baseConfig, + fakeTransformer({ + transform: async () => ({ body: { kind: "binary", mediaType: "text/markdown", bytes: new Uint8Array([1]) } }) + }) + ); + + await expect(step.run(makeStepContext({ body: htmlBody }))).resolves.toMatchObject({ + status: "failed", + reason: "wrong_output_type" + }); + }); + + it("maps an abort during transform to a timeout failure", async () => { + const step = makeStep( + baseConfig, + fakeTransformer({ + transform: async () => { + throw new DOMException("Aborted", "AbortError"); + } + }) + ); + + await expect(step.run(makeStepContext({ body: htmlBody }))).resolves.toMatchObject({ + status: "failed", + reason: "timeout" + }); + }); + + it("rethrows non-abort transformer errors", async () => { + const step = makeStep( + baseConfig, + fakeTransformer({ + transform: async () => { + throw new Error("boom"); + } + }) + ); + + await expect(step.run(makeStepContext({ body: htmlBody }))).rejects.toThrow("boom"); + }); +}); diff --git a/tests/bundles/default-engine-descriptors.test.ts b/tests/bundles/default-engine-descriptors.test.ts index 8614c2f..6eb01e2 100644 --- a/tests/bundles/default-engine-descriptors.test.ts +++ b/tests/bundles/default-engine-descriptors.test.ts @@ -4,7 +4,12 @@ import { describe, expect, it } from "vitest"; import type { LlmProvider, LlmProviderRegistry } from "../../src/contracts/extensions/llm-provider.js"; import type { ResourceLoader } from "../../src/contracts/host/host-tools.js"; import type { SourceProvider, SourceProviderRegistry } from "../../src/contracts/extensions/source-provider.js"; +import type { + ContentTransformer, + ContentTransformerRegistry +} from "../../src/contracts/extensions/content-transformer.js"; import { DEFAULT_ENGINE_DESCRIPTOR_BUNDLE } from "../../src/bundles/default-engine-descriptors.js"; +import { contentTransformerRegistryKey } from "../../src/contracts/extensions/content-transformer.js"; import { llmProviderRegistryKey } from "../../src/contracts/extensions/llm-provider.js"; import { sourceProviderRegistryKey } from "../../src/contracts/extensions/source-provider.js"; import { createExtensionServicesBuilder } from "../../src/engine/internal/extension-services.js"; @@ -16,7 +21,7 @@ describe("default engine descriptor bundle", () => { const firecrawl = DEFAULT_ENGINE_DESCRIPTOR_BUNDLE.sourceProviders.firecrawl; const firecrawlConfig = firecrawl.parseConfig({ baseUrl: "https://firecrawl.example" }); const firecrawlProvider = await firecrawl.create({ - name: "default-firecrawl", + name: "firecrawl-markdown", config: firecrawlConfig, deps: { tools: createTestHostTools({ httpFetch: async () => new Response("{}") }), logger: createTestLogger() } }); @@ -24,7 +29,7 @@ describe("default engine descriptor bundle", () => { const openAi = DEFAULT_ENGINE_DESCRIPTOR_BUNDLE.llmProviders["openai-chat"]; const openAiConfig = openAi.parseConfig({ baseUrl: "https://llm.example/v1", model: "model" }); const openAiProvider = await openAi.create({ - name: "default-llm", + name: "llm-default", config: openAiConfig, deps: { tools: createTestHostTools({ httpFetch: async () => new Response("{}") }), logger: createTestLogger() } }); @@ -33,6 +38,19 @@ describe("default engine descriptor bundle", () => { expect(openAiProvider.chat).toEqual(expect.any(Function)); }); + it("round-trips the mdream content transformer descriptor", async () => { + const mdream = DEFAULT_ENGINE_DESCRIPTOR_BUNDLE.contentTransformers.mdream; + const config = mdream.parseConfig({}); + const transformer = await mdream.create({ + name: "mdream-default", + config, + deps: { tools: createTestHostTools(), logger: createTestLogger() } + }); + + expect(transformer.supports).toEqual(expect.any(Function)); + expect(transformer.transform).toEqual(expect.any(Function)); + }); + it("round-trips step descriptors through parse and create", async () => { const services = servicesWithProviders(sourceProvider(), llmProvider()); const deps = { @@ -72,10 +90,23 @@ describe("default engine descriptor bundle", () => { services, deps }); + const transformConfig = DEFAULT_ENGINE_DESCRIPTOR_BUNDLE.pipelineSteps.transform.parseConfig({ + transformer: "content-transformer", + target: "text/markdown" + }); + const transformStep = await DEFAULT_ENGINE_DESCRIPTOR_BUNDLE.pipelineSteps.transform.create({ + name: "transform", + type: "transform", + timeoutSeconds: 60, + config: transformConfig, + services, + deps + }); expect(loadSourceStep.run).toEqual(expect.any(Function)); expect(llmStep.run).toEqual(expect.any(Function)); expect(truncateStep.run).toEqual(expect.any(Function)); + expect(transformStep.run).toEqual(expect.any(Function)); }); }); @@ -91,9 +122,31 @@ function servicesWithProviders(sourceProvider: SourceProvider, llmProvider: LlmP const builder = createExtensionServicesBuilder(); builder.register(sourceProviderRegistryKey, providerRegistry("source-provider", sourceProvider)); builder.register(llmProviderRegistryKey, providerRegistry("llm-provider", llmProvider)); + builder.register( + contentTransformerRegistryKey, + contentTransformerRegistry("content-transformer", contentTransformer()) + ); return builder.build(); } +function contentTransformer(): ContentTransformer { + return { + supports: () => true, + transform: async () => ({ body: { kind: "text", mediaType: "text/markdown", content: "md" } }) + }; +} + +function contentTransformerRegistry(expectedName: string, transformer: ContentTransformer): ContentTransformerRegistry { + const wrap = { name: expectedName, type: "test", transformer }; + return { + require: (name: string) => { + if (name !== expectedName) throw new Error(`Unknown transformer: ${name}`); + return wrap; + }, + tryGet: (name: string) => (name === expectedName ? wrap : undefined) + }; +} + function providerRegistry( expectedName: string, provider: TProvider diff --git a/tests/engine/activation-lazy.test.ts b/tests/engine/activation-lazy.test.ts index b393a98..46d3798 100644 --- a/tests/engine/activation-lazy.test.ts +++ b/tests/engine/activation-lazy.test.ts @@ -18,6 +18,7 @@ describe("lazy provider/registry activation", () => { sourceProviders: { "bad-provider": { type: "validated-source", config: { requiredField: "" } } }, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: { type: "markdown", config: {} } }, pipelines: { @@ -41,6 +42,7 @@ describe("lazy provider/registry activation", () => { } } satisfies SourceProviderDescriptor }, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: markdownRendererDescriptor }, pipelineSteps: { @@ -60,6 +62,7 @@ describe("lazy provider/registry activation", () => { sourceProviders: { "bad-provider": { type: "validated-source", config: { requiredField: "" } } }, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: { type: "markdown", config: {} } }, pipelines: { @@ -83,6 +86,7 @@ describe("lazy provider/registry activation", () => { } } satisfies SourceProviderDescriptor }, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: markdownRendererDescriptor }, pipelineSteps: { diff --git a/tests/engine/activation-pipeline.test.ts b/tests/engine/activation-pipeline.test.ts index 84d7848..b8ff127 100644 --- a/tests/engine/activation-pipeline.test.ts +++ b/tests/engine/activation-pipeline.test.ts @@ -30,6 +30,7 @@ describe("pipeline activation", () => { const engine = await createEngine({ config: { sourceProviders: {}, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: { type: "markdown", config: {} } }, pipelines: { @@ -49,6 +50,7 @@ describe("pipeline activation", () => { }, descriptors: { sourceProviders: {}, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: markdownRendererDescriptor }, pipelineSteps: { "static-body": staticBodyStepDescriptor } @@ -75,12 +77,14 @@ describe("pipeline activation", () => { await createEngine({ config: { sourceProviders: { "never-used": { type: "never-built", config: {} } }, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: { type: "markdown", config: {} } }, pipelines: {} }, descriptors: { sourceProviders: { "never-built": sourceProv }, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: markdownRendererDescriptor }, pipelineSteps: {} diff --git a/tests/engine/create-engine.test.ts b/tests/engine/create-engine.test.ts index 807a995..46f8369 100644 --- a/tests/engine/create-engine.test.ts +++ b/tests/engine/create-engine.test.ts @@ -13,6 +13,7 @@ describe("createEngine", () => { const engine = await createEngine({ config: { sourceProviders: {}, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: { type: "markdown", config: {} } }, pipelines: { @@ -26,6 +27,7 @@ describe("createEngine", () => { }, descriptors: { sourceProviders: {}, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: markdownRendererDescriptor }, pipelineSteps: { "static-body": staticBodyStepDescriptor } @@ -54,8 +56,14 @@ describe("createEngine", () => { it("reports unknown pipelines clearly", async () => { const engine = await createEngine({ - config: { sourceProviders: {}, llmProviders: {}, outputRenderers: {}, pipelines: {} }, - descriptors: { sourceProviders: {}, llmProviders: {}, outputRenderers: {}, pipelineSteps: {} }, + config: { sourceProviders: {}, contentTransformers: {}, llmProviders: {}, outputRenderers: {}, pipelines: {} }, + descriptors: { + sourceProviders: {}, + contentTransformers: {}, + llmProviders: {}, + outputRenderers: {}, + pipelineSteps: {} + }, tools: createTestHostTools(), logger: createTestLogger() }); From 2e00fa90111c338308c98c80db6eb914b9d51d40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Sou=C5=A1ek?= <15769039+sousekd@users.noreply.github.com> Date: Fri, 12 Jun 2026 16:45:13 +0200 Subject: [PATCH 2/2] Add readability transformer: declined outcome, clean-combined pipeline, truthful Firecrawl media types --- .env.example | 48 +++-- AGENTS.md | 10 + README.md | 4 +- compose.deploy.yaml | 10 +- compose.yaml | 10 +- config/llm-context-loader.yaml | 93 +++++++- docs/ARCHITECTURE.md | 2 +- docs/CONFIGURATION.md | 45 ++-- docs/CUSTOMIZATION.md | 78 +++++-- docs/ROADMAP.md | 36 ++-- docs/agents/extension-authoring.md | 11 +- docs/agents/smoke-testing.md | 18 +- package-lock.json | 197 +++++++++++++++++ package.json | 2 + .../mdream/mdream-transformer-config.ts | 6 + .../mdream/mdream-transformer.ts | 1 + .../readability-transformer-config.ts | 31 +++ .../readability-transformer-descriptor.ts | 20 ++ .../readability/readability-transformer.ts | 95 ++++++++ .../transform/transform-step-config.ts | 2 + .../transform/transform-step-descriptor.ts | 1 + .../transform/transform-step.ts | 3 + .../firecrawl/firecrawl-provider.ts | 52 ++++- src/bundles/default-engine-descriptors.ts | 6 +- .../extensions/content-transformer.ts | 12 +- src/shared/media-types.ts | 5 + .../mdream/mdream-transformer.test.ts | 11 +- .../readability-transformer-config.test.ts | 50 +++++ .../readability-transformer.test.ts | 172 +++++++++++++++ .../transform/transform-step-config.test.ts | 19 +- .../transform/transform-step.test.ts | 59 ++++- .../firecrawl/firecrawl-provider.test.ts | 204 ++++++++++++++++++ .../default-engine-descriptors.test.ts | 20 +- tests/shared/media-types.test.ts | 36 +++- 34 files changed, 1252 insertions(+), 117 deletions(-) create mode 100644 src/builtins/content-transformers/readability/readability-transformer-config.ts create mode 100644 src/builtins/content-transformers/readability/readability-transformer-descriptor.ts create mode 100644 src/builtins/content-transformers/readability/readability-transformer.ts create mode 100644 tests/builtins/content-transformers/readability/readability-transformer-config.test.ts create mode 100644 tests/builtins/content-transformers/readability/readability-transformer.test.ts diff --git a/.env.example b/.env.example index 99aa713..ae8d489 100644 --- a/.env.example +++ b/.env.example @@ -22,27 +22,30 @@ LOG_LEVEL=info LOG_PRETTY=auto # --- Pipeline ------------------------------------------------------------ -# Active pipeline selected by HTTP adapters: truncate, clean-deterministic, or clean-llm. Default: truncate (load-source + truncate) -DEFAULT_PIPELINE=truncate +# Active pipeline selected by HTTP adapters. Default: truncate (load-source + truncate) +DEFAULT_PIPELINE=clean-combined -# Maximum concurrent source-loading groups. Default: 1 -SOURCE_CONCURRENCY=4 +# Maximum concurrent source-loading groups. Default: 1; raise only if providers can handle it. +SOURCE_CONCURRENCY=5 -# Maximum concurrent LLM workflow groups. Default: 1 +# Maximum concurrent processing (transform) groups. Default: 5 +PROCESS_CONCURRENCY=5 + +# Maximum concurrent LLM workflow groups. Default: 1; raise only if the model endpoint can handle it. LLM_CONCURRENCY=2 -# Output renderer used by the default pipeline. Default: debug-xml +# Output renderer override used by shipped pipelines. Default: debug-xml DEFAULT_OUTPUT_RENDERER=debug-xml # Include skipped steps in the debug-xml footer. Default: false DEBUG_XML_INCLUDE_SKIPPED=false -# Desired maximum output size used by summarize and truncate steps. Default: 25000 +# Desired maximum output size used by summarize and truncate steps. Default: 25000; larger values return more content. OUTPUT_TARGET_CHARS=35000 # --- Source provider ----------------------------------------------------- -# Active source provider name used by pipelines (truncate, clean-deterministic, clean-llm). Default: http-default (native HTTP fetch, no external service needed) -SOURCE_PROVIDER=http-default +# Optional source-provider override. Leave empty to use the selected pipeline's fallback. +SOURCE_PROVIDER= # Firecrawl base URL. Required only when a referenced provider uses it (e.g. firecrawl-markdown, firecrawl-html). FIRECRAWL_BASE_URL=https://firecrawl.example @@ -50,19 +53,28 @@ FIRECRAWL_BASE_URL=https://firecrawl.example # Optional Firecrawl bearer token. Default: empty (no token) FIRECRAWL_API_KEY= -# Docling Serve base URL. Required only when a referenced provider uses it (e.g. docling-default). +# Firecrawl PDF parsing. Set to false to return raw PDF bytes (binary body) instead of parsed text. +FIRECRAWL_PARSE_PDF= + +# Docling Serve base URL. Required only when a referenced provider uses it (e.g. docling-ocr). DOCLING_BASE_URL=https://docling.example # Optional Docling API key (X-Api-Key header). Default: empty (no token) DOCLING_API_KEY= -# --- Content transformer ------------------------------------------------- -# Isolate main content and filter boilerplate before conversion. Default: true -MDREAM_MINIMAL=true - -# Post-conversion link and whitespace cleanup. Default: true +# --- Content transformers ------------------------------------------------ +# Post-conversion link and whitespace cleanup for mdream-convert. Ignored when minimal=true (mdream-aggressive). Default: true MDREAM_CLEAN=true +# Minimum content length for Readability's isProbablyReaderable gate. Default: 140 +READABILITY_MIN_CONTENT_CHARS=140 + +# Minimum readerable score for Readability's isProbablyReaderable gate. Default: 20 +READABILITY_MIN_SCORE=20 + +# Maximum DOM elements Readability parses; 0 = unlimited. Default: 0 +READABILITY_MAX_ELEMENTS=0 + # --- LLM provider -------------------------------------------------------- # OpenAI-compatible Chat Completions /v1 base URL. Required only when a referenced provider uses it (e.g. llm-default). LLM_BASE_URL=https://openai-compatible.example/v1 @@ -73,7 +85,7 @@ LLM_API_KEY= # Model identifier sent to the chat-completions endpoint. Required only when a referenced provider uses it (e.g. llm-default). LLM_MODEL=model-name -# Model context window in tokens. Default: empty (disables the context-fit gate) +# Model context window in tokens. Default: empty (disables the context-fit gate); set to enable it. LLM_CONTEXT_TOKENS=131072 # Conservative chars-per-token estimate used by the context-fit gate. Default: 3.5 @@ -86,10 +98,10 @@ LLM_SAFETY_MARGIN_TOKENS=128 # Timeout in seconds for the source-loading step. Default: 20 LOAD_SOURCE_TIMEOUT_SECONDS=20 -# Timeout in seconds for the clean LLM pass. Default: 60 +# Timeout in seconds for the clean LLM pass. Default: 60; raise for slower models. CLEAN_TIMEOUT_SECONDS=90 -# Minimum body length before the clean LLM pass runs. Default: 1000 +# Minimum body length before the clean LLM pass runs. Default: 1000; lower values run cleanup more often. CLEAN_MIN_INPUT_CHARS=500 # Timeout in seconds for the summarize LLM pass. Default: 60 diff --git a/AGENTS.md b/AGENTS.md index 38c1297..58e1446 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,3 +37,13 @@ Read the row that matches your task. Each focused doc is self-contained for its - When changing behavior, keep the docs that describe it accurate in the same change. - No gratuitous comments. Inline comments that narrate a change are prohibited ([coding-conventions](docs/agents/coding-conventions.md)). Before adding a comment, ask: will it still be useful after the transient context is gone? - Don't over-document trivial changes. A new knob, helper, or internal refactor does not warrant updates to `ARCHITECTURE.md` or `extension-authoring.md`. Update only docs that directly describe the changed surface. + +## Terminal Behavior on Windows + +When using the terminal on Windows: + +- Prefer single-line PowerShell commands. +- Avoid interactive commands, pagers, prompts, and commands that wait for input. +- Prefer `pwsh` or PowerShell with `-NoLogo -NoProfile`. +- For git commands, use non-interactive flags such as `--no-pager` and `--no-edit` when appropriate. +- If a command appears stuck after output is printed, do not blindly rerun it. First check whether the command already completed and inspect the visible terminal output. diff --git a/README.md b/README.md index f6b7bfe..e434261 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ LLM Context Loader is a small HTTP service that turns URLs into markdown for LLM ## Current Shape -- **Pipelines:** `truncate` (default, no LLM), `clean-deterministic` (Firecrawl HTML + mdream, no LLM), and `clean-llm` (Firecrawl + LLM), selected via `DEFAULT_PIPELINE`. +- **Pipelines:** `truncate` (default, no LLM), `clean-deterministic` (Firecrawl HTML + Readability + mdream, no LLM), `clean-llm` (Firecrawl + LLM), and `clean-combined` (Firecrawl HTML + Readability + mdream + LLM summarize), selected via `DEFAULT_PIPELINE`. - **Source providers:** native HTTP fetch, Firecrawl, Docling. -- **Content transformers:** `mdream` (HTML to markdown). +- **Content transformers:** `readability` (article HTML extraction), `mdream` (HTML to markdown). - **LLM providers:** OpenAI-compatible `/chat/completions`. - **Output renderers:** `debug-xml` and `passthrough`, selected per pipeline in YAML. diff --git a/compose.deploy.yaml b/compose.deploy.yaml index 48e8d5f..81fbae5 100644 --- a/compose.deploy.yaml +++ b/compose.deploy.yaml @@ -15,21 +15,25 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l # Pipeline concurrency and renderer selection. SOURCE_CONCURRENCY: "${SOURCE_CONCURRENCY:-1}" + PROCESS_CONCURRENCY: "${PROCESS_CONCURRENCY:-5}" LLM_CONCURRENCY: "${LLM_CONCURRENCY:-1}" DEFAULT_OUTPUT_RENDERER: "${DEFAULT_OUTPUT_RENDERER:-debug-xml}" DEBUG_XML_INCLUDE_SKIPPED: "${DEBUG_XML_INCLUDE_SKIPPED:-false}" OUTPUT_TARGET_CHARS: "${OUTPUT_TARGET_CHARS:-25000}" # Source provider. - SOURCE_PROVIDER: "${SOURCE_PROVIDER:-http-default}" + SOURCE_PROVIDER: "${SOURCE_PROVIDER:-}" FIRECRAWL_BASE_URL: "${FIRECRAWL_BASE_URL:-}" FIRECRAWL_API_KEY: "${FIRECRAWL_API_KEY:-}" + FIRECRAWL_PARSE_PDF: "${FIRECRAWL_PARSE_PDF:-}" DOCLING_BASE_URL: "${DOCLING_BASE_URL:-}" DOCLING_API_KEY: "${DOCLING_API_KEY:-}" - # Content transformer. - MDREAM_MINIMAL: "${MDREAM_MINIMAL:-true}" + # Content transformers. MDREAM_CLEAN: "${MDREAM_CLEAN:-true}" + READABILITY_MIN_CONTENT_CHARS: "${READABILITY_MIN_CONTENT_CHARS:-140}" + READABILITY_MIN_SCORE: "${READABILITY_MIN_SCORE:-20}" + READABILITY_MAX_ELEMENTS: "${READABILITY_MAX_ELEMENTS:-0}" # LLM provider. LLM_BASE_URL: "${LLM_BASE_URL:-}" diff --git a/compose.yaml b/compose.yaml index b98c586..2530eeb 100644 --- a/compose.yaml +++ b/compose.yaml @@ -15,21 +15,25 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l # Pipeline concurrency and renderer selection. SOURCE_CONCURRENCY: "${SOURCE_CONCURRENCY:-1}" + PROCESS_CONCURRENCY: "${PROCESS_CONCURRENCY:-5}" LLM_CONCURRENCY: "${LLM_CONCURRENCY:-1}" DEFAULT_OUTPUT_RENDERER: "${DEFAULT_OUTPUT_RENDERER:-debug-xml}" DEBUG_XML_INCLUDE_SKIPPED: "${DEBUG_XML_INCLUDE_SKIPPED:-false}" OUTPUT_TARGET_CHARS: "${OUTPUT_TARGET_CHARS:-25000}" # Source provider. - SOURCE_PROVIDER: "${SOURCE_PROVIDER:-http-default}" + SOURCE_PROVIDER: "${SOURCE_PROVIDER:-}" FIRECRAWL_BASE_URL: "${FIRECRAWL_BASE_URL:-}" FIRECRAWL_API_KEY: "${FIRECRAWL_API_KEY:-}" + FIRECRAWL_PARSE_PDF: "${FIRECRAWL_PARSE_PDF:-}" DOCLING_BASE_URL: "${DOCLING_BASE_URL:-}" DOCLING_API_KEY: "${DOCLING_API_KEY:-}" - # Content transformer. - MDREAM_MINIMAL: "${MDREAM_MINIMAL:-true}" + # Content transformers. MDREAM_CLEAN: "${MDREAM_CLEAN:-true}" + READABILITY_MIN_CONTENT_CHARS: "${READABILITY_MIN_CONTENT_CHARS:-140}" + READABILITY_MIN_SCORE: "${READABILITY_MIN_SCORE:-20}" + READABILITY_MAX_ELEMENTS: "${READABILITY_MAX_ELEMENTS:-0}" # LLM provider. LLM_BASE_URL: "${LLM_BASE_URL:-}" diff --git a/config/llm-context-loader.yaml b/config/llm-context-loader.yaml index 58b1a65..cd617a6 100644 --- a/config/llm-context-loader.yaml +++ b/config/llm-context-loader.yaml @@ -26,7 +26,7 @@ outputRenderers: type: passthrough config: {} -# Source providers turn an input URL into source markdown. +# Source providers turn an input URL into a pipeline body. sourceProviders: http-default: type: http @@ -40,7 +40,7 @@ sourceProviders: baseUrl: ${FIRECRAWL_BASE_URL:-} apiKey: ${FIRECRAWL_API_KEY:-} output: rawHtml - parsePdf: true + parsePdf: ${FIRECRAWL_PARSE_PDF:-true} firecrawl-markdown: type: firecrawl config: @@ -49,8 +49,8 @@ sourceProviders: output: markdown onlyMainContent: true stripBase64Images: true - parsePdf: true - docling-default: + parsePdf: ${FIRECRAWL_PARSE_PDF:-true} + docling-ocr: type: docling config: baseUrl: ${DOCLING_BASE_URL:-} @@ -61,11 +61,20 @@ sourceProviders: # Content transformers rewrite the body between representations (e.g. HTML to markdown). contentTransformers: - mdream-default: + readability-default: + type: readability + config: + minContentLength: ${READABILITY_MIN_CONTENT_CHARS:-140} + minScore: ${READABILITY_MIN_SCORE:-20} + maxElements: ${READABILITY_MAX_ELEMENTS:-0} + mdream-convert: type: mdream config: - minimal: ${MDREAM_MINIMAL:-true} clean: ${MDREAM_CLEAN:-true} + mdream-aggressive: + type: mdream + config: + minimal: true # LLM providers run prompt-rendered chat-completions passes. llmProviders: @@ -98,11 +107,11 @@ pipelines: config: targetChars: ${OUTPUT_TARGET_CHARS:-25000} - # Deterministic HTML-to-markdown path: fetch HTML, convert with mdream, then truncate. clean-deterministic: outputRenderer: ${DEFAULT_OUTPUT_RENDERER:-debug-xml} limiters: source: ${SOURCE_CONCURRENCY:-1} + process: ${PROCESS_CONCURRENCY:-5} steps: - type: load-source name: fetch @@ -111,11 +120,18 @@ pipelines: config: provider: ${SOURCE_PROVIDER:-firecrawl-html} - type: transform - name: html_to_markdown + name: clean + concurrencyGroup: process + config: + transformer: readability-default + target: text/html + emitDiagnostics: true + - type: transform + name: convert + concurrencyGroup: process config: - transformer: mdream-default + transformer: mdream-convert target: text/markdown - onUnsupported: skip emitDiagnostics: true - type: truncate name: truncate @@ -181,3 +197,60 @@ pipelines: name: truncate config: targetChars: ${OUTPUT_TARGET_CHARS:-25000} + + clean-combined: + outputRenderer: ${DEFAULT_OUTPUT_RENDERER:-debug-xml} + limiters: + source: ${SOURCE_CONCURRENCY:-1} + process: ${PROCESS_CONCURRENCY:-5} + llm: ${LLM_CONCURRENCY:-1} + steps: + - type: load-source + name: fetch + concurrencyGroup: source + timeoutSeconds: ${LOAD_SOURCE_TIMEOUT_SECONDS:-20} + config: + provider: ${SOURCE_PROVIDER:-firecrawl-html} + - type: transform + name: clean + concurrencyGroup: process + config: + transformer: readability-default + target: text/html + emitDiagnostics: true + - type: transform + name: convert + concurrencyGroup: process + config: + transformer: mdream-convert + target: text/markdown + emitDiagnostics: true + - type: capture-urls + name: capture_source_urls + concurrencyGroup: process + config: + artifact: trusted-urls + - type: llm-pass + name: summarize + concurrencyGroup: llm + timeoutSeconds: ${SUMMARIZE_TIMEOUT_SECONDS:-60} + config: + provider: llm-default + minInputChars: ${OUTPUT_TARGET_CHARS:-25000} + outputReserveChars: ${OUTPUT_TARGET_CHARS:-25000} + templates: + system: ../templates/summarize.system.md + user: ../templates/summarize.user.md + vars: + targetChars: ${OUTPUT_TARGET_CHARS:-25000} + - type: verify-urls + name: verify_after_summarize + concurrencyGroup: llm + config: + artifact: trusted-urls + onHallucination: report + maxReportedUrls: 50 + - type: truncate + name: truncate + config: + targetChars: ${OUTPUT_TARGET_CHARS:-25000} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index fdeb4c8..71d8a60 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -68,7 +68,7 @@ The current built-ins are: - HTTP adapters: `open-webui`, `jina`. - Source providers: `http`, `firecrawl`, `docling`. -- Content transformers: `mdream`. +- Content transformers: `readability`, `mdream`. - LLM providers: `openai-chat`. - Pipeline steps: `load-source`, `llm-pass`, `transform`, `truncate`, `capture-urls`, `verify-urls`. - Output renderers: `debug-xml`, `passthrough`. diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 46c3098..f76c52e 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -47,9 +47,6 @@ These are consumed by helper scripts and ignored by the Node app. The default YAML file uses environment substitution for provider URLs, tokens, concurrency, timeouts, and renderer selection. Most of these values are shown in [.env.example](../.env.example) and passed through the Compose files. -The default (`truncate`) pipeline needs no provider configuration — it fetches and truncates. The `clean-deterministic` pipeline loads HTML from `firecrawl-html` and converts it to markdown with the `mdream` transformer (no LLM); its Firecrawl env vars are required only when it is active (set `DEFAULT_PIPELINE=clean-deterministic`). The `clean-llm` pipeline uses an LLM and Firecrawl; those env vars are -required only when `clean-llm` is active (set `DEFAULT_PIPELINE=clean-llm`). - Validation is lazy: the service validates and constructs only the providers and pipelines that are reachable from the active configuration. An unused provider with missing env vars will not fail startup. @@ -58,11 +55,11 @@ The placeholders below are grouped by the part of the pipeline they configure. ### Source provider selection -| Variable | Default / behavior | Purpose | -| ----------------- | ------------------ | -------------------------------------------------------------- | -| `SOURCE_PROVIDER` | `http-default` | Named source provider used by the `load-source` pipeline step. | +| Variable | Default / behavior | Purpose | +| ----------------- | -------------------------- | ---------------------------------------------------- | +| `SOURCE_PROVIDER` | selected pipeline fallback | Optional source-provider override for `load-source`. | -The built-in `http` provider has no environment variables — it fetches the input URL directly. Each pipeline supplies its own `SOURCE_PROVIDER` fallback: `http-default` for `truncate`, `firecrawl-html` for `clean-deterministic`, and `firecrawl-markdown` for `clean-llm`. +Leave `SOURCE_PROVIDER` empty to use the selected pipeline's fallback: `http-default` for `truncate`, `firecrawl-html` for `clean-deterministic` and `clean-combined`, and `firecrawl-markdown` for `clean-llm`. Set it only when intentionally overriding that choice. ### Source provider (Firecrawl) @@ -73,19 +70,24 @@ The built-in `http` provider has no environment variables — it fetches the inp ### Source provider (Docling) -| Variable | Default / behavior | Purpose | -| ------------------ | ------------------ | ---------------------------------------------------------------------------------------------- | -| `DOCLING_BASE_URL` | empty | Docling Serve base URL. Required only when a pipeline referencing `docling-default` is active. | -| `DOCLING_API_KEY` | empty | Optional Docling API key. | +| Variable | Default / behavior | Purpose | +| ------------------ | ------------------ | ------------------------------------------------------------------------------------------ | +| `DOCLING_BASE_URL` | empty | Docling Serve base URL. Required only when a pipeline referencing `docling-ocr` is active. | +| `DOCLING_API_KEY` | empty | Optional Docling API key. | ### Content transformer (mdream) -| Variable | Default / behavior | Purpose | -| ---------------- | ------------------ | ---------------------------------------------------------------- | -| `MDREAM_MINIMAL` | `true` | Isolate main content and filter boilerplate in `mdream-default`. | -| `MDREAM_CLEAN` | `true` | Post-conversion link and whitespace cleanup. | +| Variable | Default / behavior | Purpose | +| -------------- | ------------------ | -------------------------------------------------------------------------------------------------------------------- | +| `MDREAM_CLEAN` | `true` | Post-conversion link and whitespace cleanup for `mdream-convert`. Ignored when `minimal=true` (`mdream-aggressive`). | + +### Content transformer (readability) -These are read only when a pipeline using the `transform` step (such as `clean-deterministic`) is active. +| Variable | Default / behavior | Purpose | +| ------------------------------- | ------------------ | -------------------------------------------------------------------------------- | +| `READABILITY_MIN_CONTENT_CHARS` | `140` | Minimum content length for `isProbablyReaderable` gate in `readability-default`. | +| `READABILITY_MIN_SCORE` | `20` | Minimum readerable score for `isProbablyReaderable` gate. | +| `READABILITY_MAX_ELEMENTS` | `0` | Maximum DOM elements Readability parses; `0` = unlimited (DoS guardrail). | ### LLM provider (OpenAI-compatible) @@ -110,10 +112,11 @@ These are read only when a pipeline using the `transform` step (such as `clean-d ### Concurrency -| Variable | Default / behavior | Purpose | -| -------------------- | ------------------ | ------------------------------------------------------------- | -| `SOURCE_CONCURRENCY` | `1` | Maximum concurrent source-loading groups. | -| `LLM_CONCURRENCY` | `1` | Maximum concurrent LLM workflow groups (used by `clean-llm`). | +| Variable | Default / behavior | Purpose | +| --------------------- | ------------------ | ------------------------------------------------- | +| `SOURCE_CONCURRENCY` | `1` | Maximum concurrent source-loading groups. | +| `PROCESS_CONCURRENCY` | `5` | Maximum concurrent processing (transform) groups. | +| `LLM_CONCURRENCY` | `1` | Maximum concurrent LLM workflow groups. | ### Output rendering @@ -129,7 +132,7 @@ These are read only when a pipeline using the `transform` step (such as `clean-d | `OWUI_AUTH_TOKEN` | empty | Optional bearer token for the Open WebUI adapter route. | | `JINA_AUTH_TOKEN` | empty | Optional bearer token for the Jina-style adapter route. | -`.env.example` intentionally sets `LLM_CONTEXT_TOKENS` to a large example value so the context-fit gate is enabled in copied local configs. Leave it blank when you want to disable that gate. +`.env.example` shows practical local overrides for concurrency, output size, clean-pass thresholds, and `LLM_CONTEXT_TOKENS`. Leave an env value blank or unset it when you want the YAML fallback or schema default instead. ## Environment Substitution Syntax diff --git a/docs/CUSTOMIZATION.md b/docs/CUSTOMIZATION.md index 71b1685..b38dede 100644 --- a/docs/CUSTOMIZATION.md +++ b/docs/CUSTOMIZATION.md @@ -144,14 +144,22 @@ config: parsePdf: true ``` -| Knob | Values | Default | Purpose | -| ------------------- | --------------------------------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------- | -| `output` | `markdown` \| `html` \| `rawHtml` | `markdown` | Which format Firecrawl returns. `html` is cleaned main-content HTML. `rawHtml` is the full JS-rendered DOM (use for Readability testing). | -| `onlyMainContent` | bool | `true` | When enabled Firecrawl extracts the main page content and strips headers, nav, footers. No-op for `rawHtml`. | -| `stripBase64Images` | bool | `true` | Maps to `removeBase64Images` — replaces inline data URIs with short placeholders in markdown output. No-op for `html`/`rawHtml`. | -| `parsePdf` | bool | `true` | When enabled Firecrawl parses PDF files to markdown via `parsers: ["pdf"]`. | +| Knob | Values | Default | Purpose | +| ------------------- | --------------------------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `output` | `markdown` \| `html` \| `rawHtml` | `markdown` | Which format Firecrawl returns. `html` is cleaned main-content HTML. `rawHtml` is "raw" — full JS-rendered DOM for web pages, viewer `` wrapper for images, and bare extracted text for PDFs (use for Readability testing). | +| `onlyMainContent` | bool | `true` | When enabled Firecrawl extracts the main page content and strips headers, nav, footers. No-op for `rawHtml`. | +| `stripBase64Images` | bool | `true` | Maps to `removeBase64Images` — replaces inline data URIs with short placeholders in markdown output. No-op for `html`/`rawHtml`. | +| `parsePdf` | bool | `true` | Configurable via `FIRECRAWL_PARSE_PDF`. When `true`, Firecrawl parses PDF files to extracted text. When `false`, raw PDF bytes are returned as a binary body (`application/pdf`) — the pipeline fails with `unconverted_binary` until a PDF converter (e.g. Docling) is wired up. | -The provider calls `/v2/scrape`. Upstream HTTP, parse, empty, and network failures are converted to degradable upstream errors. Title is read from `data.metadata.title` (format-independent). `output: markdown` returns `text/markdown`; `output: html` and `output: rawHtml` return `text/html`. +The provider calls `/v2/scrape`. Upstream HTTP, parse, empty, and network failures are converted to degradable upstream errors. Title is read from `data.metadata.title` (format-independent). The media type of returned bodies is derived truthfully: + +| `output` | HTML / docx source | image source | PDF source (parsePdf:true) | +| -------- | ------------------ | --------------- | -------------------------- | +| markdown | `text/markdown` | `text/markdown` | `text/markdown` | +| html | `text/html` | `text/html` | `text/html` (wrapped) | +| rawHtml | `text/html` | `text/html` | **`text/plain`** | + +`rawHtml` is "raw": for web pages and images it returns HTML, but for PDFs it returns bare extracted text. The provider detects this by looking at whether the content starts with an HTML tag — if not, it labels it `text/plain` so downstream transformers (which gate on `isHtmlMediaType`) skip it correctly. When `parsePdf:false`, PDFs are returned as binary (`application/pdf`) from base64-decoded raw bytes supplied by Firecrawl's empty-`parsers` mode. ### `docling` @@ -176,19 +184,50 @@ The provider calls `POST /v1/convert/source`. Title is read from `document.json_ Content transformers convert a loaded body from one representation to another in-process, with no upstream call. The `transform` pipeline step selects a transformer by name and applies it to the current body. +### `readability` + +Extracts main-article HTML from raw HTML using [Mozilla Readability](https://github.com/mozilla/readability). It uses `isProbablyReaderable` as a suitability gate: article-like pages become cleaned HTML; other pages return `declined` and keep the current body unchanged. + +```yaml +config: + minContentLength: ${READABILITY_MIN_CONTENT_CHARS:-140} + minScore: ${READABILITY_MIN_SCORE:-20} + maxElements: ${READABILITY_MAX_ELEMENTS:-0} +``` + +| Knob | Values | Default | Purpose | +| ------------------ | ------------ | ------- | ---------------------------------------------------------------------------------------------------- | +| `minContentLength` | positive int | `140` | Minimum content length for `isProbablyReaderable`. | +| `minScore` | number >= 0 | `20` | Minimum readerable score for `isProbablyReaderable`. | +| `maxElements` | int >= 0 | `0` | Maximum DOM elements Readability parses. `0` = unlimited (DoS guardrail). Maps to `maxElemsToParse`. | + +Supports text `text/html` or `application/xhtml+xml` bodies targeting `text/html`. The incoming title is preserved; when absent, the extracted article title is used instead. + +Decline reasons are `not_readerable` and `parse_empty`. Runtime errors propagate to the orchestrator; the failed step has no effect, so downstream steps continue from the prior body and the run rolls up as `degraded`. + ### `mdream` Converts HTML bodies to markdown using the [`@mdream/js`](https://www.npmjs.com/package/@mdream/js) library. Pure-JS, no native dependencies. +Two shipped instances are declared in the config: + +**`mdream-convert`** — used after `readability` in shipped pipelines. It converts HTML to markdown without re-extracting main content. + ```yaml config: - minimal: ${MDREAM_MINIMAL:-true} clean: ${MDREAM_CLEAN:-true} ``` +**`mdream-aggressive`** — extraction + conversion in one pass using mdream's minimal preset. Suitable for custom pipelines without a prior readability step. + +```yaml +config: + minimal: true +``` + | Knob | Values | Default | Purpose | | --------- | ------ | ------- | -------------------------------------------------------------------------------------------------------- | -| `minimal` | bool | `true` | Apply mdream's minimal preset (isolate main content, filter boilerplate). Takes precedence over `clean`. | +| `minimal` | bool | `false` | Apply mdream's minimal preset (isolate main content, filter boilerplate). Takes precedence over `clean`. | | `clean` | bool | `true` | Clean up the markdown output: drop tracking params, redundant and empty links, and collapse blank lines. | The transformer supports text bodies whose media type is `text/html` or `application/xhtml+xml` and a requested target of `text/markdown`. The input URL is passed to mdream as the conversion origin, so relative links and images resolve against it. The original body title is preserved. @@ -297,7 +336,7 @@ Mustache escaping is disabled for prompt templates so markdown is passed through ```yaml config: - transformer: mdream-default + transformer: mdream-convert target: text/markdown onUnsupported: skip emitDiagnostics: false @@ -310,9 +349,10 @@ Applies a named content transformer to the current body. The step resolves `tran | `transformer` | string | — | Name of a configured `contentTransformers` instance. Required. | | `target` | string | — | Media type the transformer must produce, for example `text/markdown`. Required. | | `onUnsupported` | `skip` \| `fail` | `skip` | What to do when the transformer does not support the current body (wrong source media type or representation). | +| `onDeclined` | `skip` \| `fail` | `skip` | What to do when the transformer returns `declined` (e.g. `not_readerable`, `parse_empty`). | | `emitDiagnostics` | bool | `false` | When enabled, transformer-reported diagnostics are surfaced as child nodes in the step report. | -The step skips with `no_body` when there is no body. When the transformer does not support the current body it skips with `unsupported`, or fails with that reason when `onUnsupported: fail`. A transform aborted by the step timeout fails with `timeout`. If the transformer returns a body that does not match `target`, the step fails with `wrong_output_type`. +The step skips with `no_body` when there is no body. When the transformer does not support the current body it skips with `unsupported` (or fails with `onUnsupported: fail`). When the transformer returns `declined`, the step skips with that reason (or fails with `onDeclined: fail`). A transform aborted by the step timeout fails with `timeout`. If the transformer returns a body that does not match `target`, the step fails with `wrong_output_type`. ### `truncate` @@ -358,7 +398,7 @@ Shipped defaults: `verify_after_clean` uses `rollback` with `maxReportedUrls: 0` ## Shipped Pipelines -The config ships three pipelines, selected via `DEFAULT_PIPELINE` (defaults to `truncate`). +The config ships four pipelines, selected via `DEFAULT_PIPELINE` (defaults to `truncate`). ### `truncate` (default) @@ -371,10 +411,10 @@ Fetches the URL directly and truncates to budget. No external services required. ### `clean-deterministic` ```text -load-source(firecrawl-html) -> transform(html -> markdown via mdream) -> truncate +load-source(firecrawl-html) -> transform(clean via readability) -> transform(convert via mdream) -> truncate ``` -Loads HTML from the `firecrawl-html` source instance (Firecrawl with `output: rawHtml`) and converts it to markdown deterministically with the `mdream` transformer, with no LLM passes. Defaults `SOURCE_PROVIDER` to `firecrawl-html`. Select with `DEFAULT_PIPELINE=clean-deterministic`. +Loads Firecrawl raw HTML, extracts article HTML with `readability` when suitable, converts HTML to markdown with `mdream`, then truncates. No LLM passes. Pipeline fallback: `firecrawl-html`. Select with `DEFAULT_PIPELINE=clean-deterministic`. ### `clean-llm` @@ -382,4 +422,12 @@ Loads HTML from the `firecrawl-html` source instance (Firecrawl with `output: ra load-source(firecrawl-markdown) -> capture-urls -> llm-pass(clean) -> verify-urls(rollback) -> llm-pass(summarize) -> verify-urls(report) -> truncate ``` -The original reference pipeline. Loads markdown from a Firecrawl instance, runs clean and summarize LLM passes with URL-hallucination gates, then truncates. Defaults `SOURCE_PROVIDER` to `firecrawl-markdown`. Select with `DEFAULT_PIPELINE=clean-llm`. +The original reference pipeline. Loads markdown from a Firecrawl instance, runs clean and summarize LLM passes with URL-hallucination gates, then truncates. Pipeline fallback: `firecrawl-markdown`. Select with `DEFAULT_PIPELINE=clean-llm`. + +### `clean-combined` + +```text +load-source(firecrawl-html) -> transform(clean via readability) -> transform(convert via mdream) -> capture-urls -> llm-pass(summarize) -> verify-urls(report) -> truncate +``` + +Combines the deterministic HTML-to-markdown path with an LLM summarize pass. Pipeline fallback: `firecrawl-html`. Select with `DEFAULT_PIPELINE=clean-combined`. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index e806798..8a877ff 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -13,36 +13,25 @@ Implemented today: - `POST /` for Open WebUI's external web-loader contract. - `GET /r/` and `GET /r?url=` for limited Jina Reader-style URL-to-markdown compatibility. -- Firecrawl fetch provider using `/v2/scrape` with markdown output. -- OpenAI-compatible Chat Completions LLM provider. -- Optional clean stage, optional summarize stage, final truncation, and XML diagnostic footer. -- YAML-driven configuration for pipelines, providers, output renderers, and HTTP adapters. +- Firecrawl source provider using `/v2/scrape` with markdown, html, and rawHtml output. +- Docling source provider (experimental, PDFs and Office docs → markdown via Docling Serve API). +- Deterministic content transformers: Readability (article extraction) and mdream (HTML → markdown). +- Optional LLM clean stage, optional LLM summarize stage, final truncation, and XML diagnostic footer. +- YAML-driven configuration for pipelines, providers, content transformers, output renderers, and HTTP adapters. - Docker image published by CI to GHCR. Current provider implementations are intentionally few. The interfaces exist so replacements can be added without rewriting the use case. ## Direction -The biggest usability problem today is speed. Running every page through one or two LLM passes just to strip boilerplate is slow on limited hardware and invites hallucination. Most pages do not need a model to become clean Markdown. - -The direction is to make the default path deterministic and reserve the LLM for work only a model does well: - -```text -fetch (HTML or Markdown) - -> [HTML] extract main content - -> [HTML] convert HTML -> Markdown - -> [optional] LLM summarize / index - -> keep-honest repair (URLs, code blocks) - -> truncate to budget -``` - -The URL-in / clean-Markdown-within-a-budget contract does not change. What changes is how the middle is done: established extraction and conversion tools instead of a model. We own the plumbing — content-type routing, budgets, honesty gates — and the surfaces — HTTP/MCP/CLI, observability, caching. We do not write our own readability heuristics or HTML parser. +Deterministic extraction (readability + mdream) handles most pages in milliseconds. The next step is to make the pipeline self-aware: route through deterministic or LLM paths based on URL heuristics and per-step signals, reserving LLM passes for pages that need them. ## Coming soon -1. **In-process main-content extraction.** A new pipeline step that extracts content from HTML using Readability. This creates a fully deterministic HTML-to-markdown path with no external service dependency for boilerplate removal. -2. **Demote the LLM clean pass.** Once deterministic clean matches or beats it on representative URLs, make the clean stage optional and only execute it for URLs where deterministic clean did not produce desired results. Keep summarize. -3. **Keep summaries honest.** Extend the existing URL quality gate from all-or-nothing rejection into a deterministic repair pass, and implement fenced-code-block verification/repair. +1. **Conditional step execution based on signals.** Engine-level `skipIfSignal` / `runIfSignal` configuration on pipeline steps. When a signal name is set and the condition is met, the step is skipped entirely without running. +2. **URL heuristic step to classify input.** A pipeline step that matches the URL against patterns (path, domain, expected content type) and sets runtime signals for downstream steps. Use the feature to avoid Readibility step on GitHub domain etc. +3. **Route binary content to Docling.** Use the Docling source provider based on URL heuristics as an alternative to Firecrawl for PDFs and other binary content. +4. **Keep summaries honest.** Extend the existing URL quality gate from all-or-nothing rejection into a deterministic repair pass, and implement fenced-code-block verification/repair. A couple of polish items ride along: @@ -53,8 +42,9 @@ A couple of polish items ride along: Near-term additions once the pivot above settles. -- **Playwright fetch provider (HTML/DOM)** to remove the hard dependency on a running Firecrawl instance for HTML pages. -- **Docling for document conversion.** Use Docling to convert PDFs, Office documents, and other document-type files to markdown. +- **Playwright source provider** to remove the hard dependency on a running Firecrawl instance for HTML pages. +- **Docling content transformer** to convert PDFs, Office documents, and other document-type files to markdown. +- **Crawl4AI source provider** as a potentially better alternative to Firecrawl. ## Mid term diff --git a/docs/agents/extension-authoring.md b/docs/agents/extension-authoring.md index dfdeb28..62d3d70 100644 --- a/docs/agents/extension-authoring.md +++ b/docs/agents/extension-authoring.md @@ -65,7 +65,16 @@ Implement `SourceProvider` (`src/contracts/extensions/source-provider.ts`). The ### Content Transformers -Implement `ContentTransformer` (`src/contracts/extensions/content-transformer.ts`): `supports({ sourceKind, sourceMediaType, request })` and `transform({ url, body, request }, { signal }): Promise`. `supports` gates `transform`: the step only invokes a matching transformer, so `transform` may throw `InternalError` for inputs that bypass the gate. Built-in example: `src/builtins/content-transformers/mdream/`. +Implement `ContentTransformer` (`src/contracts/extensions/content-transformer.ts`): `supports({ sourceKind, sourceMediaType, request })` and `transform({ url, body, request }, { signal }): Promise`. + +The result type is a discriminated union: + +- `{ outcome: "transformed", body: BodyContent, diagnostics?: ContentTransformDiagnostic[] }` — the transformer performed a conversion. The step only applies effects on this branch and runs its honesty gate (`outputMatchesTarget`). +- `{ outcome: "declined", reason?: string }` — the transformer chose not to transform (e.g. not suitable, empty parse). The step's `onDeclined` knob decides whether this becomes a skip or a failure. Declined outcomes never apply body effects, so the previous body version passes through unchanged. + +`supports` gates `transform`: the step only invokes a matching transformer, so `transform` may throw `InternalError` for inputs that bypass the gate. Use `declined` for deliberate "not suitable" decisions (e.g. `isProbablyReaderable` returned false). + +Built-in examples: `src/builtins/content-transformers/readability/` (article HTML extraction, outputs `declined`), `src/builtins/content-transformers/mdream/` (HTML to markdown, always `transformed`). ### LLM Providers diff --git a/docs/agents/smoke-testing.md b/docs/agents/smoke-testing.md index d00e86a..4d8bd01 100644 --- a/docs/agents/smoke-testing.md +++ b/docs/agents/smoke-testing.md @@ -16,9 +16,10 @@ Artifacts land in `scripts/out/` (gitignored). ## Picking URLs -- Keep the set small — **5 URLs or fewer**. Each provider × surface × environment combination reruns the whole set, so it adds up fast. +- Keep the set small — **5 URLs or fewer** unless the task explicitly asks for broader coverage. Each provider × surface × environment combination reruns the whole set, so it adds up fast. - Cover a couple of content shapes: at least one ordinary HTML article, and at least one real document (a true PDF, confirmed by content type rather than a `.pdf` in the path). -- Skip raw images; they only produce binary-rejection results and teach nothing new. +- Verify selected URLs with a lightweight `GET` and record content type before running the matrix. `HEAD` is not enough: some sites reject it, and docs URLs rot into 404s. +- Skip raw images unless the task asks for a negative/control URL; they usually produce binary-rejection or tiny metadata-only results. ```powershell # gather-urls.ps1 reads SEARX_BASE from the environment and does NOT load .env. @@ -50,6 +51,9 @@ npm run dev ./scripts/smoke-jina.ps1 -UrlFile scripts/out/urls-smoke.txt -Concurrency 2 ./scripts/smoke-owui.ps1 -UrlFile scripts/out/urls-smoke.txt -Concurrency 2 -BatchSize 3 +# Quick inline smoke test (no script needed): +curl.exe -s --max-time 30 "http://localhost:3010/r/$([System.Uri]::EscapeDataString('https://en.wikipedia.org/wiki/Rust_(programming_language)'))" | Select-String -Pattern '=14.0.0" + } + }, "node_modules/@pinojs/redact": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", @@ -1399,6 +1410,12 @@ "node": "18 || 20 || >=22" } }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC" + }, "node_modules/brace-expansion": { "version": "5.0.6", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", @@ -1502,6 +1519,40 @@ "node": ">= 8" } }, + "node_modules/css-select": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", + "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", + "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cssom": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", + "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==", + "license": "MIT" + }, "node_modules/dateformat": { "version": "4.6.3", "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", @@ -1546,6 +1597,61 @@ "node": ">=6" } }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -1569,6 +1675,18 @@ "once": "^1.4.0" } }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/es-module-lexer": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", @@ -1866,6 +1984,37 @@ "dev": true, "license": "MIT" }, + "node_modules/htmlparser2": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "entities": "^7.0.1" + } + }, + "node_modules/htmlparser2/node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/ipaddr.js": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.4.0.tgz", @@ -2028,6 +2177,36 @@ } ] }, + "node_modules/linkedom": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.12.tgz", + "integrity": "sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q==", + "license": "ISC", + "dependencies": { + "css-select": "^5.1.0", + "cssom": "^0.5.0", + "html-escaper": "^3.0.3", + "htmlparser2": "^10.0.0", + "uhyphen": "^0.2.0" + }, + "engines": { + "node": ">=16" + }, + "peerDependencies": { + "canvas": ">= 2" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/linkedom/node_modules/html-escaper": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz", + "integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==", + "license": "MIT" + }, "node_modules/loupe": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", @@ -2148,6 +2327,18 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/on-exit-leak-free": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", @@ -2840,6 +3031,12 @@ "node": ">=14.17" } }, + "node_modules/uhyphen": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/uhyphen/-/uhyphen-0.2.0.tgz", + "integrity": "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==", + "license": "ISC" + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", diff --git a/package.json b/package.json index a2572be..c56dd2d 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,9 @@ }, "dependencies": { "@mdream/js": "^1.3.0", + "@mozilla/readability": "^0.6.0", "fastify": "^5.2.1", + "linkedom": "^0.18.12", "mustache": "^4.2.0", "p-limit": "^6.2.0", "pino": "^9.6.0", diff --git a/src/builtins/content-transformers/mdream/mdream-transformer-config.ts b/src/builtins/content-transformers/mdream/mdream-transformer-config.ts index 3889cae..e795f85 100644 --- a/src/builtins/content-transformers/mdream/mdream-transformer-config.ts +++ b/src/builtins/content-transformers/mdream/mdream-transformer-config.ts @@ -5,6 +5,12 @@ * shared preprocessor to treat blank placeholders as omitted values. These knobs * tune transformer behavior per instance; routing intent (the target media type) * comes from the requesting step, not this config. + * + * Note on `minimal`: withMinimalPreset returns 0 HTML-to-markdown chars when fed + * readability-extracted article HTML from table-heavy layouts (HN, Quora, etc.) + * because it tries to re-extract main content from already-extracted markup. + * Default false is safe when readability precedes mdream; use minimal=true only + * in pipelines without a prior extraction step. */ import { z } from "zod"; diff --git a/src/builtins/content-transformers/mdream/mdream-transformer.ts b/src/builtins/content-transformers/mdream/mdream-transformer.ts index b6362ef..aa49a12 100644 --- a/src/builtins/content-transformers/mdream/mdream-transformer.ts +++ b/src/builtins/content-transformers/mdream/mdream-transformer.ts @@ -60,6 +60,7 @@ export class MdreamTransformer implements ContentTransformer { if (!markdown) diagnostics.push({ code: "empty_output" }); return { + outcome: "transformed", body: { kind: "text", mediaType: mediaTypes.markdown, content: markdown, title: body.title }, diagnostics }; diff --git a/src/builtins/content-transformers/readability/readability-transformer-config.ts b/src/builtins/content-transformers/readability/readability-transformer-config.ts new file mode 100644 index 0000000..7b722a7 --- /dev/null +++ b/src/builtins/content-transformers/readability/readability-transformer-config.ts @@ -0,0 +1,31 @@ +/** + * Parses YAML configuration for the built-in readability content transformer. + * + * The three knobs mirror the Readability.js suitability gate and parsing + * limits: minContentLength and minScore feed `isProbablyReaderable`, while + * maxElements caps the internal parse buffer as a DoS guardrail. A value of 0 + * for maxElements means unlimited. + * + * Environment substitution runs before this schema, so number fields use the + * shared preprocessors to treat blank placeholders as omitted values. + */ + +import { z } from "zod"; + +import { emptyStringAsUndefined } from "../../../shared/config-coercion.js"; + +const readabilityTransformerConfigSchema = z + .object({ + minContentLength: z.preprocess(emptyStringAsUndefined, z.coerce.number().int().positive().default(140)), + minScore: z.preprocess(emptyStringAsUndefined, z.coerce.number().min(0).default(20)), + maxElements: z.preprocess(emptyStringAsUndefined, z.coerce.number().int().min(0).default(0)) + }) + .strict(); + +/** Represents parsed readability transformer configuration. */ +export type ReadabilityTransformerConfig = z.infer; + +/** Parses readability transformer configuration from YAML. */ +export function parseReadabilityTransformerConfig(raw: unknown): ReadabilityTransformerConfig { + return Object.freeze(readabilityTransformerConfigSchema.parse(raw)); +} diff --git a/src/builtins/content-transformers/readability/readability-transformer-descriptor.ts b/src/builtins/content-transformers/readability/readability-transformer-descriptor.ts new file mode 100644 index 0000000..7b58813 --- /dev/null +++ b/src/builtins/content-transformers/readability/readability-transformer-descriptor.ts @@ -0,0 +1,20 @@ +/** + * Exposes the readability content transformer descriptor to engine bundles. + * + * The transformer uses the Mozilla Readability library via linkedom to extract + * article-level content from raw HTML. Construction resolves config; runtime + * only needs the config and logger since all DOM work is in-process. + */ + +import { parseReadabilityTransformerConfig } from "./readability-transformer-config.js"; +import { ReadabilityTransformer } from "./readability-transformer.js"; + +import type { ContentTransformerDescriptor } from "../../../contracts/extensions/content-transformer.js"; +import type { ReadabilityTransformerConfig } from "./readability-transformer-config.js"; + +/** Defines the built-in readability content transformer type. */ +export const readabilityTransformerDescriptor = { + type: "readability", + parseConfig: parseReadabilityTransformerConfig, + create: args => new ReadabilityTransformer(args.config, { logger: args.deps.logger }) +} satisfies ContentTransformerDescriptor; diff --git a/src/builtins/content-transformers/readability/readability-transformer.ts b/src/builtins/content-transformers/readability/readability-transformer.ts new file mode 100644 index 0000000..4e51efa --- /dev/null +++ b/src/builtins/content-transformers/readability/readability-transformer.ts @@ -0,0 +1,95 @@ +/** + * Extracts main article HTML from raw HTML using Mozilla Readability. + * + * Given a raw HTML page, this transformer first checks whether the document + * *seems* readerable via `isProbablyReaderable`. If not, it returns a + * "declined" outcome so the pipeline can fall through to downstream steps + * unchanged. If the gate passes, the document is parsed with Readability to + * extract the article content. The output is *still HTML* (not markdown), so + * another transformer (typically mdream) further downstream converts it. + */ + +import { Readability, isProbablyReaderable } from "@mozilla/readability"; +import { parseHTML } from "linkedom"; + +import { InternalError } from "../../../shared/errors.js"; +import { isHtmlMediaType, mediaTypes } from "../../../shared/media-types.js"; + +import type { + ContentTransformDiagnostic, + ContentTransformResult, + ContentTransformer +} from "../../../contracts/extensions/content-transformer.js"; +import type { BodyContent } from "../../../contracts/pipeline/context.js"; +import type { Logger } from "../../../shared/logger.js"; +import type { ReadabilityTransformerConfig } from "./readability-transformer-config.js"; + +/** Transforms raw HTML text bodies into cleaned article HTML via Readability. */ +export class ReadabilityTransformer implements ContentTransformer { + /** Creates a Readability transformer instance. */ + constructor( + private readonly config: ReadabilityTransformerConfig, + private readonly deps: { readonly logger: Logger } + ) {} + + /** Supports HTML or XHTML text bodies targeting HTML output. */ + supports(input: { + readonly sourceKind: BodyContent["kind"]; + readonly sourceMediaType: string; + readonly request: { readonly targetMediaType: string }; + }): boolean { + return ( + input.sourceKind === "text" && + isHtmlMediaType(input.sourceMediaType) && + input.request.targetMediaType === mediaTypes.html + ); + } + + /** Attempts to extract article content; declines when the page isn't article-like. */ + async transform( + input: { readonly url: string; readonly body: BodyContent; readonly request: { readonly targetMediaType: string } }, + opts: { readonly signal: AbortSignal } + ): Promise { + opts.signal.throwIfAborted(); + + const body = input.body; + if (body.kind !== "text") + throw new InternalError("readability transformer requires a text body", "readability_non_text_body"); + + const dom = parseHTML(body.content); + const doc = dom.document as unknown as Document; + + if (!doc.documentElement) return { outcome: "declined", reason: "parse_empty" }; + + if ( + !isProbablyReaderable(doc, { + minContentLength: this.config.minContentLength, + minScore: this.config.minScore + }) + ) + return { outcome: "declined", reason: "not_readerable" }; + + const article = new Readability(doc, { maxElemsToParse: this.config.maxElements || undefined }).parse(); + if (!article || !article.content) return { outcome: "declined", reason: "parse_empty" }; + + const originalLength = body.content.length; + const articleLength = article.content.length; + const diagnostics: ContentTransformDiagnostic[] = [ + { + code: "readability", + message: `${originalLength} html chars -> ${articleLength} article chars` + } + ]; + + return { + outcome: "transformed", + body: { + kind: "text", + mediaType: mediaTypes.html, + content: article.content, + title: body.title || (article.title ?? undefined) + }, + diagnostics + }; + } +} diff --git a/src/builtins/pipeline-steps/transform/transform-step-config.ts b/src/builtins/pipeline-steps/transform/transform-step-config.ts index aef90ab..d546738 100644 --- a/src/builtins/pipeline-steps/transform/transform-step-config.ts +++ b/src/builtins/pipeline-steps/transform/transform-step-config.ts @@ -15,6 +15,7 @@ const transformStepConfigSchema = z transformer: z.string().min(1, "transform step requires a transformer name"), target: z.string().min(1, "transform step requires a target media type"), onUnsupported: z.enum(["skip", "fail"]).default("skip"), + onDeclined: z.enum(["skip", "fail"]).default("skip"), emitDiagnostics: z.preprocess(booleanStringAsBooleanOrUndefined, z.boolean().default(false)) }) .strict(); @@ -26,6 +27,7 @@ export type TransformStepConfig = z.infer; export interface TransformStepOptions { readonly target: string; readonly onUnsupported: "skip" | "fail"; + readonly onDeclined: "skip" | "fail"; readonly emitDiagnostics: boolean; } diff --git a/src/builtins/pipeline-steps/transform/transform-step-descriptor.ts b/src/builtins/pipeline-steps/transform/transform-step-descriptor.ts index c11bda4..a6fe497 100644 --- a/src/builtins/pipeline-steps/transform/transform-step-descriptor.ts +++ b/src/builtins/pipeline-steps/transform/transform-step-descriptor.ts @@ -24,6 +24,7 @@ export const transformStepDescriptor = { { target: args.config.target, onUnsupported: args.config.onUnsupported, + onDeclined: args.config.onDeclined, emitDiagnostics: args.config.emitDiagnostics }, { transformer, logger: args.deps.logger } diff --git a/src/builtins/pipeline-steps/transform/transform-step.ts b/src/builtins/pipeline-steps/transform/transform-step.ts index a84714b..5bd6663 100644 --- a/src/builtins/pipeline-steps/transform/transform-step.ts +++ b/src/builtins/pipeline-steps/transform/transform-step.ts @@ -46,6 +46,9 @@ export class TransformStep implements PipelineStep { throw error; } + if (result.outcome === "declined") + return { status: this.config.onDeclined === "fail" ? "failed" : "skipped", reason: result.reason ?? "declined" }; + if (!outputMatchesTarget(result.body, this.config.target)) return { status: "failed", reason: "wrong_output_type" }; return { diff --git a/src/builtins/source-providers/firecrawl/firecrawl-provider.ts b/src/builtins/source-providers/firecrawl/firecrawl-provider.ts index 9c595c3..1b1febd 100644 --- a/src/builtins/source-providers/firecrawl/firecrawl-provider.ts +++ b/src/builtins/source-providers/firecrawl/firecrawl-provider.ts @@ -10,7 +10,7 @@ import { z } from "zod"; import { UpstreamError, isAbortError } from "../../../shared/errors.js"; -import { mediaTypes } from "../../../shared/media-types.js"; +import { looksLikeHtml, mediaTypes } from "../../../shared/media-types.js"; import { joinUrl } from "../../../shared/urls.js"; import type { SourceDocument, SourceProvider } from "../../../contracts/extensions/source-provider.js"; @@ -93,7 +93,28 @@ export class FirecrawlProvider implements SourceProvider { upstreamStatus: response.status }); const title = data.title ?? (typeof metadata.title === "string" ? metadata.title : undefined); - const mediaType: string = this.config.output === "markdown" ? mediaTypes.markdown : mediaTypes.html; + const contentType = typeof metadata.contentType === "string" ? metadata.contentType : ""; + + if (!this.config.parsePdf) { + const essence = contentType.split(";", 1)[0].trim().toLowerCase(); + if (essence === mediaTypes.pdf) { + const raw = extractBase64Payload(content, this.config.output); + if (!raw) { + throw new UpstreamError("Firecrawl returned empty base64 PDF payload", "empty", { + upstreamStatus: response.status + }); + } + const bytes = Buffer.from(raw, "base64"); + if (bytes.byteLength === 0) { + throw new UpstreamError("Firecrawl returned empty PDF bytes after base64 decode", "empty", { + upstreamStatus: response.status + }); + } + return { kind: "binary", bytes, mediaType: mediaTypes.pdf, title }; + } + } + + const mediaType = deriveTextMediaType(this.config.output, content); return { kind: "text", content, mediaType, title }; } catch (error) { if (error instanceof UpstreamError || isAbortError(error)) throw error; @@ -108,3 +129,30 @@ export class FirecrawlProvider implements SourceProvider { return headers; } } + +/** + * Derives a truthful media type for the text returned by Firecrawl. + * + * - markdown output is always `text/markdown`. + * - html output is always `text/html` (Firecrawl wraps non-HTML content). + * - rawHtml is "raw": real HTML for web pages/sites, bare text for PDFs. + */ +function deriveTextMediaType(output: string, content: string): string { + if (output === "markdown") return mediaTypes.markdown; + if (output === "html") return mediaTypes.html; + return looksLikeHtml(content) ? mediaTypes.html : mediaTypes.plainText; +} + +/** + * Extracts the base64-encoded payload from a Firecrawl response field. + * + * With parsers:[], Firecrawl returns base64 of the raw file bytes. When + * output=html, the base64 is wrapped in `…`; + * for markdown and rawHtml it is bare. + */ +function extractBase64Payload(content: string, output: string): string | undefined { + const trimmed = content.trim(); + if (!trimmed) return undefined; + if (output !== "html") return trimmed; + return trimmed.replace(/^()?/i, "").replace(/(<\/body>)?<\/html>$/i, ""); +} diff --git a/src/bundles/default-engine-descriptors.ts b/src/bundles/default-engine-descriptors.ts index fc9919f..2a28b32 100644 --- a/src/bundles/default-engine-descriptors.ts +++ b/src/bundles/default-engine-descriptors.ts @@ -7,6 +7,7 @@ */ import { mdreamTransformerDescriptor } from "../builtins/content-transformers/mdream/mdream-transformer-descriptor.js"; +import { readabilityTransformerDescriptor } from "../builtins/content-transformers/readability/readability-transformer-descriptor.js"; import { openAiChatProviderDescriptor } from "../builtins/llm-providers/openai-chat/openai-chat-provider-descriptor.js"; import { debugXmlRendererDescriptor } from "../builtins/output-renderers/debug-xml/debug-xml-renderer-descriptor.js"; import { passthroughRendererDescriptor } from "../builtins/output-renderers/passthrough/passthrough-renderer-descriptor.js"; @@ -42,7 +43,10 @@ export const DEFAULT_ENGINE_DESCRIPTOR_BUNDLE: EngineDescriptorBundle = Object.f [httpProviderDescriptor, firecrawlProviderDescriptor, doclingProviderDescriptor], descriptor => descriptor.type ), - contentTransformers: createDescriptorRecord([mdreamTransformerDescriptor], descriptor => descriptor.type), + contentTransformers: createDescriptorRecord( + [mdreamTransformerDescriptor, readabilityTransformerDescriptor], + descriptor => descriptor.type + ), llmProviders: createDescriptorRecord([openAiChatProviderDescriptor], descriptor => descriptor.type), outputRenderers: createDescriptorRecord( [debugXmlRendererDescriptor, passthroughRendererDescriptor], diff --git a/src/contracts/extensions/content-transformer.ts b/src/contracts/extensions/content-transformer.ts index 564cbfd..1d46804 100644 --- a/src/contracts/extensions/content-transformer.ts +++ b/src/contracts/extensions/content-transformer.ts @@ -28,11 +28,21 @@ export interface ContentTransformDiagnostic { } /** Carries the transformed body and optional diagnostics. */ -export interface ContentTransformResult { +export interface ContentTransformedResult { + readonly outcome: "transformed"; readonly body: BodyContent; readonly diagnostics?: ReadonlyArray; } +/** Indicates the transformer chose not to transform this input (e.g. not suitable). */ +export interface ContentDeclinedResult { + readonly outcome: "declined"; + readonly reason?: string; +} + +/** A content transformer may transform the body or decline. */ +export type ContentTransformResult = ContentTransformedResult | ContentDeclinedResult; + /** Transforms one pipeline body representation into another. */ export interface ContentTransformer { /** Reports whether this instance can satisfy the requested transform. */ diff --git a/src/shared/media-types.ts b/src/shared/media-types.ts index f14d218..8376825 100644 --- a/src/shared/media-types.ts +++ b/src/shared/media-types.ts @@ -33,3 +33,8 @@ export function isHtmlMediaType(mediaType: string): boolean { const essence = mediaType.split(";", 1)[0].trim().toLowerCase(); return essence === mediaTypes.html || essence === mediaTypes.xhtml; } + +/** Returns whether trimmed text starts with an HTML tag or XML declaration. */ +export function looksLikeHtml(text: string): boolean { + return /^\s* { { signal } ); + expect(result.outcome).toBe("transformed"); + if (result.outcome !== "transformed") throw new Error("Expected transformed outcome"); expect(result.body.kind).toBe("text"); + if (result.body.kind !== "text") throw new Error("Expected text body"); expect(result.body.mediaType).toBe(mediaTypes.markdown); expect(result.body.title).toBe("Doc"); - if (result.body.kind === "text") { - expect(result.body.content).toContain("# Hello"); - expect(result.body.content).toContain("](https://example.com"); - } + expect(result.body.content).toContain("# Hello"); + expect(result.body.content).toContain("](https://example.com"); }); it("emits a summary diagnostic and flags empty output", async () => { @@ -91,6 +92,8 @@ describe("MdreamTransformer.transform", () => { }, { signal } ); + expect(empty.outcome).toBe("transformed"); + if (empty.outcome !== "transformed") throw new Error("Expected transformed outcome"); expect(empty.diagnostics?.some(diagnostic => diagnostic.code === "empty_output")).toBe(true); expect(empty.diagnostics?.some(diagnostic => diagnostic.code === "mdream")).toBe(true); }); diff --git a/tests/builtins/content-transformers/readability/readability-transformer-config.test.ts b/tests/builtins/content-transformers/readability/readability-transformer-config.test.ts new file mode 100644 index 0000000..3f36025 --- /dev/null +++ b/tests/builtins/content-transformers/readability/readability-transformer-config.test.ts @@ -0,0 +1,50 @@ +/** Verifies the readability content transformer config parsing. */ +import { describe, expect, it } from "vitest"; + +import { parseReadabilityTransformerConfig } from "../../../../src/builtins/content-transformers/readability/readability-transformer-config.js"; + +describe("parseReadabilityTransformerConfig", () => { + it("applies defaults for all fields", () => { + expect(parseReadabilityTransformerConfig({})).toEqual({ + minContentLength: 140, + minScore: 20, + maxElements: 0 + }); + }); + + it("parses explicit numeric values", () => { + expect( + parseReadabilityTransformerConfig({ + minContentLength: "100", + minScore: "15", + maxElements: "50" + }) + ).toEqual({ + minContentLength: 100, + minScore: 15, + maxElements: 50 + }); + }); + + it("treats blank strings as undefined (env placeholder fallback)", () => { + expect( + parseReadabilityTransformerConfig({ + minContentLength: "", + minScore: "", + maxElements: "" + }) + ).toEqual({ + minContentLength: 140, + minScore: 20, + maxElements: 0 + }); + }); + + it("rejects negative minContentLength", () => { + expect(() => parseReadabilityTransformerConfig({ minContentLength: "-1" })).toThrow(); + }); + + it("rejects unknown keys", () => { + expect(() => parseReadabilityTransformerConfig({ extra: 1 })).toThrow(); + }); +}); diff --git a/tests/builtins/content-transformers/readability/readability-transformer.test.ts b/tests/builtins/content-transformers/readability/readability-transformer.test.ts new file mode 100644 index 0000000..625b055 --- /dev/null +++ b/tests/builtins/content-transformers/readability/readability-transformer.test.ts @@ -0,0 +1,172 @@ +/** Verifies the readability content transformer support matrix, decline behavior, and conversion. */ +import { describe, expect, it } from "vitest"; + +import { ReadabilityTransformer } from "../../../../src/builtins/content-transformers/readability/readability-transformer.js"; +import { InternalError } from "../../../../src/shared/errors.js"; +import { mediaTypes } from "../../../../src/shared/media-types.js"; +import { createTestLogger } from "../../../helpers/logger.js"; + +import type { BodyContent } from "../../../../src/contracts/pipeline/context.js"; + +function articleHtml(title?: string): string { + const header = title ? `${title}` : ""; + return `${header}

Article

${"Article content paragraph. ".repeat(60)}

`; +} + +function shortHtml(): string { + return "

Too short

"; +} + +function makeTransformer(config: { minContentLength?: number; minScore?: number; maxElements?: number } = {}) { + return new ReadabilityTransformer( + { + minContentLength: config.minContentLength ?? 140, + minScore: config.minScore ?? 20, + maxElements: config.maxElements ?? 0 + }, + { logger: createTestLogger() } + ); +} + +const htmlRequest = { targetMediaType: mediaTypes.html }; + +describe("ReadabilityTransformer.supports", () => { + it("supports HTML and XHTML text bodies targeting HTML", () => { + const transformer = makeTransformer(); + expect(transformer.supports({ sourceKind: "text", sourceMediaType: "text/html", request: htmlRequest })).toBe(true); + expect( + transformer.supports({ sourceKind: "text", sourceMediaType: "application/xhtml+xml", request: htmlRequest }) + ).toBe(true); + }); + + it("rejects non-HTML sources, non-text kinds, and non-HTML targets", () => { + const transformer = makeTransformer(); + expect(transformer.supports({ sourceKind: "text", sourceMediaType: "text/plain", request: htmlRequest })).toBe( + false + ); + expect(transformer.supports({ sourceKind: "binary", sourceMediaType: "text/html", request: htmlRequest })).toBe( + false + ); + expect( + transformer.supports({ + sourceKind: "text", + sourceMediaType: "text/html", + request: { targetMediaType: "text/markdown" } + }) + ).toBe(false); + }); +}); + +describe("ReadabilityTransformer.transform", () => { + const signal = new AbortController().signal; + + it("extracts article content from readerable HTML", async () => { + const transformer = makeTransformer(); + const body: BodyContent = { + kind: "text", + mediaType: "text/html", + content: articleHtml("Test Page"), + title: "Original Title" + }; + + const result = await transformer.transform({ url: "https://example.com", body, request: htmlRequest }, { signal }); + + expect(result.outcome).toBe("transformed"); + if (result.outcome !== "transformed") return; + expect(result.body.kind).toBe("text"); + if (result.body.kind !== "text") return; + expect(result.body.mediaType).toBe(mediaTypes.html); + expect(result.body.content).toContain("Article content"); + expect(result.body.title).toBe("Original Title"); + }); + + it("falls back to the article title when the incoming body has no title", async () => { + const transformer = makeTransformer(); + const body: BodyContent = { + kind: "text", + mediaType: "text/html", + content: articleHtml("Page Title") + }; + + const result = await transformer.transform({ url: "https://example.com", body, request: htmlRequest }, { signal }); + + expect(result.outcome).toBe("transformed"); + if (result.outcome !== "transformed") return; + expect(result.body.title).toBe("Page Title"); + }); + + it("adds a readability diagnostic with char counts", async () => { + const transformer = makeTransformer(); + const body: BodyContent = { + kind: "text", + mediaType: "text/html", + content: articleHtml() + }; + + const result = await transformer.transform({ url: "https://example.com", body, request: htmlRequest }, { signal }); + + expect(result.outcome).toBe("transformed"); + if (result.outcome !== "transformed") return; + expect(result.diagnostics?.some(d => d.code === "readability")).toBe(true); + }); + + it("declines not_readerable for short boilerplate HTML", async () => { + const transformer = makeTransformer(); + const body: BodyContent = { + kind: "text", + mediaType: "text/html", + content: shortHtml() + }; + + const result = await transformer.transform({ url: "https://example.com", body, request: htmlRequest }, { signal }); + + expect(result.outcome).toBe("declined"); + if (result.outcome !== "declined") return; + expect(result.reason).toBe("not_readerable"); + }); + + it("declines parse_empty for a document with no document element", async () => { + const transformer = makeTransformer(); + const body: BodyContent = { + kind: "text", + mediaType: "text/html", + content: "" + }; + + const result = await transformer.transform({ url: "https://example.com", body, request: htmlRequest }, { signal }); + + expect(result.outcome).toBe("declined"); + if (result.outcome !== "declined") return; + expect(result.reason).toBe("parse_empty"); + }); + + it("throws when given a non-text body", async () => { + const transformer = makeTransformer(); + await expect( + transformer.transform( + { + url: "https://example.com/", + body: { kind: "binary", mediaType: "application/pdf", bytes: new Uint8Array([1]) }, + request: htmlRequest + }, + { signal } + ) + ).rejects.toBeInstanceOf(InternalError); + }); + + it("throws if the signal is already aborted", async () => { + const transformer = makeTransformer(); + const controller = new AbortController(); + controller.abort(); + await expect( + transformer.transform( + { + url: "https://example.com/", + body: { kind: "text", mediaType: "text/html", content: articleHtml() }, + request: htmlRequest + }, + { signal: controller.signal } + ) + ).rejects.toThrow(); + }); +}); diff --git a/tests/builtins/pipeline-steps/transform/transform-step-config.test.ts b/tests/builtins/pipeline-steps/transform/transform-step-config.test.ts index b1ba992..847a3eb 100644 --- a/tests/builtins/pipeline-steps/transform/transform-step-config.test.ts +++ b/tests/builtins/pipeline-steps/transform/transform-step-config.test.ts @@ -5,10 +5,11 @@ import { parseTransformStepConfig } from "../../../../src/builtins/pipeline-step describe("parseTransformStepConfig", () => { it("applies defaults for optional fields", () => { - expect(parseTransformStepConfig({ transformer: "mdream-default", target: "text/markdown" })).toEqual({ - transformer: "mdream-default", + expect(parseTransformStepConfig({ transformer: "mdream-convert", target: "text/markdown" })).toEqual({ + transformer: "mdream-convert", target: "text/markdown", onUnsupported: "skip", + onDeclined: "skip", emitDiagnostics: false }); }); @@ -18,15 +19,27 @@ describe("parseTransformStepConfig", () => { expect(() => parseTransformStepConfig({ transformer: "x" })).toThrow(); }); - it("validates the onUnsupported enum and coerces emitDiagnostics", () => { + it("validates the onUnsupported and onDeclined enums and coerces emitDiagnostics", () => { expect(() => parseTransformStepConfig({ transformer: "x", target: "text/markdown", onUnsupported: "explode" }) ).toThrow(); + expect(() => + parseTransformStepConfig({ transformer: "x", target: "text/markdown", onDeclined: "explode" }) + ).toThrow(); expect( parseTransformStepConfig({ transformer: "x", target: "text/markdown", emitDiagnostics: "true" }).emitDiagnostics ).toBe(true); }); + it("parses explicit onDeclined values", () => { + expect(parseTransformStepConfig({ transformer: "x", target: "text/markdown", onDeclined: "fail" }).onDeclined).toBe( + "fail" + ); + expect(parseTransformStepConfig({ transformer: "x", target: "text/markdown", onDeclined: "skip" }).onDeclined).toBe( + "skip" + ); + }); + it("rejects unknown keys", () => { expect(() => parseTransformStepConfig({ transformer: "x", target: "text/markdown", extra: 1 })).toThrow(); }); diff --git a/tests/builtins/pipeline-steps/transform/transform-step.test.ts b/tests/builtins/pipeline-steps/transform/transform-step.test.ts index 1fb9178..fa7eea2 100644 --- a/tests/builtins/pipeline-steps/transform/transform-step.test.ts +++ b/tests/builtins/pipeline-steps/transform/transform-step.test.ts @@ -11,8 +11,20 @@ import type { } from "../../../../src/contracts/extensions/content-transformer.js"; import type { TransformStepOptions } from "../../../../src/builtins/pipeline-steps/transform/transform-step-config.js"; -const baseConfig: TransformStepOptions = { target: "text/markdown", onUnsupported: "skip", emitDiagnostics: false }; -const markdownResult: ContentTransformResult = { body: { kind: "text", mediaType: "text/markdown", content: "# md" } }; +const baseConfig: TransformStepOptions = { + target: "text/markdown", + onUnsupported: "skip", + onDeclined: "skip", + emitDiagnostics: false +}; +const markdownResult: ContentTransformResult = { + outcome: "transformed", + body: { kind: "text", mediaType: "text/markdown", content: "# md" } +}; + +function declinedResult(reason?: string): ContentTransformResult { + return { outcome: "declined", reason }; +} function fakeTransformer(overrides: Partial = {}): ContentTransformer { return { @@ -92,6 +104,7 @@ describe("TransformStep", () => { { ...baseConfig, emitDiagnostics: true }, fakeTransformer({ transform: async () => ({ + outcome: "transformed", body: markdownResult.body, diagnostics: [{ code: "mdream", message: "10 -> 4" }, { code: "empty_output" }] }) @@ -111,7 +124,12 @@ describe("TransformStep", () => { it("fails when the transformer returns a different media type", async () => { const step = makeStep( baseConfig, - fakeTransformer({ transform: async () => ({ body: { kind: "text", mediaType: "text/plain", content: "x" } }) }) + fakeTransformer({ + transform: async () => ({ + outcome: "transformed", + body: { kind: "text", mediaType: "text/plain", content: "x" } + }) + }) ); await expect(step.run(makeStepContext({ body: htmlBody }))).resolves.toMatchObject({ @@ -124,7 +142,10 @@ describe("TransformStep", () => { const step = makeStep( baseConfig, fakeTransformer({ - transform: async () => ({ body: { kind: "binary", mediaType: "text/markdown", bytes: new Uint8Array([1]) } }) + transform: async () => ({ + outcome: "transformed", + body: { kind: "binary", mediaType: "text/markdown", bytes: new Uint8Array([1]) } + }) }) ); @@ -150,6 +171,36 @@ describe("TransformStep", () => { }); }); + it("skips when the transformer declines with onDeclined default skip", async () => { + const step = makeStep(baseConfig, fakeTransformer({ transform: async () => declinedResult("not_readerable") })); + + await expect(step.run(makeStepContext({ body: htmlBody }))).resolves.toMatchObject({ + status: "skipped", + reason: "not_readerable" + }); + }); + + it("fails when the transformer declines with onDeclined fail", async () => { + const step = makeStep( + { ...baseConfig, onDeclined: "fail" }, + fakeTransformer({ transform: async () => declinedResult("not_readerable") }) + ); + + await expect(step.run(makeStepContext({ body: htmlBody }))).resolves.toMatchObject({ + status: "failed", + reason: "not_readerable" + }); + }); + + it("surfaces a default declined reason when the transformer omits the reason", async () => { + const step = makeStep(baseConfig, fakeTransformer({ transform: async () => declinedResult() })); + + await expect(step.run(makeStepContext({ body: htmlBody }))).resolves.toMatchObject({ + status: "skipped", + reason: "declined" + }); + }); + it("rethrows non-abort transformer errors", async () => { const step = makeStep( baseConfig, diff --git a/tests/builtins/source-providers/firecrawl/firecrawl-provider.test.ts b/tests/builtins/source-providers/firecrawl/firecrawl-provider.test.ts index 10734e2..af99e36 100644 --- a/tests/builtins/source-providers/firecrawl/firecrawl-provider.test.ts +++ b/tests/builtins/source-providers/firecrawl/firecrawl-provider.test.ts @@ -69,6 +69,210 @@ describe("FirecrawlProvider", () => { expect(document).toEqual({ kind: "text", content: "Raw", mediaType: "text/html" }); }); + it("labels rawHtml PDF content as text/plain", async () => { + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "rawHtml" }), + { + httpFetch: async () => + jsonResponse({ + success: true, + data: { + rawHtml: "Dummy PDF file", + metadata: { contentType: "application/pdf" } + } + }), + logger: createTestLogger() + } + ); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document).toEqual({ + kind: "text", + content: "Dummy PDF file", + mediaType: "text/plain" + }); + }); + + it("labels rawHtml image viewer as text/html", async () => { + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "rawHtml" }), + { + httpFetch: async () => + jsonResponse({ + success: true, + data: { + rawHtml: + '', + metadata: { contentType: "image/jpeg" } + } + }), + logger: createTestLogger() + } + ); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.mediaType).toBe("text/html"); + }); + + it("labels html PDF wrapper as text/html", async () => { + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "html" }), + { + httpFetch: async () => + jsonResponse({ + success: true, + data: { + html: "Dummy PDF file", + metadata: { contentType: "application/pdf" } + } + }), + logger: createTestLogger() + } + ); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document).toEqual({ + kind: "text", + content: "Dummy PDF file", + mediaType: "text/html" + }); + }); + + it("labels markdown PDF content as text/markdown", async () => { + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "markdown" }), + { + httpFetch: async () => + jsonResponse({ + success: true, + data: { + markdown: "Dummy PDF file", + metadata: { contentType: "application/pdf" } + } + }), + logger: createTestLogger() + } + ); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document).toEqual({ + kind: "text", + content: "Dummy PDF file", + mediaType: "text/markdown" + }); + }); + + it("returns binary body when parsePdf is false for a PDF", async () => { + const base64Payload = Buffer.from("%PDF-1.4 test document").toString("base64"); + const fetchFn: typeof fetch = async (_input, init) => { + return jsonResponse({ + success: true, + data: { markdown: base64Payload, metadata: { contentType: "application/pdf" } } + }); + }; + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "markdown", parsePdf: false }), + { httpFetch: fetchFn, logger: createTestLogger() } + ); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.kind).toBe("binary"); + if (document.kind === "binary") { + expect(document.mediaType).toBe("application/pdf"); + expect(new TextDecoder().decode(document.bytes)).toBe("%PDF-1.4 test document"); + } + }); + + it("strips html wrapper from binary base64 when parsePdf is false and output is html", async () => { + const payload = "JVBERi0xLjQK"; + const fetchFn: typeof fetch = async () => { + return jsonResponse({ + success: true, + data: { html: payload, metadata: { contentType: "application/pdf" } } + }); + }; + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "html", parsePdf: false }), + { httpFetch: fetchFn, logger: createTestLogger() } + ); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.kind).toBe("binary"); + if (document.kind === "binary") { + expect(new TextDecoder().decode(document.bytes)).toBe("%PDF-1.4\n"); + } + }); + + it("returns text body when parsePdf is false for a non-PDF (HTML) source", async () => { + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "markdown", parsePdf: false }), + { + httpFetch: async () => + jsonResponse({ + success: true, + data: { + markdown: "# Hello from HTML", + metadata: { contentType: "text/html; charset=UTF-8" } + } + }), + logger: createTestLogger() + } + ); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document).toEqual({ + kind: "text", + content: "# Hello from HTML", + mediaType: "text/markdown" + }); + }); + + it("returns text body when parsePdf is false for an image source", async () => { + const base64Payload = Buffer.from("RIFF fake").toString("base64"); + const fetchFn: typeof fetch = async () => { + return jsonResponse({ + success: true, + data: { + markdown: `![](https://example.com/img.jpg)`, + metadata: { contentType: "image/png" } + } + }); + }; + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", parsePdf: false }), + { httpFetch: fetchFn, logger: createTestLogger() } + ); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.kind).toBe("text"); + }); + + it("rejects empty base64 when parsePdf is false for PDF", async () => { + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", parsePdf: false }), + { + httpFetch: async () => + jsonResponse({ + success: true, + data: { markdown: "", metadata: { contentType: "application/pdf" } } + }), + logger: createTestLogger() + } + ); + + await expect(provider.load("https://example.com", { signal: new AbortController().signal })).rejects.toMatchObject({ + upstreamCode: "empty" + }); + }); + it("omits parsers when parsePdf is false", async () => { let requestBody: Record | undefined; const fetchFn: typeof fetch = async (_input, init) => { diff --git a/tests/bundles/default-engine-descriptors.test.ts b/tests/bundles/default-engine-descriptors.test.ts index 6eb01e2..6584623 100644 --- a/tests/bundles/default-engine-descriptors.test.ts +++ b/tests/bundles/default-engine-descriptors.test.ts @@ -42,7 +42,20 @@ describe("default engine descriptor bundle", () => { const mdream = DEFAULT_ENGINE_DESCRIPTOR_BUNDLE.contentTransformers.mdream; const config = mdream.parseConfig({}); const transformer = await mdream.create({ - name: "mdream-default", + name: "mdream-convert", + config, + deps: { tools: createTestHostTools(), logger: createTestLogger() } + }); + + expect(transformer.supports).toEqual(expect.any(Function)); + expect(transformer.transform).toEqual(expect.any(Function)); + }); + + it("round-trips the readability content transformer descriptor", async () => { + const readability = DEFAULT_ENGINE_DESCRIPTOR_BUNDLE.contentTransformers.readability; + const config = readability.parseConfig({ minContentLength: "200", minScore: "15", maxElements: "0" }); + const transformer = await readability.create({ + name: "readability-default", config, deps: { tools: createTestHostTools(), logger: createTestLogger() } }); @@ -132,7 +145,10 @@ function servicesWithProviders(sourceProvider: SourceProvider, llmProvider: LlmP function contentTransformer(): ContentTransformer { return { supports: () => true, - transform: async () => ({ body: { kind: "text", mediaType: "text/markdown", content: "md" } }) + transform: async () => ({ + outcome: "transformed", + body: { kind: "text", mediaType: "text/markdown", content: "md" } + }) }; } diff --git a/tests/shared/media-types.test.ts b/tests/shared/media-types.test.ts index 432a715..0055440 100644 --- a/tests/shared/media-types.test.ts +++ b/tests/shared/media-types.test.ts @@ -1,7 +1,7 @@ /** Verifies media-type predicates and constants. */ import { describe, expect, it } from "vitest"; -import { isTextLike, mediaTypes } from "../../src/shared/media-types.js"; +import { isTextLike, looksLikeHtml, mediaTypes } from "../../src/shared/media-types.js"; describe("mediaTypes", () => { it("exports expected text media type constants", () => { @@ -48,3 +48,37 @@ describe("isTextLike", () => { expect(isTextLike("")).toBe(false); }); }); + +describe("looksLikeHtml", () => { + it("returns true for text starting with ", () => { + expect(looksLikeHtml("Hello")).toBe(true); + }); + + it("returns true for text starting with { + expect(looksLikeHtml("\n")).toBe(true); + }); + + it("returns true with leading whitespace", () => { + expect(looksLikeHtml(" \n")).toBe(true); + }); + + it("returns false for bare plain text", () => { + expect(looksLikeHtml("Dummy PDF file")).toBe(false); + }); + + it("returns false for markdown image reference", () => { + expect(looksLikeHtml("![](https://example.com/img.jpg)")).toBe(false); + }); + + it("returns false for empty string", () => { + expect(looksLikeHtml("")).toBe(false); + }); + + it("returns true for linkime-type HTML", () => { + expect(looksLikeHtml('')).toBe(true); + }); + + it("returns false for markdown heading", () => { + expect(looksLikeHtml("# Hello world\n\nSome text")).toBe(false); + }); +});