diff --git a/.env.example b/.env.example index 8931826..ae8d489 100644 --- a/.env.example +++ b/.env.example @@ -23,50 +23,69 @@ LOG_PRETTY=auto # --- Pipeline ------------------------------------------------------------ # Active pipeline selected by HTTP adapters. Default: truncate (load-source + truncate) -DEFAULT_PIPELINE=truncate +DEFAULT_PIPELINE=clean-combined -# Maximum concurrent source-loading groups. Default: 1 -SOURCE_CONCURRENCY=4 +# Maximum concurrent source-loading groups. Default: 1; raise only if providers can handle it. +SOURCE_CONCURRENCY=5 -# Maximum concurrent LLM workflow groups. Default: 1 +# Maximum concurrent processing (transform) groups. Default: 5 +PROCESS_CONCURRENCY=5 + +# Maximum concurrent LLM workflow groups. Default: 1; raise only if the model endpoint can handle it. LLM_CONCURRENCY=2 -# Output renderer used by the default pipeline. Default: debug-xml +# Output renderer override used by shipped pipelines. Default: debug-xml DEFAULT_OUTPUT_RENDERER=debug-xml # Include skipped steps in the debug-xml footer. Default: false DEBUG_XML_INCLUDE_SKIPPED=false -# Desired maximum output size used by summarize and truncate steps. Default: 25000 +# Desired maximum output size used by summarize and truncate steps. Default: 25000; larger values return more content. OUTPUT_TARGET_CHARS=35000 # --- Source provider ----------------------------------------------------- -# Active source provider name used by pipelines (truncate and clean-llm). Default: default-http (native HTTP fetch, no external service needed) -SOURCE_PROVIDER=default-http +# Optional source-provider override. Leave empty to use the selected pipeline's fallback. +SOURCE_PROVIDER= -# Firecrawl base URL. Required only when a referenced provider uses it (e.g. default-firecrawl). +# Firecrawl base URL. Required only when a referenced provider uses it (e.g. firecrawl-markdown, firecrawl-html). FIRECRAWL_BASE_URL=https://firecrawl.example # Optional Firecrawl bearer token. Default: empty (no token) FIRECRAWL_API_KEY= -# Docling Serve base URL. Required only when a referenced provider uses it (e.g. default-docling). +# Firecrawl PDF parsing. Set to false to return raw PDF bytes (binary body) instead of parsed text. +FIRECRAWL_PARSE_PDF= + +# Docling Serve base URL. Required only when a referenced provider uses it (e.g. docling-ocr). DOCLING_BASE_URL=https://docling.example # Optional Docling API key (X-Api-Key header). Default: empty (no token) DOCLING_API_KEY= +# --- Content transformers ------------------------------------------------ +# Post-conversion link and whitespace cleanup for mdream-convert. Ignored when minimal=true (mdream-aggressive). Default: true +MDREAM_CLEAN=true + +# Minimum content length for Readability's isProbablyReaderable gate. Default: 140 +READABILITY_MIN_CONTENT_CHARS=140 + +# Minimum readerable score for Readability's isProbablyReaderable gate. Default: 20 +READABILITY_MIN_SCORE=20 + +# Maximum DOM elements Readability parses; 0 = unlimited. Default: 0 +READABILITY_MAX_ELEMENTS=0 + # --- LLM provider -------------------------------------------------------- -# OpenAI-compatible Chat Completions /v1 base URL. Required only when a referenced provider uses it (e.g. default-llm). +# OpenAI-compatible Chat Completions /v1 base URL. Required only when a referenced provider uses it (e.g. llm-default). LLM_BASE_URL=https://openai-compatible.example/v1 # Optional LLM bearer token. Default: empty (no token) LLM_API_KEY= -# Model identifier sent to the chat-completions endpoint. Required only when a referenced provider uses it (e.g. default-llm). +# Model identifier sent to the chat-completions endpoint. Required only when a referenced provider uses it (e.g. llm-default). LLM_MODEL=model-name -# Model context window in tokens. Default: empty (disables the context-fit gate) +# Model context window in tokens. Default: empty (disables the context-fit gate); set to enable it. LLM_CONTEXT_TOKENS=131072 # Conservative chars-per-token estimate used by the context-fit gate. Default: 3.5 @@ -79,10 +98,10 @@ LLM_SAFETY_MARGIN_TOKENS=128 # Timeout in seconds for the source-loading step. Default: 20 LOAD_SOURCE_TIMEOUT_SECONDS=20 -# Timeout in seconds for the clean LLM pass. Default: 60 +# Timeout in seconds for the clean LLM pass. Default: 60; raise for slower models. CLEAN_TIMEOUT_SECONDS=90 -# Minimum body length before the clean LLM pass runs. Default: 1000 +# Minimum body length before the clean LLM pass runs. Default: 1000; lower values run cleanup more often. CLEAN_MIN_INPUT_CHARS=500 # Timeout in seconds for the summarize LLM pass. Default: 60 diff --git a/AGENTS.md b/AGENTS.md index 38c1297..58e1446 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,3 +37,13 @@ Read the row that matches your task. Each focused doc is self-contained for its - When changing behavior, keep the docs that describe it accurate in the same change. - No gratuitous comments. Inline comments that narrate a change are prohibited ([coding-conventions](docs/agents/coding-conventions.md)). Before adding a comment, ask: will it still be useful after the transient context is gone? - Don't over-document trivial changes. A new knob, helper, or internal refactor does not warrant updates to `ARCHITECTURE.md` or `extension-authoring.md`. Update only docs that directly describe the changed surface. + +## Terminal Behavior on Windows + +When using the terminal on Windows: + +- Prefer single-line PowerShell commands. +- Avoid interactive commands, pagers, prompts, and commands that wait for input. +- Prefer `pwsh` or PowerShell with `-NoLogo -NoProfile`. +- For git commands, use non-interactive flags such as `--no-pager` and `--no-edit` when appropriate. +- If a command appears stuck after output is printed, do not blindly rerun it. First check whether the command already completed and inspect the visible terminal output. diff --git a/README.md b/README.md index ebf15cb..e434261 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,10 @@ LLM Context Loader is a small HTTP service that turns URLs into markdown for LLM ## Current Shape -- **Pipelines:** `truncate` (default, no LLM) and `clean-llm` (Firecrawl + LLM), selected via `DEFAULT_PIPELINE`. +- **Pipelines:** `truncate` (default, no LLM), `clean-deterministic` (Firecrawl HTML + Readability + mdream, no LLM), `clean-llm` (Firecrawl + LLM), and `clean-combined` (Firecrawl HTML + Readability + mdream + LLM summarize), selected via `DEFAULT_PIPELINE`. - **Source providers:** native HTTP fetch, Firecrawl, Docling. -- **LLM provider:** OpenAI-compatible `/chat/completions`. +- **Content transformers:** `readability` (article HTML extraction), `mdream` (HTML to markdown). +- **LLM providers:** OpenAI-compatible `/chat/completions`. - **Output renderers:** `debug-xml` and `passthrough`, selected per pipeline in YAML. ## Quick Start diff --git a/compose.deploy.yaml b/compose.deploy.yaml index 8ea88b6..81fbae5 100644 --- a/compose.deploy.yaml +++ b/compose.deploy.yaml @@ -15,18 +15,26 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l # Pipeline concurrency and renderer selection. SOURCE_CONCURRENCY: "${SOURCE_CONCURRENCY:-1}" + PROCESS_CONCURRENCY: "${PROCESS_CONCURRENCY:-5}" LLM_CONCURRENCY: "${LLM_CONCURRENCY:-1}" DEFAULT_OUTPUT_RENDERER: "${DEFAULT_OUTPUT_RENDERER:-debug-xml}" DEBUG_XML_INCLUDE_SKIPPED: "${DEBUG_XML_INCLUDE_SKIPPED:-false}" OUTPUT_TARGET_CHARS: "${OUTPUT_TARGET_CHARS:-25000}" # Source provider. - SOURCE_PROVIDER: "${SOURCE_PROVIDER:-default-http}" + SOURCE_PROVIDER: "${SOURCE_PROVIDER:-}" FIRECRAWL_BASE_URL: "${FIRECRAWL_BASE_URL:-}" FIRECRAWL_API_KEY: "${FIRECRAWL_API_KEY:-}" + FIRECRAWL_PARSE_PDF: "${FIRECRAWL_PARSE_PDF:-}" DOCLING_BASE_URL: "${DOCLING_BASE_URL:-}" DOCLING_API_KEY: "${DOCLING_API_KEY:-}" + # Content transformers. + MDREAM_CLEAN: "${MDREAM_CLEAN:-true}" + READABILITY_MIN_CONTENT_CHARS: "${READABILITY_MIN_CONTENT_CHARS:-140}" + READABILITY_MIN_SCORE: "${READABILITY_MIN_SCORE:-20}" + READABILITY_MAX_ELEMENTS: "${READABILITY_MAX_ELEMENTS:-0}" + # LLM provider. LLM_BASE_URL: "${LLM_BASE_URL:-}" LLM_API_KEY: "${LLM_API_KEY:-}" diff --git a/compose.yaml b/compose.yaml index 7dbb7c6..2530eeb 100644 --- a/compose.yaml +++ b/compose.yaml @@ -15,18 +15,26 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l # Pipeline concurrency and renderer selection. SOURCE_CONCURRENCY: "${SOURCE_CONCURRENCY:-1}" + PROCESS_CONCURRENCY: "${PROCESS_CONCURRENCY:-5}" LLM_CONCURRENCY: "${LLM_CONCURRENCY:-1}" DEFAULT_OUTPUT_RENDERER: "${DEFAULT_OUTPUT_RENDERER:-debug-xml}" DEBUG_XML_INCLUDE_SKIPPED: "${DEBUG_XML_INCLUDE_SKIPPED:-false}" OUTPUT_TARGET_CHARS: "${OUTPUT_TARGET_CHARS:-25000}" # Source provider. - SOURCE_PROVIDER: "${SOURCE_PROVIDER:-default-http}" + SOURCE_PROVIDER: "${SOURCE_PROVIDER:-}" FIRECRAWL_BASE_URL: "${FIRECRAWL_BASE_URL:-}" FIRECRAWL_API_KEY: "${FIRECRAWL_API_KEY:-}" + FIRECRAWL_PARSE_PDF: "${FIRECRAWL_PARSE_PDF:-}" DOCLING_BASE_URL: "${DOCLING_BASE_URL:-}" DOCLING_API_KEY: "${DOCLING_API_KEY:-}" + # Content transformers. + MDREAM_CLEAN: "${MDREAM_CLEAN:-true}" + READABILITY_MIN_CONTENT_CHARS: "${READABILITY_MIN_CONTENT_CHARS:-140}" + READABILITY_MIN_SCORE: "${READABILITY_MIN_SCORE:-20}" + READABILITY_MAX_ELEMENTS: "${READABILITY_MAX_ELEMENTS:-0}" + # LLM provider. LLM_BASE_URL: "${LLM_BASE_URL:-}" LLM_API_KEY: "${LLM_API_KEY:-}" diff --git a/config/llm-context-loader.yaml b/config/llm-context-loader.yaml index 2de37f9..cd617a6 100644 --- a/config/llm-context-loader.yaml +++ b/config/llm-context-loader.yaml @@ -26,15 +26,22 @@ outputRenderers: type: passthrough config: {} -# Source providers turn an input URL into source markdown. +# Source providers turn an input URL into a pipeline body. sourceProviders: - default-http: + http-default: type: http config: userAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" maxBytes: 5000000 titleFromHtml: true - default-firecrawl: + firecrawl-html: + type: firecrawl + config: + baseUrl: ${FIRECRAWL_BASE_URL:-} + apiKey: ${FIRECRAWL_API_KEY:-} + output: rawHtml + parsePdf: ${FIRECRAWL_PARSE_PDF:-true} + firecrawl-markdown: type: firecrawl config: baseUrl: ${FIRECRAWL_BASE_URL:-} @@ -42,8 +49,8 @@ sourceProviders: output: markdown onlyMainContent: true stripBase64Images: true - parsePdf: true - default-docling: + parsePdf: ${FIRECRAWL_PARSE_PDF:-true} + docling-ocr: type: docling config: baseUrl: ${DOCLING_BASE_URL:-} @@ -52,9 +59,26 @@ sourceProviders: doOcr: true tableMode: accurate +# Content transformers rewrite the body between representations (e.g. HTML to markdown). +contentTransformers: + readability-default: + type: readability + config: + minContentLength: ${READABILITY_MIN_CONTENT_CHARS:-140} + minScore: ${READABILITY_MIN_SCORE:-20} + maxElements: ${READABILITY_MAX_ELEMENTS:-0} + mdream-convert: + type: mdream + config: + clean: ${MDREAM_CLEAN:-true} + mdream-aggressive: + type: mdream + config: + minimal: true + # LLM providers run prompt-rendered chat-completions passes. llmProviders: - default-llm: + llm-default: type: openai-chat config: baseUrl: ${LLM_BASE_URL:-} @@ -77,7 +101,38 @@ pipelines: concurrencyGroup: source timeoutSeconds: ${LOAD_SOURCE_TIMEOUT_SECONDS:-20} config: - provider: ${SOURCE_PROVIDER:-default-http} + provider: ${SOURCE_PROVIDER:-http-default} + - type: truncate + name: truncate + config: + targetChars: ${OUTPUT_TARGET_CHARS:-25000} + + clean-deterministic: + outputRenderer: ${DEFAULT_OUTPUT_RENDERER:-debug-xml} + limiters: + source: ${SOURCE_CONCURRENCY:-1} + process: ${PROCESS_CONCURRENCY:-5} + steps: + - type: load-source + name: fetch + concurrencyGroup: source + timeoutSeconds: ${LOAD_SOURCE_TIMEOUT_SECONDS:-20} + config: + provider: ${SOURCE_PROVIDER:-firecrawl-html} + - type: transform + name: clean + concurrencyGroup: process + config: + transformer: readability-default + target: text/html + emitDiagnostics: true + - type: transform + name: convert + concurrencyGroup: process + config: + transformer: mdream-convert + target: text/markdown + emitDiagnostics: true - type: truncate name: truncate config: @@ -94,7 +149,7 @@ pipelines: concurrencyGroup: source timeoutSeconds: ${LOAD_SOURCE_TIMEOUT_SECONDS:-20} config: - provider: ${SOURCE_PROVIDER:-default-firecrawl} + provider: ${SOURCE_PROVIDER:-firecrawl-markdown} - type: capture-urls name: capture_source_urls concurrencyGroup: source @@ -105,7 +160,7 @@ pipelines: concurrencyGroup: llm timeoutSeconds: ${CLEAN_TIMEOUT_SECONDS:-60} config: - provider: default-llm + provider: llm-default minInputChars: ${CLEAN_MIN_INPUT_CHARS:-1000} outputReserveRatio: 1.0 templates: @@ -123,7 +178,64 @@ pipelines: concurrencyGroup: llm timeoutSeconds: ${SUMMARIZE_TIMEOUT_SECONDS:-60} config: - provider: default-llm + provider: llm-default + minInputChars: ${OUTPUT_TARGET_CHARS:-25000} + outputReserveChars: ${OUTPUT_TARGET_CHARS:-25000} + templates: + system: ../templates/summarize.system.md + user: ../templates/summarize.user.md + vars: + targetChars: ${OUTPUT_TARGET_CHARS:-25000} + - type: verify-urls + name: verify_after_summarize + concurrencyGroup: llm + config: + artifact: trusted-urls + onHallucination: report + maxReportedUrls: 50 + - type: truncate + name: truncate + config: + targetChars: ${OUTPUT_TARGET_CHARS:-25000} + + clean-combined: + outputRenderer: ${DEFAULT_OUTPUT_RENDERER:-debug-xml} + limiters: + source: ${SOURCE_CONCURRENCY:-1} + process: ${PROCESS_CONCURRENCY:-5} + llm: ${LLM_CONCURRENCY:-1} + steps: + - type: load-source + name: fetch + concurrencyGroup: source + timeoutSeconds: ${LOAD_SOURCE_TIMEOUT_SECONDS:-20} + config: + provider: ${SOURCE_PROVIDER:-firecrawl-html} + - type: transform + name: clean + concurrencyGroup: process + config: + transformer: readability-default + target: text/html + emitDiagnostics: true + - type: transform + name: convert + concurrencyGroup: process + config: + transformer: mdream-convert + target: text/markdown + emitDiagnostics: true + - type: capture-urls + name: capture_source_urls + concurrencyGroup: process + config: + artifact: trusted-urls + - type: llm-pass + name: summarize + concurrencyGroup: llm + timeoutSeconds: ${SUMMARIZE_TIMEOUT_SECONDS:-60} + config: + provider: llm-default minInputChars: ${OUTPUT_TARGET_CHARS:-25000} outputReserveChars: ${OUTPUT_TARGET_CHARS:-25000} templates: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b9fc5e1..71d8a60 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -30,7 +30,7 @@ HTTP adapter `src/main.ts` parses bootstrap environment, creates the root logger, loads app assembly, builds Fastify, and installs shutdown handling. `src/adapters/http/http-app.ts` owns Fastify setup, request correlation, shared error handling, the silent `/health` route, and adapter registration. -`src/engine/create-engine.ts` is the programmatic engine boundary. It accepts `EngineConfig`, `EngineDescriptors`, `HostTools`, and a logger, then builds provider and output renderer registries, registers construction-time extension services, builds compiled pipelines internally, binds `PipelineHandle`s, and exposes `EngineRuntime`. +`src/engine/create-engine.ts` is the programmatic engine boundary. It accepts `EngineConfig`, `EngineDescriptors`, `HostTools`, and a logger, then builds provider, content transformer, and output renderer registries, registers construction-time extension services, builds compiled pipelines internally, binds `PipelineHandle`s, and exposes `EngineRuntime`. `src/app/compose.ts` is the service assembly point. It loads `AppConfig` from YAML, builds the host-tools bag, calls `createEngine(...)`, and creates HTTP adapter plugins over engine-provided pipeline handles. The composed result type is `ComposedApp` with two top-level keys: `adapters` (the `http` adapter set) and `registries` (`sourceProviders`, `llmProviders`, `outputRenderers`, and pipeline discovery info). @@ -44,11 +44,12 @@ src/ core/ framework-free pipeline engine (orchestrator, runner, effects, body store); `core/pipeline/` mirrors `contracts/pipeline/` contracts/ framework-free contracts grouped by intent pipeline/ pipeline ports and types: step, diagnostics, context, report, and handle definitions - extensions/ source provider, LLM provider, output renderer, registry, and resolved wrapper contracts + extensions/ source provider, content transformer, LLM provider, output renderer, registry, and resolved wrapper contracts host/ HostTools and ExtensionServices construction-time service contracts engine/ programmatic engine API, EngineConfig, EngineRuntime, createEngine, and engine-local builders builtins/ individual engine built-ins collected by descriptor bundles source-providers/ + content-transformers/ llm-providers/ pipeline-steps/ output-renderers/ @@ -67,8 +68,9 @@ The current built-ins are: - HTTP adapters: `open-webui`, `jina`. - Source providers: `http`, `firecrawl`, `docling`. +- Content transformers: `readability`, `mdream`. - LLM providers: `openai-chat`. -- Pipeline steps: `load-source`, `llm-pass`, `truncate`, `capture-urls`, `verify-urls`. +- Pipeline steps: `load-source`, `llm-pass`, `transform`, `truncate`, `capture-urls`, `verify-urls`. - Output renderers: `debug-xml`, `passthrough`. ## Dependency Boundaries @@ -82,6 +84,7 @@ The import graph is enforced by [tests/architecture/import-boundaries.test.ts](. - `src/adapters/http/` owns Fastify integration, HTTP adapter contracts, HTTP adapter construction, the HTTP descriptor bundle, and HTTP built-ins. Fastify imports are allowed only in this layer. - `src/adapters/http/builtins//` may import its own files, the shared adapter auth helper, HTTP adapter contracts, contracts, shared code, or external packages. - `src/builtins/source-providers//` and `src/builtins/llm-providers//` may import their own files, contracts, shared code, or external packages. +- `src/builtins/content-transformers//` may import their own files, contracts, shared code, or external packages. - `src/builtins/pipeline-steps//` may import their own files, contracts, shared code, or external packages. - `src/builtins/output-renderers//` may import their own files, contracts, shared code, or external packages. - `src/bundles/` collects engine built-in descriptors. It may import concrete engine built-ins, contracts, and shared helpers, but not HTTP adapter descriptors. @@ -166,7 +169,7 @@ Bootstrap environment is intentionally small and parsed by `src/config/env-confi The YAML document is translated to `AppConfig` before runtime construction: -1. `src/config/yaml/yaml-config.ts` validates the coarse YAML shape: `httpAdapters`, `outputRenderers`, `sourceProviders`, `llmProviders`, and `pipelines`. +1. `src/config/yaml/yaml-config.ts` validates the coarse YAML shape: `httpAdapters`, `outputRenderers`, `sourceProviders`, `contentTransformers`, `llmProviders`, and `pipelines`. 2. `src/config/yaml/yaml-app-config.ts` maps engine-owned sections to `AppConfig.engineConfig`, HTTP adapter declarations to `AppConfig.adapters.http`, and `schemaVersion` to app metadata. Pipeline activation follows a tri-state `enabled`: `true` compiles, `false` parks it, **omitted** activates only when an HTTP adapter references it. 3. Each built-in descriptor parses its own `config` block with a local schema during engine or adapter construction. diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 77947f8..f76c52e 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -47,9 +47,6 @@ These are consumed by helper scripts and ignored by the Node app. The default YAML file uses environment substitution for provider URLs, tokens, concurrency, timeouts, and renderer selection. Most of these values are shown in [.env.example](../.env.example) and passed through the Compose files. -The default (`truncate`) pipeline needs no provider configuration — it fetches and truncates. The `clean-llm` pipeline uses an LLM and Firecrawl; those env vars are -required only when `clean-llm` is active (set `DEFAULT_PIPELINE=clean-llm`). - Validation is lazy: the service validates and constructs only the providers and pipelines that are reachable from the active configuration. An unused provider with missing env vars will not fail startup. @@ -58,33 +55,47 @@ The placeholders below are grouped by the part of the pipeline they configure. ### Source provider selection -| Variable | Default / behavior | Purpose | -| ----------------- | ------------------ | -------------------------------------------------------------- | -| `SOURCE_PROVIDER` | `default-http` | Named source provider used by the `load-source` pipeline step. | +| Variable | Default / behavior | Purpose | +| ----------------- | -------------------------- | ---------------------------------------------------- | +| `SOURCE_PROVIDER` | selected pipeline fallback | Optional source-provider override for `load-source`. | -The built-in `http` provider has no environment variables — it fetches the input URL directly. +Leave `SOURCE_PROVIDER` empty to use the selected pipeline's fallback: `http-default` for `truncate`, `firecrawl-html` for `clean-deterministic` and `clean-combined`, and `firecrawl-markdown` for `clean-llm`. Set it only when intentionally overriding that choice. ### Source provider (Firecrawl) -| Variable | Default / behavior | Purpose | -| -------------------- | ------------------ | -------------------------------------------------------------------------------------------- | -| `FIRECRAWL_BASE_URL` | empty | Firecrawl base URL. Required only when a pipeline referencing `default-firecrawl` is active. | -| `FIRECRAWL_API_KEY` | empty | Optional Firecrawl bearer token. | +| Variable | Default / behavior | Purpose | +| -------------------- | ------------------ | ------------------------------------------------------------------------------------------------------------------ | +| `FIRECRAWL_BASE_URL` | empty | Firecrawl base URL. Required when any Firecrawl-based provider is active (`firecrawl-markdown`, `firecrawl-html`). | +| `FIRECRAWL_API_KEY` | empty | Optional Firecrawl bearer token. | ### Source provider (Docling) -| Variable | Default / behavior | Purpose | -| ------------------ | ------------------ | ---------------------------------------------------------------------------------------------- | -| `DOCLING_BASE_URL` | empty | Docling Serve base URL. Required only when a pipeline referencing `default-docling` is active. | -| `DOCLING_API_KEY` | empty | Optional Docling API key. | +| Variable | Default / behavior | Purpose | +| ------------------ | ------------------ | ------------------------------------------------------------------------------------------ | +| `DOCLING_BASE_URL` | empty | Docling Serve base URL. Required only when a pipeline referencing `docling-ocr` is active. | +| `DOCLING_API_KEY` | empty | Optional Docling API key. | + +### Content transformer (mdream) + +| Variable | Default / behavior | Purpose | +| -------------- | ------------------ | -------------------------------------------------------------------------------------------------------------------- | +| `MDREAM_CLEAN` | `true` | Post-conversion link and whitespace cleanup for `mdream-convert`. Ignored when `minimal=true` (`mdream-aggressive`). | + +### Content transformer (readability) + +| Variable | Default / behavior | Purpose | +| ------------------------------- | ------------------ | -------------------------------------------------------------------------------- | +| `READABILITY_MIN_CONTENT_CHARS` | `140` | Minimum content length for `isProbablyReaderable` gate in `readability-default`. | +| `READABILITY_MIN_SCORE` | `20` | Minimum readerable score for `isProbablyReaderable` gate. | +| `READABILITY_MAX_ELEMENTS` | `0` | Maximum DOM elements Readability parses; `0` = unlimited (DoS guardrail). | ### LLM provider (OpenAI-compatible) | Variable | Default / behavior | Purpose | | -------------------------- | ------------------ | --------------------------------------------------------------------------------------------------------------------------- | -| `LLM_BASE_URL` | empty | OpenAI-compatible API base URL, usually ending in `/v1`. Required only when a pipeline referencing `default-llm` is active. | +| `LLM_BASE_URL` | empty | OpenAI-compatible API base URL, usually ending in `/v1`. Required only when a pipeline referencing `llm-default` is active. | | `LLM_API_KEY` | empty | Optional LLM bearer token. | -| `LLM_MODEL` | empty | Model identifier sent to chat completions. Required only when a pipeline referencing `default-llm` is active. | +| `LLM_MODEL` | empty | Model identifier sent to chat completions. Required only when a pipeline referencing `llm-default` is active. | | `LLM_CONTEXT_TOKENS` | empty in YAML | Optional model context window in tokens. Empty disables the context-fit gate. | | `LLM_CHARS_PER_TOKEN` | `3.5` | Conservative chars-per-token estimator used for the context-fit gate. | | `LLM_SAFETY_MARGIN_TOKENS` | `128` | Extra tokens reserved for chat-template framing and estimator drift. | @@ -101,10 +112,11 @@ The built-in `http` provider has no environment variables — it fetches the inp ### Concurrency -| Variable | Default / behavior | Purpose | -| -------------------- | ------------------ | ------------------------------------------------------------- | -| `SOURCE_CONCURRENCY` | `1` | Maximum concurrent source-loading groups. | -| `LLM_CONCURRENCY` | `1` | Maximum concurrent LLM workflow groups (used by `clean-llm`). | +| Variable | Default / behavior | Purpose | +| --------------------- | ------------------ | ------------------------------------------------- | +| `SOURCE_CONCURRENCY` | `1` | Maximum concurrent source-loading groups. | +| `PROCESS_CONCURRENCY` | `5` | Maximum concurrent processing (transform) groups. | +| `LLM_CONCURRENCY` | `1` | Maximum concurrent LLM workflow groups. | ### Output rendering @@ -120,7 +132,7 @@ The built-in `http` provider has no environment variables — it fetches the inp | `OWUI_AUTH_TOKEN` | empty | Optional bearer token for the Open WebUI adapter route. | | `JINA_AUTH_TOKEN` | empty | Optional bearer token for the Jina-style adapter route. | -`.env.example` intentionally sets `LLM_CONTEXT_TOKENS` to a large example value so the context-fit gate is enabled in copied local configs. Leave it blank when you want to disable that gate. +`.env.example` shows practical local overrides for concurrency, output size, clean-pass thresholds, and `LLM_CONTEXT_TOKENS`. Leave an env value blank or unset it when you want the YAML fallback or schema default instead. ## Environment Substitution Syntax diff --git a/docs/CUSTOMIZATION.md b/docs/CUSTOMIZATION.md index 6dcb386..b38dede 100644 --- a/docs/CUSTOMIZATION.md +++ b/docs/CUSTOMIZATION.md @@ -15,6 +15,7 @@ schemaVersion: 1 httpAdapters: {} outputRenderers: {} sourceProviders: {} +contentTransformers: {} llmProviders: {} pipelines: {} ``` @@ -143,14 +144,22 @@ config: parsePdf: true ``` -| Knob | Values | Default | Purpose | -| ------------------- | --------------------------------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------- | -| `output` | `markdown` \| `html` \| `rawHtml` | `markdown` | Which format Firecrawl returns. `html` is cleaned main-content HTML. `rawHtml` is the full JS-rendered DOM (use for Readability testing). | -| `onlyMainContent` | bool | `true` | When enabled Firecrawl extracts the main page content and strips headers, nav, footers. No-op for `rawHtml`. | -| `stripBase64Images` | bool | `true` | Maps to `removeBase64Images` — replaces inline data URIs with short placeholders in markdown output. No-op for `html`/`rawHtml`. | -| `parsePdf` | bool | `true` | When enabled Firecrawl parses PDF files to markdown via `parsers: ["pdf"]`. | +| Knob | Values | Default | Purpose | +| ------------------- | --------------------------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `output` | `markdown` \| `html` \| `rawHtml` | `markdown` | Which format Firecrawl returns. `html` is cleaned main-content HTML. `rawHtml` is "raw" — full JS-rendered DOM for web pages, viewer `` wrapper for images, and bare extracted text for PDFs (use for Readability testing). | +| `onlyMainContent` | bool | `true` | When enabled Firecrawl extracts the main page content and strips headers, nav, footers. No-op for `rawHtml`. | +| `stripBase64Images` | bool | `true` | Maps to `removeBase64Images` — replaces inline data URIs with short placeholders in markdown output. No-op for `html`/`rawHtml`. | +| `parsePdf` | bool | `true` | Configurable via `FIRECRAWL_PARSE_PDF`. When `true`, Firecrawl parses PDF files to extracted text. When `false`, raw PDF bytes are returned as a binary body (`application/pdf`) — the pipeline fails with `unconverted_binary` until a PDF converter (e.g. Docling) is wired up. | -The provider calls `/v2/scrape`. Upstream HTTP, parse, empty, and network failures are converted to degradable upstream errors. Title is read from `data.metadata.title` (format-independent). `output: markdown` returns `text/markdown`; `output: html` and `output: rawHtml` return `text/html`. +The provider calls `/v2/scrape`. Upstream HTTP, parse, empty, and network failures are converted to degradable upstream errors. Title is read from `data.metadata.title` (format-independent). The media type of returned bodies is derived truthfully: + +| `output` | HTML / docx source | image source | PDF source (parsePdf:true) | +| -------- | ------------------ | --------------- | -------------------------- | +| markdown | `text/markdown` | `text/markdown` | `text/markdown` | +| html | `text/html` | `text/html` | `text/html` (wrapped) | +| rawHtml | `text/html` | `text/html` | **`text/plain`** | + +`rawHtml` is "raw": for web pages and images it returns HTML, but for PDFs it returns bare extracted text. The provider detects this by looking at whether the content starts with an HTML tag — if not, it labels it `text/plain` so downstream transformers (which gate on `isHtmlMediaType`) skip it correctly. When `parsePdf:false`, PDFs are returned as binary (`application/pdf`) from base64-decoded raw bytes supplied by Firecrawl's empty-`parsers` mode. ### `docling` @@ -171,6 +180,58 @@ config: The provider calls `POST /v1/convert/source`. Title is read from `document.json_content.name` (the JSON format is always requested internally regardless of the `output` setting). Upstream HTTP, parse, empty, and network failures are converted to degradable upstream errors. `output: markdown` returns `text/markdown`; `output: html` returns `text/html`. +## Content Transformers + +Content transformers convert a loaded body from one representation to another in-process, with no upstream call. The `transform` pipeline step selects a transformer by name and applies it to the current body. + +### `readability` + +Extracts main-article HTML from raw HTML using [Mozilla Readability](https://github.com/mozilla/readability). It uses `isProbablyReaderable` as a suitability gate: article-like pages become cleaned HTML; other pages return `declined` and keep the current body unchanged. + +```yaml +config: + minContentLength: ${READABILITY_MIN_CONTENT_CHARS:-140} + minScore: ${READABILITY_MIN_SCORE:-20} + maxElements: ${READABILITY_MAX_ELEMENTS:-0} +``` + +| Knob | Values | Default | Purpose | +| ------------------ | ------------ | ------- | ---------------------------------------------------------------------------------------------------- | +| `minContentLength` | positive int | `140` | Minimum content length for `isProbablyReaderable`. | +| `minScore` | number >= 0 | `20` | Minimum readerable score for `isProbablyReaderable`. | +| `maxElements` | int >= 0 | `0` | Maximum DOM elements Readability parses. `0` = unlimited (DoS guardrail). Maps to `maxElemsToParse`. | + +Supports text `text/html` or `application/xhtml+xml` bodies targeting `text/html`. The incoming title is preserved; when absent, the extracted article title is used instead. + +Decline reasons are `not_readerable` and `parse_empty`. Runtime errors propagate to the orchestrator; the failed step has no effect, so downstream steps continue from the prior body and the run rolls up as `degraded`. + +### `mdream` + +Converts HTML bodies to markdown using the [`@mdream/js`](https://www.npmjs.com/package/@mdream/js) library. Pure-JS, no native dependencies. + +Two shipped instances are declared in the config: + +**`mdream-convert`** — used after `readability` in shipped pipelines. It converts HTML to markdown without re-extracting main content. + +```yaml +config: + clean: ${MDREAM_CLEAN:-true} +``` + +**`mdream-aggressive`** — extraction + conversion in one pass using mdream's minimal preset. Suitable for custom pipelines without a prior readability step. + +```yaml +config: + minimal: true +``` + +| Knob | Values | Default | Purpose | +| --------- | ------ | ------- | -------------------------------------------------------------------------------------------------------- | +| `minimal` | bool | `false` | Apply mdream's minimal preset (isolate main content, filter boilerplate). Takes precedence over `clean`. | +| `clean` | bool | `true` | Clean up the markdown output: drop tracking params, redundant and empty links, and collapse blank lines. | + +The transformer supports text bodies whose media type is `text/html` or `application/xhtml+xml` and a requested target of `text/markdown`. The input URL is passed to mdream as the conversion origin, so relative links and images resolve against it. The original body title is preserved. + ## LLM Providers ### `openai-chat` @@ -214,7 +275,7 @@ pipelines: `outputRenderer` references a named output renderer instance. `limiters` declares concurrency group names and their maximum concurrency. A step can opt into a group with `concurrencyGroup`. -Adjacent steps that share the same `concurrencyGroup` share one limiter acquisition. In the default pipeline, `llm-pass(clean)`, `verify_after_clean`, `llm-pass(summarize)`, and `verify_after_summarize` all use the `llm` group, so a URL holds one LLM workflow slot across both passes and their URL checks. Inserting a step with a different (or no) group between them splits that acquisition. +Adjacent steps that share the same `concurrencyGroup` share one limiter acquisition. In the `clean-llm` pipeline, `llm-pass(clean)`, `verify_after_clean`, `llm-pass(summarize)`, and `verify_after_summarize` all use the `llm` group, so a URL holds one LLM workflow slot across both passes and their URL checks. Inserting a step with a different (or no) group between them splits that acquisition. `timeoutSeconds` applies to both limiter waiting and step execution. For one adjacent concurrency block, limiter waiting uses the longest `timeoutSeconds` value in that block. @@ -224,7 +285,7 @@ Adjacent steps that share the same `concurrencyGroup` share one limiter acquisit ```yaml config: - provider: ${SOURCE_PROVIDER:-default-http} + provider: ${SOURCE_PROVIDER:-http-default} ``` Loads the initial body from a named source provider. If a body already exists, the step skips with `body_present`. @@ -233,7 +294,7 @@ Loads the initial body from a named source provider. If a body already exists, t ```yaml config: - provider: default-llm + provider: llm-default minInputChars: 1500 maxInputChars: 80000 outputReserveRatio: 1.0 @@ -271,6 +332,28 @@ Both templates are required. `templates.vars` is an optional map of literal valu Mustache escaping is disabled for prompt templates so markdown is passed through as-is. +### `transform` + +```yaml +config: + transformer: mdream-convert + target: text/markdown + onUnsupported: skip + emitDiagnostics: false +``` + +Applies a named content transformer to the current body. The step resolves `transformer` from the `contentTransformers` registry and asks it to produce `target`. + +| Knob | Values | Default | Purpose | +| ----------------- | ---------------- | ------- | -------------------------------------------------------------------------------------------------------------- | +| `transformer` | string | — | Name of a configured `contentTransformers` instance. Required. | +| `target` | string | — | Media type the transformer must produce, for example `text/markdown`. Required. | +| `onUnsupported` | `skip` \| `fail` | `skip` | What to do when the transformer does not support the current body (wrong source media type or representation). | +| `onDeclined` | `skip` \| `fail` | `skip` | What to do when the transformer returns `declined` (e.g. `not_readerable`, `parse_empty`). | +| `emitDiagnostics` | bool | `false` | When enabled, transformer-reported diagnostics are surfaced as child nodes in the step report. | + +The step skips with `no_body` when there is no body. When the transformer does not support the current body it skips with `unsupported` (or fails with `onUnsupported: fail`). When the transformer returns `declined`, the step skips with that reason (or fails with `onDeclined: fail`). A transform aborted by the step timeout fails with `timeout`. If the transformer returns a body that does not match `target`, the step fails with `wrong_output_type`. + ### `truncate` ```yaml @@ -313,12 +396,38 @@ In both modes a hallucination makes the pipeline rollup `degraded` (an earlier b Shipped defaults: `verify_after_clean` uses `rollback` with `maxReportedUrls: 0` (silent rollback), `verify_after_summarize` uses `report` with `maxReportedUrls: 50`. -## Default Pipeline +## Shipped Pipelines + +The config ships four pipelines, selected via `DEFAULT_PIPELINE` (defaults to `truncate`). + +### `truncate` (default) + +```text +load-source(http-default) -> truncate +``` + +Fetches the URL directly and truncates to budget. No external services required. + +### `clean-deterministic` + +```text +load-source(firecrawl-html) -> transform(clean via readability) -> transform(convert via mdream) -> truncate +``` + +Loads Firecrawl raw HTML, extracts article HTML with `readability` when suitable, converts HTML to markdown with `mdream`, then truncates. No LLM passes. Pipeline fallback: `firecrawl-html`. Select with `DEFAULT_PIPELINE=clean-deterministic`. + +### `clean-llm` + +```text +load-source(firecrawl-markdown) -> capture-urls -> llm-pass(clean) -> verify-urls(rollback) -> llm-pass(summarize) -> verify-urls(report) -> truncate +``` + +The original reference pipeline. Loads markdown from a Firecrawl instance, runs clean and summarize LLM passes with URL-hallucination gates, then truncates. Pipeline fallback: `firecrawl-markdown`. Select with `DEFAULT_PIPELINE=clean-llm`. -The shipped default pipeline is: +### `clean-combined` ```text -load-source -> capture-urls -> llm-pass(clean) -> verify-urls(rollback) -> llm-pass(summarize) -> verify-urls(report) -> truncate +load-source(firecrawl-html) -> transform(clean via readability) -> transform(convert via mdream) -> capture-urls -> llm-pass(summarize) -> verify-urls(report) -> truncate ``` -That shape is a starting point, not a fixed contract. Prefer small YAML changes and verify behavior with a few representative URLs before relying on a customized pipeline. +Combines the deterministic HTML-to-markdown path with an LLM summarize pass. Pipeline fallback: `firecrawl-html`. Select with `DEFAULT_PIPELINE=clean-combined`. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 66b24d6..8a877ff 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -13,36 +13,25 @@ Implemented today: - `POST /` for Open WebUI's external web-loader contract. - `GET /r/` and `GET /r?url=` for limited Jina Reader-style URL-to-markdown compatibility. -- Firecrawl fetch provider using `/v2/scrape` with markdown output. -- OpenAI-compatible Chat Completions LLM provider. -- Optional clean stage, optional summarize stage, final truncation, and XML diagnostic footer. -- YAML-driven configuration for pipelines, providers, output renderers, and HTTP adapters. +- Firecrawl source provider using `/v2/scrape` with markdown, html, and rawHtml output. +- Docling source provider (experimental, PDFs and Office docs → markdown via Docling Serve API). +- Deterministic content transformers: Readability (article extraction) and mdream (HTML → markdown). +- Optional LLM clean stage, optional LLM summarize stage, final truncation, and XML diagnostic footer. +- YAML-driven configuration for pipelines, providers, content transformers, output renderers, and HTTP adapters. - Docker image published by CI to GHCR. Current provider implementations are intentionally few. The interfaces exist so replacements can be added without rewriting the use case. ## Direction -The biggest usability problem today is speed. Running every page through one or two LLM passes just to strip boilerplate is slow on limited hardware and invites hallucination. Most pages do not need a model to become clean Markdown. - -The direction is to make the default path deterministic and reserve the LLM for work only a model does well: - -```text -fetch (HTML or Markdown) - -> [HTML] extract main content - -> [HTML] convert HTML -> Markdown - -> [optional] LLM summarize / index - -> keep-honest repair (URLs, code blocks) - -> truncate to budget -``` - -The URL-in / clean-Markdown-within-a-budget contract does not change. What changes is how the middle is done: established extraction and conversion tools instead of a model. We own the plumbing — content-type routing, budgets, honesty gates — and the surfaces — HTTP/MCP/CLI, observability, caching. We do not write our own readability heuristics or HTML parser. +Deterministic extraction (readability + mdream) handles most pages in milliseconds. The next step is to make the pipeline self-aware: route through deterministic or LLM paths based on URL heuristics and per-step signals, reserving LLM passes for pages that need them. ## Coming soon -1. **Deterministic extract and convert steps.** First implementations: main-content extraction with Mozilla Readability (in-process, needs only a DOM such as linkedom) and HTML-to-Markdown with node-html-markdown or Turndown. Composed, these replace the LLM clean pass on the HTML path. -2. **Demote the LLM clean pass.** Once deterministic clean matches or beats it on representative URLs, make the clean stage optional and only execute it for URLs where deterministic clean did not produce desired results. Keep summarize. -3. **Keep summaries honest.** Extend the existing URL quality gate from all-or-nothing rejection into a deterministic repair pass, and implement fenced-code-block verification/repair. +1. **Conditional step execution based on signals.** Engine-level `skipIfSignal` / `runIfSignal` configuration on pipeline steps. When a signal name is set and the condition is met, the step is skipped entirely without running. +2. **URL heuristic step to classify input.** A pipeline step that matches the URL against patterns (path, domain, expected content type) and sets runtime signals for downstream steps. Use the feature to avoid Readibility step on GitHub domain etc. +3. **Route binary content to Docling.** Use the Docling source provider based on URL heuristics as an alternative to Firecrawl for PDFs and other binary content. +4. **Keep summaries honest.** Extend the existing URL quality gate from all-or-nothing rejection into a deterministic repair pass, and implement fenced-code-block verification/repair. A couple of polish items ride along: @@ -53,8 +42,9 @@ A couple of polish items ride along: Near-term additions once the pivot above settles. -- **Playwright fetch provider (HTML/DOM)** to remove the hard dependency on a running Firecrawl instance for HTML pages. -- **Docling for document conversion.** Use Docling to convert PDFs, Office documents, and other document-type files to markdown. +- **Playwright source provider** to remove the hard dependency on a running Firecrawl instance for HTML pages. +- **Docling content transformer** to convert PDFs, Office documents, and other document-type files to markdown. +- **Crawl4AI source provider** as a potentially better alternative to Firecrawl. ## Mid term diff --git a/docs/agents/architecture-rules.md b/docs/agents/architecture-rules.md index a2a9f7d..5587638 100644 --- a/docs/agents/architecture-rules.md +++ b/docs/agents/architecture-rules.md @@ -34,7 +34,7 @@ Classify the file you are touching before editing it. If the change does not fit ## Placement Rules -- New source provider, LLM provider, pipeline step, or output renderer: put implementation and descriptor under `src/builtins/...`, add its descriptor to `src/bundles/default-engine-descriptors.ts` only when it belongs in the default service bundle, and test under `tests/builtins/...`. +- New source provider, content transformer, LLM provider, pipeline step, or output renderer: put implementation and descriptor under `src/builtins/...`, add its descriptor to `src/bundles/default-engine-descriptors.ts` only when it belongs in the default service bundle, and test under `tests/builtins/...`. - New HTTP adapter: put implementation, config, descriptor, and adapter-local helpers under `src/adapters/http/builtins//`, add it to `src/adapters/http/descriptor-bundle.ts` only when it belongs in the hosted default bundle, and test under `tests/adapters/http/builtins//`. - New adapter surface such as MCP or CLI: create a sibling under `src/adapters//`, consume `EngineRuntime` or `PipelineHandle`, and update the architecture test in the same change. - New host singleton such as clock, cache, telemetry, persistence, or fetch policy: prefer a `HostTools` key in `src/contracts/host/host-tools.ts`, with the concrete host implementation wired from `src/app/`. diff --git a/docs/agents/extension-authoring.md b/docs/agents/extension-authoring.md index 6f70666..62d3d70 100644 --- a/docs/agents/extension-authoring.md +++ b/docs/agents/extension-authoring.md @@ -1,12 +1,12 @@ # Extension Authoring -This guide is for adding a new built-in implementation (source provider, LLM provider, pipeline step, output renderer, or HTTP adapter) inside this repository. Each category sits behind a small descriptor contract in its own folder, which keeps adding a built-in a localized change. Keep this guide and the live contracts in sync. +This guide is for adding a new built-in implementation (source provider, content transformer, LLM provider, pipeline step, output renderer, or HTTP adapter) inside this repository. Each category sits behind a small descriptor contract in its own folder, which keeps adding a built-in a localized change. Keep this guide and the live contracts in sync. Read [docs/ARCHITECTURE.md](../ARCHITECTURE.md) and [architecture-rules.md](architecture-rules.md) first. This document assumes you already know the runtime flow, dependency graph, and the three host-capability surfaces. ## Common Shape -All five extension categories follow the same shape: +All six extension categories follow the same shape: 1. A small **runtime instance interface** under `src/contracts/extensions/`, `src/contracts/pipeline/`, or the adapter surface describing behavior only. These interfaces do not carry `name` or `type`. 2. A **descriptor** included by the appropriate descriptor bundle. Each descriptor declares: @@ -63,6 +63,19 @@ Note the `.provider` deref — registries hold `Resolved*` wrappers. Implement `SourceProvider` (`src/contracts/extensions/source-provider.ts`). The interface has `load(url, { signal }): Promise`. Throw `UpstreamError` for HTTP, parse, or empty-response failures (use the constructors in `src/shared/errors.ts`); let abort errors propagate. Built-in example: `src/builtins/source-providers/firecrawl/`. +### Content Transformers + +Implement `ContentTransformer` (`src/contracts/extensions/content-transformer.ts`): `supports({ sourceKind, sourceMediaType, request })` and `transform({ url, body, request }, { signal }): Promise`. + +The result type is a discriminated union: + +- `{ outcome: "transformed", body: BodyContent, diagnostics?: ContentTransformDiagnostic[] }` — the transformer performed a conversion. The step only applies effects on this branch and runs its honesty gate (`outputMatchesTarget`). +- `{ outcome: "declined", reason?: string }` — the transformer chose not to transform (e.g. not suitable, empty parse). The step's `onDeclined` knob decides whether this becomes a skip or a failure. Declined outcomes never apply body effects, so the previous body version passes through unchanged. + +`supports` gates `transform`: the step only invokes a matching transformer, so `transform` may throw `InternalError` for inputs that bypass the gate. Use `declined` for deliberate "not suitable" decisions (e.g. `isProbablyReaderable` returned false). + +Built-in examples: `src/builtins/content-transformers/readability/` (article HTML extraction, outputs `declined`), `src/builtins/content-transformers/mdream/` (HTML to markdown, always `transformed`). + ### LLM Providers Implement `LlmProvider` (`src/contracts/extensions/llm-provider.ts`). Same error rules as source providers. Built-in example: `src/builtins/llm-providers/openai-chat/`. diff --git a/docs/agents/smoke-testing.md b/docs/agents/smoke-testing.md index d00e86a..4d8bd01 100644 --- a/docs/agents/smoke-testing.md +++ b/docs/agents/smoke-testing.md @@ -16,9 +16,10 @@ Artifacts land in `scripts/out/` (gitignored). ## Picking URLs -- Keep the set small — **5 URLs or fewer**. Each provider × surface × environment combination reruns the whole set, so it adds up fast. +- Keep the set small — **5 URLs or fewer** unless the task explicitly asks for broader coverage. Each provider × surface × environment combination reruns the whole set, so it adds up fast. - Cover a couple of content shapes: at least one ordinary HTML article, and at least one real document (a true PDF, confirmed by content type rather than a `.pdf` in the path). -- Skip raw images; they only produce binary-rejection results and teach nothing new. +- Verify selected URLs with a lightweight `GET` and record content type before running the matrix. `HEAD` is not enough: some sites reject it, and docs URLs rot into 404s. +- Skip raw images unless the task asks for a negative/control URL; they usually produce binary-rejection or tiny metadata-only results. ```powershell # gather-urls.ps1 reads SEARX_BASE from the environment and does NOT load .env. @@ -50,6 +51,9 @@ npm run dev ./scripts/smoke-jina.ps1 -UrlFile scripts/out/urls-smoke.txt -Concurrency 2 ./scripts/smoke-owui.ps1 -UrlFile scripts/out/urls-smoke.txt -Concurrency 2 -BatchSize 3 +# Quick inline smoke test (no script needed): +curl.exe -s --max-time 30 "http://localhost:3010/r/$([System.Uri]::EscapeDataString('https://en.wikipedia.org/wiki/Rust_(programming_language)'))" | Select-String -Pattern '=20.19.0" + } + }, + "node_modules/@mdream/js/node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "license": "MIT" + }, + "node_modules/@mozilla/readability": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.6.0.tgz", + "integrity": "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@pinojs/redact": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", @@ -1370,6 +1410,12 @@ "node": "18 || 20 || >=22" } }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC" + }, "node_modules/brace-expansion": { "version": "5.0.6", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", @@ -1473,6 +1519,40 @@ "node": ">= 8" } }, + "node_modules/css-select": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", + "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", + "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cssom": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", + "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==", + "license": "MIT" + }, "node_modules/dateformat": { "version": "4.6.3", "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", @@ -1517,6 +1597,61 @@ "node": ">=6" } }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -1540,6 +1675,18 @@ "once": "^1.4.0" } }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/es-module-lexer": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", @@ -1837,6 +1984,37 @@ "dev": true, "license": "MIT" }, + "node_modules/htmlparser2": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "entities": "^7.0.1" + } + }, + "node_modules/htmlparser2/node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/ipaddr.js": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.4.0.tgz", @@ -1999,6 +2177,36 @@ } ] }, + "node_modules/linkedom": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.12.tgz", + "integrity": "sha512-jalJsOwIKuQJSeTvsgzPe9iJzyfVaEJiEXl+25EkKevsULHvMJzpNqwvj1jOESWdmgKDiXObyjOYwlUqG7wo1Q==", + "license": "ISC", + "dependencies": { + "css-select": "^5.1.0", + "cssom": "^0.5.0", + "html-escaper": "^3.0.3", + "htmlparser2": "^10.0.0", + "uhyphen": "^0.2.0" + }, + "engines": { + "node": ">=16" + }, + "peerDependencies": { + "canvas": ">= 2" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/linkedom/node_modules/html-escaper": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz", + "integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==", + "license": "MIT" + }, "node_modules/loupe": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", @@ -2119,6 +2327,18 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/on-exit-leak-free": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", @@ -2811,6 +3031,12 @@ "node": ">=14.17" } }, + "node_modules/uhyphen": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/uhyphen/-/uhyphen-0.2.0.tgz", + "integrity": "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==", + "license": "ISC" + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", diff --git a/package.json b/package.json index 8ba2952..c56dd2d 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,10 @@ "format:check": "prettier --check \"*.ts\" \"src/**/*.ts\" \"tests/**/*.ts\"" }, "dependencies": { + "@mdream/js": "^1.3.0", + "@mozilla/readability": "^0.6.0", "fastify": "^5.2.1", + "linkedom": "^0.18.12", "mustache": "^4.2.0", "p-limit": "^6.2.0", "pino": "^9.6.0", diff --git a/scripts/inspect-suspects.ps1 b/scripts/inspect-suspects.ps1 index 44d609b..0a4d0bd 100644 --- a/scripts/inspect-suspects.ps1 +++ b/scripts/inspect-suspects.ps1 @@ -3,7 +3,7 @@ # poor cleanup result. # # For each URL three artifacts are written to scripts/out/: -# .source.md - raw Firecrawl markdown +# .source.html - raw HTML from Firecrawl (universal baseline for any pipeline) # .clean.md - loader's /r/ body (including footer) # .footer.txt - extracted line (or empty) # @@ -95,7 +95,7 @@ foreach ($url in $Urls) { Write-Host "`n=== $slug ===" -ForegroundColor Cyan Write-Host " $url" -ForegroundColor DarkGray - $body = @{ url = $url; formats = @('markdown'); onlyMainContent = $true } | ConvertTo-Json -Compress + $body = @{ url = $url; formats = @('rawHtml') } | ConvertTo-Json -Compress $src = '' try { @@ -103,7 +103,7 @@ foreach ($url in $Urls) { -Uri "$($firecrawl.BaseUrl)/v2/scrape" ` -Headers $firecrawlHeaders -ContentType 'application/json' ` -Body $body -TimeoutSec 120 - $src = $firecrawlResponse.data.markdown + $src = $firecrawlResponse.data.rawHtml if (-not $src) { $src = $firecrawlResponse.data.content } } catch { @@ -111,7 +111,7 @@ foreach ($url in $Urls) { $fail++ continue } - Set-Content -LiteralPath (Join-Path $outDir "$slug.source.md") -Value $src -Encoding utf8 + Set-Content -LiteralPath (Join-Path $outDir "$slug.source.html") -Value $src -Encoding utf8 $clean = '' try { diff --git a/src/builtins/content-transformers/mdream/mdream-transformer-config.ts b/src/builtins/content-transformers/mdream/mdream-transformer-config.ts new file mode 100644 index 0000000..e795f85 --- /dev/null +++ b/src/builtins/content-transformers/mdream/mdream-transformer-config.ts @@ -0,0 +1,33 @@ +/** + * Parses YAML configuration for the built-in mdream content transformer. + * + * Environment substitution runs before this schema, so boolean fields use the + * shared preprocessor to treat blank placeholders as omitted values. These knobs + * tune transformer behavior per instance; routing intent (the target media type) + * comes from the requesting step, not this config. + * + * Note on `minimal`: withMinimalPreset returns 0 HTML-to-markdown chars when fed + * readability-extracted article HTML from table-heavy layouts (HN, Quora, etc.) + * because it tries to re-extract main content from already-extracted markup. + * Default false is safe when readability precedes mdream; use minimal=true only + * in pipelines without a prior extraction step. + */ + +import { z } from "zod"; + +import { booleanStringAsBooleanOrUndefined } from "../../../shared/config-coercion.js"; + +const mdreamTransformerConfigSchema = z + .object({ + minimal: z.preprocess(booleanStringAsBooleanOrUndefined, z.boolean().default(false)), + clean: z.preprocess(booleanStringAsBooleanOrUndefined, z.boolean().default(true)) + }) + .strict(); + +/** Represents parsed mdream transformer configuration. */ +export type MdreamTransformerConfig = z.infer; + +/** Parses mdream transformer configuration from YAML. */ +export function parseMdreamTransformerConfig(raw: unknown): MdreamTransformerConfig { + return Object.freeze(mdreamTransformerConfigSchema.parse(raw)); +} diff --git a/src/builtins/content-transformers/mdream/mdream-transformer-descriptor.ts b/src/builtins/content-transformers/mdream/mdream-transformer-descriptor.ts new file mode 100644 index 0000000..d832d30 --- /dev/null +++ b/src/builtins/content-transformers/mdream/mdream-transformer-descriptor.ts @@ -0,0 +1,20 @@ +/** + * Exposes the mdream content transformer descriptor to engine bundles. + * + * The transformer is a self-contained leaf with no host tool or service + * dependencies; configured identity is added later by engine registry + * construction. + */ + +import { parseMdreamTransformerConfig } from "./mdream-transformer-config.js"; +import { MdreamTransformer } from "./mdream-transformer.js"; + +import type { ContentTransformerDescriptor } from "../../../contracts/extensions/content-transformer.js"; +import type { MdreamTransformerConfig } from "./mdream-transformer-config.js"; + +/** Defines the built-in mdream content transformer type. */ +export const mdreamTransformerDescriptor = { + type: "mdream", + parseConfig: parseMdreamTransformerConfig, + create: args => new MdreamTransformer(args.config, { logger: args.deps.logger }) +} satisfies ContentTransformerDescriptor; diff --git a/src/builtins/content-transformers/mdream/mdream-transformer.ts b/src/builtins/content-transformers/mdream/mdream-transformer.ts new file mode 100644 index 0000000..aa49a12 --- /dev/null +++ b/src/builtins/content-transformers/mdream/mdream-transformer.ts @@ -0,0 +1,74 @@ +/** + * Converts HTML bodies to Markdown using the pure-JS mdream engine. + * + * `htmlToMarkdown` is synchronous and has no native dependencies. The instance + * passes the source URL as the mdream `origin` so relative links and images + * resolve to absolute URLs. The `minimal` preset additionally isolates main + * content and filters boilerplate; `clean` applies link and whitespace cleanup. + */ + +import { htmlToMarkdown, withMinimalPreset, type MdreamOptions } from "@mdream/js"; + +import { InternalError } from "../../../shared/errors.js"; +import { isHtmlMediaType, mediaTypes } from "../../../shared/media-types.js"; + +import type { + ContentTransformDiagnostic, + ContentTransformRequest, + ContentTransformResult, + ContentTransformer +} from "../../../contracts/extensions/content-transformer.js"; +import type { BodyContent } from "../../../contracts/pipeline/context.js"; +import type { Logger } from "../../../shared/logger.js"; +import type { MdreamTransformerConfig } from "./mdream-transformer-config.js"; + +/** Transforms HTML text bodies into Markdown via mdream. */ +export class MdreamTransformer implements ContentTransformer { + /** Creates an mdream transformer instance. */ + constructor( + private readonly config: MdreamTransformerConfig, + private readonly deps: { readonly logger: Logger } + ) {} + + /** Supports HTML or XHTML text bodies targeting Markdown output. */ + supports(input: { + readonly sourceKind: BodyContent["kind"]; + readonly sourceMediaType: string; + readonly request: ContentTransformRequest; + }): boolean { + return ( + input.sourceKind === "text" && + isHtmlMediaType(input.sourceMediaType) && + input.request.targetMediaType === mediaTypes.markdown + ); + } + + /** Renders the HTML body to Markdown, preserving the source title. */ + async transform( + input: { readonly url: string; readonly body: BodyContent; readonly request: ContentTransformRequest }, + opts: { readonly signal: AbortSignal } + ): Promise { + opts.signal.throwIfAborted(); + const body = input.body; + if (body.kind !== "text") + throw new InternalError("mdream transformer requires a text body", "mdream_non_text_body"); + + const markdown = htmlToMarkdown(body.content, this.buildOptions(input.url)).trim(); + const diagnostics: ContentTransformDiagnostic[] = [ + { code: "mdream", message: `${body.content.length} html chars -> ${markdown.length} markdown chars` } + ]; + if (!markdown) diagnostics.push({ code: "empty_output" }); + + return { + outcome: "transformed", + body: { kind: "text", mediaType: mediaTypes.markdown, content: markdown, title: body.title }, + diagnostics + }; + } + + /** Builds mdream options from instance config, threading the source URL. */ + private buildOptions(origin: string): Partial { + const base: Partial = { origin }; + return this.config.minimal ? withMinimalPreset(base) : { ...base, clean: this.config.clean }; + } +} diff --git a/src/builtins/content-transformers/readability/readability-transformer-config.ts b/src/builtins/content-transformers/readability/readability-transformer-config.ts new file mode 100644 index 0000000..7b722a7 --- /dev/null +++ b/src/builtins/content-transformers/readability/readability-transformer-config.ts @@ -0,0 +1,31 @@ +/** + * Parses YAML configuration for the built-in readability content transformer. + * + * The three knobs mirror the Readability.js suitability gate and parsing + * limits: minContentLength and minScore feed `isProbablyReaderable`, while + * maxElements caps the internal parse buffer as a DoS guardrail. A value of 0 + * for maxElements means unlimited. + * + * Environment substitution runs before this schema, so number fields use the + * shared preprocessors to treat blank placeholders as omitted values. + */ + +import { z } from "zod"; + +import { emptyStringAsUndefined } from "../../../shared/config-coercion.js"; + +const readabilityTransformerConfigSchema = z + .object({ + minContentLength: z.preprocess(emptyStringAsUndefined, z.coerce.number().int().positive().default(140)), + minScore: z.preprocess(emptyStringAsUndefined, z.coerce.number().min(0).default(20)), + maxElements: z.preprocess(emptyStringAsUndefined, z.coerce.number().int().min(0).default(0)) + }) + .strict(); + +/** Represents parsed readability transformer configuration. */ +export type ReadabilityTransformerConfig = z.infer; + +/** Parses readability transformer configuration from YAML. */ +export function parseReadabilityTransformerConfig(raw: unknown): ReadabilityTransformerConfig { + return Object.freeze(readabilityTransformerConfigSchema.parse(raw)); +} diff --git a/src/builtins/content-transformers/readability/readability-transformer-descriptor.ts b/src/builtins/content-transformers/readability/readability-transformer-descriptor.ts new file mode 100644 index 0000000..7b58813 --- /dev/null +++ b/src/builtins/content-transformers/readability/readability-transformer-descriptor.ts @@ -0,0 +1,20 @@ +/** + * Exposes the readability content transformer descriptor to engine bundles. + * + * The transformer uses the Mozilla Readability library via linkedom to extract + * article-level content from raw HTML. Construction resolves config; runtime + * only needs the config and logger since all DOM work is in-process. + */ + +import { parseReadabilityTransformerConfig } from "./readability-transformer-config.js"; +import { ReadabilityTransformer } from "./readability-transformer.js"; + +import type { ContentTransformerDescriptor } from "../../../contracts/extensions/content-transformer.js"; +import type { ReadabilityTransformerConfig } from "./readability-transformer-config.js"; + +/** Defines the built-in readability content transformer type. */ +export const readabilityTransformerDescriptor = { + type: "readability", + parseConfig: parseReadabilityTransformerConfig, + create: args => new ReadabilityTransformer(args.config, { logger: args.deps.logger }) +} satisfies ContentTransformerDescriptor; diff --git a/src/builtins/content-transformers/readability/readability-transformer.ts b/src/builtins/content-transformers/readability/readability-transformer.ts new file mode 100644 index 0000000..4e51efa --- /dev/null +++ b/src/builtins/content-transformers/readability/readability-transformer.ts @@ -0,0 +1,95 @@ +/** + * Extracts main article HTML from raw HTML using Mozilla Readability. + * + * Given a raw HTML page, this transformer first checks whether the document + * *seems* readerable via `isProbablyReaderable`. If not, it returns a + * "declined" outcome so the pipeline can fall through to downstream steps + * unchanged. If the gate passes, the document is parsed with Readability to + * extract the article content. The output is *still HTML* (not markdown), so + * another transformer (typically mdream) further downstream converts it. + */ + +import { Readability, isProbablyReaderable } from "@mozilla/readability"; +import { parseHTML } from "linkedom"; + +import { InternalError } from "../../../shared/errors.js"; +import { isHtmlMediaType, mediaTypes } from "../../../shared/media-types.js"; + +import type { + ContentTransformDiagnostic, + ContentTransformResult, + ContentTransformer +} from "../../../contracts/extensions/content-transformer.js"; +import type { BodyContent } from "../../../contracts/pipeline/context.js"; +import type { Logger } from "../../../shared/logger.js"; +import type { ReadabilityTransformerConfig } from "./readability-transformer-config.js"; + +/** Transforms raw HTML text bodies into cleaned article HTML via Readability. */ +export class ReadabilityTransformer implements ContentTransformer { + /** Creates a Readability transformer instance. */ + constructor( + private readonly config: ReadabilityTransformerConfig, + private readonly deps: { readonly logger: Logger } + ) {} + + /** Supports HTML or XHTML text bodies targeting HTML output. */ + supports(input: { + readonly sourceKind: BodyContent["kind"]; + readonly sourceMediaType: string; + readonly request: { readonly targetMediaType: string }; + }): boolean { + return ( + input.sourceKind === "text" && + isHtmlMediaType(input.sourceMediaType) && + input.request.targetMediaType === mediaTypes.html + ); + } + + /** Attempts to extract article content; declines when the page isn't article-like. */ + async transform( + input: { readonly url: string; readonly body: BodyContent; readonly request: { readonly targetMediaType: string } }, + opts: { readonly signal: AbortSignal } + ): Promise { + opts.signal.throwIfAborted(); + + const body = input.body; + if (body.kind !== "text") + throw new InternalError("readability transformer requires a text body", "readability_non_text_body"); + + const dom = parseHTML(body.content); + const doc = dom.document as unknown as Document; + + if (!doc.documentElement) return { outcome: "declined", reason: "parse_empty" }; + + if ( + !isProbablyReaderable(doc, { + minContentLength: this.config.minContentLength, + minScore: this.config.minScore + }) + ) + return { outcome: "declined", reason: "not_readerable" }; + + const article = new Readability(doc, { maxElemsToParse: this.config.maxElements || undefined }).parse(); + if (!article || !article.content) return { outcome: "declined", reason: "parse_empty" }; + + const originalLength = body.content.length; + const articleLength = article.content.length; + const diagnostics: ContentTransformDiagnostic[] = [ + { + code: "readability", + message: `${originalLength} html chars -> ${articleLength} article chars` + } + ]; + + return { + outcome: "transformed", + body: { + kind: "text", + mediaType: mediaTypes.html, + content: article.content, + title: body.title || (article.title ?? undefined) + }, + diagnostics + }; + } +} diff --git a/src/builtins/pipeline-steps/transform/transform-step-config.ts b/src/builtins/pipeline-steps/transform/transform-step-config.ts new file mode 100644 index 0000000..d546738 --- /dev/null +++ b/src/builtins/pipeline-steps/transform/transform-step-config.ts @@ -0,0 +1,37 @@ +/** + * Parses YAML configuration for the built-in transform pipeline step. + * + * The step is generic: `transformer` selects a configured content transformer + * instance and `target` is the required output media type the step both requests + * and asserts. `onUnsupported` decides whether an unhandled body skips or fails. + */ + +import { z } from "zod"; + +import { booleanStringAsBooleanOrUndefined } from "../../../shared/config-coercion.js"; + +const transformStepConfigSchema = z + .object({ + transformer: z.string().min(1, "transform step requires a transformer name"), + target: z.string().min(1, "transform step requires a target media type"), + onUnsupported: z.enum(["skip", "fail"]).default("skip"), + onDeclined: z.enum(["skip", "fail"]).default("skip"), + emitDiagnostics: z.preprocess(booleanStringAsBooleanOrUndefined, z.boolean().default(false)) + }) + .strict(); + +/** Represents parsed YAML configuration for a transform step. */ +export type TransformStepConfig = z.infer; + +/** Describes constructor configuration for a transform step instance. */ +export interface TransformStepOptions { + readonly target: string; + readonly onUnsupported: "skip" | "fail"; + readonly onDeclined: "skip" | "fail"; + readonly emitDiagnostics: boolean; +} + +/** Parses transform step configuration from YAML. */ +export function parseTransformStepConfig(raw: unknown): TransformStepConfig { + return Object.freeze(transformStepConfigSchema.parse(raw)); +} diff --git a/src/builtins/pipeline-steps/transform/transform-step-descriptor.ts b/src/builtins/pipeline-steps/transform/transform-step-descriptor.ts new file mode 100644 index 0000000..a6fe497 --- /dev/null +++ b/src/builtins/pipeline-steps/transform/transform-step-descriptor.ts @@ -0,0 +1,33 @@ +/** + * Exposes the transform pipeline step descriptor to engine bundles. + * + * Construction resolves the configured content transformer by name so runtime + * step execution only depends on the transformer port, target, and logger. + */ + +import { contentTransformerRegistryKey } from "../../../contracts/extensions/content-transformer.js"; +import { parseTransformStepConfig } from "./transform-step-config.js"; +import { TransformStep } from "./transform-step.js"; + +import type { PipelineStepDescriptor } from "../../../contracts/pipeline/step.js"; +import type { TransformStepConfig } from "./transform-step-config.js"; + +/** Defines the built-in transform step type. */ +export const transformStepDescriptor = { + type: "transform", + parseConfig: parseTransformStepConfig, + create: args => { + const transformer = args.services + .require(contentTransformerRegistryKey) + .require(args.config.transformer).transformer; + return new TransformStep( + { + target: args.config.target, + onUnsupported: args.config.onUnsupported, + onDeclined: args.config.onDeclined, + emitDiagnostics: args.config.emitDiagnostics + }, + { transformer, logger: args.deps.logger } + ); + } +} satisfies PipelineStepDescriptor; diff --git a/src/builtins/pipeline-steps/transform/transform-step.ts b/src/builtins/pipeline-steps/transform/transform-step.ts new file mode 100644 index 0000000..5bd6663 --- /dev/null +++ b/src/builtins/pipeline-steps/transform/transform-step.ts @@ -0,0 +1,79 @@ +/** + * Runs a configured content transformer against the current pipeline body. + * + * The step is transformer-agnostic. It asks the resolved transformer whether it + * supports the current body for the requested target, runs it, then asserts the + * result matches the requested target media type and representation before + * applying it. This intrinsic honesty gate keeps transformers truthful. + */ + +import { isAbortError } from "../../../shared/errors.js"; +import { isTextLike } from "../../../shared/media-types.js"; + +import type { + ContentTransformDiagnostic, + ContentTransformRequest, + ContentTransformer +} from "../../../contracts/extensions/content-transformer.js"; +import type { BodyContent, PipelineContext } from "../../../contracts/pipeline/context.js"; +import type { StepDiagnostics } from "../../../contracts/pipeline/diagnostics.js"; +import type { PipelineStep, StepResult } from "../../../contracts/pipeline/step.js"; +import type { Logger } from "../../../shared/logger.js"; +import type { TransformStepOptions } from "./transform-step-config.js"; + +/** Transforms the current body toward a target representation via a transformer. */ +export class TransformStep implements PipelineStep { + /** Creates a transform step instance. */ + constructor( + private readonly config: TransformStepOptions, + private readonly deps: { readonly transformer: ContentTransformer; readonly logger: Logger } + ) {} + + /** Runs the transformer when it supports the current body for the target. */ + async run(ctx: PipelineContext): Promise { + const body = ctx.body.current(); + if (!body) return { status: "skipped", reason: "no_body" }; + + const request: ContentTransformRequest = { targetMediaType: this.config.target }; + if (!this.deps.transformer.supports({ sourceKind: body.kind, sourceMediaType: body.mediaType, request })) + return { status: this.config.onUnsupported === "fail" ? "failed" : "skipped", reason: "unsupported" }; + + let result; + try { + result = await this.deps.transformer.transform({ url: ctx.input.url, body, request }, { signal: ctx.signal }); + } catch (error) { + if (isAbortError(error)) return { status: "failed", reason: "timeout" }; + throw error; + } + + if (result.outcome === "declined") + return { status: this.config.onDeclined === "fail" ? "failed" : "skipped", reason: result.reason ?? "declined" }; + + if (!outputMatchesTarget(result.body, this.config.target)) return { status: "failed", reason: "wrong_output_type" }; + + return { + status: "ok", + effects: { body: result.body }, + diagnostics: this.config.emitDiagnostics ? toStepDiagnostics(result.diagnostics) : undefined + }; + } +} + +/** Returns whether a transformer result matches the requested target type and representation. */ +function outputMatchesTarget(body: BodyContent, target: string): boolean { + if (body.mediaType !== target) return false; + return isTextLike(target) ? body.kind === "text" : body.kind === "binary"; +} + +/** Maps transformer diagnostics into a step diagnostics tree, or undefined when empty. */ +function toStepDiagnostics( + diagnostics: ReadonlyArray | undefined +): StepDiagnostics | undefined { + if (!diagnostics || diagnostics.length === 0) return undefined; + return { + children: diagnostics.map(diagnostic => ({ + name: diagnostic.code, + attributes: diagnostic.message === undefined ? undefined : { message: diagnostic.message } + })) + }; +} diff --git a/src/builtins/source-providers/firecrawl/firecrawl-provider.ts b/src/builtins/source-providers/firecrawl/firecrawl-provider.ts index 9c595c3..1b1febd 100644 --- a/src/builtins/source-providers/firecrawl/firecrawl-provider.ts +++ b/src/builtins/source-providers/firecrawl/firecrawl-provider.ts @@ -10,7 +10,7 @@ import { z } from "zod"; import { UpstreamError, isAbortError } from "../../../shared/errors.js"; -import { mediaTypes } from "../../../shared/media-types.js"; +import { looksLikeHtml, mediaTypes } from "../../../shared/media-types.js"; import { joinUrl } from "../../../shared/urls.js"; import type { SourceDocument, SourceProvider } from "../../../contracts/extensions/source-provider.js"; @@ -93,7 +93,28 @@ export class FirecrawlProvider implements SourceProvider { upstreamStatus: response.status }); const title = data.title ?? (typeof metadata.title === "string" ? metadata.title : undefined); - const mediaType: string = this.config.output === "markdown" ? mediaTypes.markdown : mediaTypes.html; + const contentType = typeof metadata.contentType === "string" ? metadata.contentType : ""; + + if (!this.config.parsePdf) { + const essence = contentType.split(";", 1)[0].trim().toLowerCase(); + if (essence === mediaTypes.pdf) { + const raw = extractBase64Payload(content, this.config.output); + if (!raw) { + throw new UpstreamError("Firecrawl returned empty base64 PDF payload", "empty", { + upstreamStatus: response.status + }); + } + const bytes = Buffer.from(raw, "base64"); + if (bytes.byteLength === 0) { + throw new UpstreamError("Firecrawl returned empty PDF bytes after base64 decode", "empty", { + upstreamStatus: response.status + }); + } + return { kind: "binary", bytes, mediaType: mediaTypes.pdf, title }; + } + } + + const mediaType = deriveTextMediaType(this.config.output, content); return { kind: "text", content, mediaType, title }; } catch (error) { if (error instanceof UpstreamError || isAbortError(error)) throw error; @@ -108,3 +129,30 @@ export class FirecrawlProvider implements SourceProvider { return headers; } } + +/** + * Derives a truthful media type for the text returned by Firecrawl. + * + * - markdown output is always `text/markdown`. + * - html output is always `text/html` (Firecrawl wraps non-HTML content). + * - rawHtml is "raw": real HTML for web pages/sites, bare text for PDFs. + */ +function deriveTextMediaType(output: string, content: string): string { + if (output === "markdown") return mediaTypes.markdown; + if (output === "html") return mediaTypes.html; + return looksLikeHtml(content) ? mediaTypes.html : mediaTypes.plainText; +} + +/** + * Extracts the base64-encoded payload from a Firecrawl response field. + * + * With parsers:[], Firecrawl returns base64 of the raw file bytes. When + * output=html, the base64 is wrapped in `…`; + * for markdown and rawHtml it is bare. + */ +function extractBase64Payload(content: string, output: string): string | undefined { + const trimmed = content.trim(); + if (!trimmed) return undefined; + if (output !== "html") return trimmed; + return trimmed.replace(/^()?/i, "").replace(/(<\/body>)?<\/html>$/i, ""); +} diff --git a/src/bundles/default-engine-descriptors.ts b/src/bundles/default-engine-descriptors.ts index d0b48a8..2a28b32 100644 --- a/src/bundles/default-engine-descriptors.ts +++ b/src/bundles/default-engine-descriptors.ts @@ -6,12 +6,15 @@ * engine construction never imports this bundle directly. */ +import { mdreamTransformerDescriptor } from "../builtins/content-transformers/mdream/mdream-transformer-descriptor.js"; +import { readabilityTransformerDescriptor } from "../builtins/content-transformers/readability/readability-transformer-descriptor.js"; import { openAiChatProviderDescriptor } from "../builtins/llm-providers/openai-chat/openai-chat-provider-descriptor.js"; import { debugXmlRendererDescriptor } from "../builtins/output-renderers/debug-xml/debug-xml-renderer-descriptor.js"; import { passthroughRendererDescriptor } from "../builtins/output-renderers/passthrough/passthrough-renderer-descriptor.js"; import { captureUrlsStepDescriptor } from "../builtins/pipeline-steps/capture-urls/capture-urls-step-descriptor.js"; import { llmPassStepDescriptor } from "../builtins/pipeline-steps/llm-pass/llm-pass-step-descriptor.js"; import { loadSourceStepDescriptor } from "../builtins/pipeline-steps/load-source/load-source-step-descriptor.js"; +import { transformStepDescriptor } from "../builtins/pipeline-steps/transform/transform-step-descriptor.js"; import { truncateStepDescriptor } from "../builtins/pipeline-steps/truncate/truncate-step-descriptor.js"; import { verifyUrlsStepDescriptor } from "../builtins/pipeline-steps/verify-urls/verify-urls-step-descriptor.js"; import { httpProviderDescriptor } from "../builtins/source-providers/http/http-provider-descriptor.js"; @@ -19,6 +22,7 @@ import { firecrawlProviderDescriptor } from "../builtins/source-providers/firecr import { doclingProviderDescriptor } from "../builtins/source-providers/docling/docling-provider-descriptor.js"; import { createDescriptorRecord } from "../shared/descriptors.js"; +import type { ContentTransformerDescriptor } from "../contracts/extensions/content-transformer.js"; import type { LlmProviderDescriptor } from "../contracts/extensions/llm-provider.js"; import type { OutputRendererDescriptor } from "../contracts/extensions/output-renderer.js"; import type { SourceProviderDescriptor } from "../contracts/extensions/source-provider.js"; @@ -27,6 +31,7 @@ import type { PipelineStepDescriptor } from "../contracts/pipeline/step.js"; /** Engine-facing descriptor bundle selected by the hosted service by default. */ export interface EngineDescriptorBundle { readonly sourceProviders: Readonly>; + readonly contentTransformers: Readonly>; readonly llmProviders: Readonly>; readonly outputRenderers: Readonly>; readonly pipelineSteps: Readonly>; @@ -38,6 +43,10 @@ export const DEFAULT_ENGINE_DESCRIPTOR_BUNDLE: EngineDescriptorBundle = Object.f [httpProviderDescriptor, firecrawlProviderDescriptor, doclingProviderDescriptor], descriptor => descriptor.type ), + contentTransformers: createDescriptorRecord( + [mdreamTransformerDescriptor, readabilityTransformerDescriptor], + descriptor => descriptor.type + ), llmProviders: createDescriptorRecord([openAiChatProviderDescriptor], descriptor => descriptor.type), outputRenderers: createDescriptorRecord( [debugXmlRendererDescriptor, passthroughRendererDescriptor], @@ -48,6 +57,7 @@ export const DEFAULT_ENGINE_DESCRIPTOR_BUNDLE: EngineDescriptorBundle = Object.f captureUrlsStepDescriptor, loadSourceStepDescriptor, llmPassStepDescriptor, + transformStepDescriptor, truncateStepDescriptor, verifyUrlsStepDescriptor ], diff --git a/src/config/yaml/yaml-app-config.ts b/src/config/yaml/yaml-app-config.ts index 921dc98..60d0253 100644 --- a/src/config/yaml/yaml-app-config.ts +++ b/src/config/yaml/yaml-app-config.ts @@ -49,6 +49,7 @@ export function yamlToAppConfig(yamlConfig: RawYamlConfig): AppConfig { return { engineConfig: { sourceProviders: yamlConfig.sourceProviders, + contentTransformers: yamlConfig.contentTransformers, llmProviders: yamlConfig.llmProviders, outputRenderers: yamlConfig.outputRenderers, pipelines diff --git a/src/config/yaml/yaml-config.ts b/src/config/yaml/yaml-config.ts index 4982c99..15cddfd 100644 --- a/src/config/yaml/yaml-config.ts +++ b/src/config/yaml/yaml-config.ts @@ -40,6 +40,7 @@ export const rawYamlConfigSchema = z .object({ schemaVersion: z.literal(1).default(1), sourceProviders: z.record(providerEntrySchema).default({}), + contentTransformers: z.record(providerEntrySchema).default({}), llmProviders: z.record(providerEntrySchema).default({}), outputRenderers: z.record(providerEntrySchema).default({}), pipelines: z.record( diff --git a/src/contracts/extensions/content-transformer.ts b/src/contracts/extensions/content-transformer.ts new file mode 100644 index 0000000..1d46804 --- /dev/null +++ b/src/contracts/extensions/content-transformer.ts @@ -0,0 +1,89 @@ +/** + * Defines the content transformer extension contract and registry service key. + * + * Content transformers rewrite one pipeline body into another representation, + * such as HTML to Markdown today and Markdown cleanup later. A transformer + * instance may handle several source/target combinations; `supports` is its + * capability matrix and `transform` performs the conversion. The requesting + * step owns routing intent (the target media type) and asserts the result. + */ + +import { createExtensionServiceKey } from "../host/extension-services.js"; + +import type { Logger } from "../../shared/logger.js"; +import type { HostTools } from "../host/host-tools.js"; +import type { BodyContent } from "../pipeline/context.js"; +import type { NamedRegistry } from "./named-registry.js"; +import type { ResolvedContentTransformer } from "./resolved-extension.js"; + +/** Describes the routing intent a step passes to a content transformer. */ +export interface ContentTransformRequest { + readonly targetMediaType: string; +} + +/** Carries observability-only detail about one transform outcome. */ +export interface ContentTransformDiagnostic { + readonly code: string; + readonly message?: string; +} + +/** Carries the transformed body and optional diagnostics. */ +export interface ContentTransformedResult { + readonly outcome: "transformed"; + readonly body: BodyContent; + readonly diagnostics?: ReadonlyArray; +} + +/** Indicates the transformer chose not to transform this input (e.g. not suitable). */ +export interface ContentDeclinedResult { + readonly outcome: "declined"; + readonly reason?: string; +} + +/** A content transformer may transform the body or decline. */ +export type ContentTransformResult = ContentTransformedResult | ContentDeclinedResult; + +/** Transforms one pipeline body representation into another. */ +export interface ContentTransformer { + /** Reports whether this instance can satisfy the requested transform. */ + supports(input: { + readonly sourceKind: BodyContent["kind"]; + readonly sourceMediaType: string; + readonly request: ContentTransformRequest; + }): boolean; + + /** Transforms the body toward the requested target representation. */ + transform( + input: { readonly url: string; readonly body: BodyContent; readonly request: ContentTransformRequest }, + opts: { readonly signal: AbortSignal } + ): Promise; +} + +/** Resolves configured content transformers by name. */ +export type ContentTransformerRegistry = NamedRegistry; + +/** Provides dependencies available while constructing a content transformer. */ +export interface ContentTransformerCreateDeps { + readonly logger: Logger; + readonly tools: HostTools; +} + +/** Provides arguments used to construct one content transformer instance. */ +export interface ContentTransformerCreateArgs { + readonly name: string; + readonly config: TConfig; + readonly deps: ContentTransformerCreateDeps; +} + +/** Defines one content transformer implementation type addressable from YAML. */ +export interface ContentTransformerDescriptor { + readonly type: string; + parseConfig(raw: unknown): TConfig; + create(args: ContentTransformerCreateArgs): ContentTransformer; +} + +/** Identifies the content transformer registry extension service. */ +export const contentTransformerRegistryKey = createExtensionServiceKey({ + id: "llmc.contentTransformerRegistry", + description: "content transformer registry" +}); diff --git a/src/contracts/extensions/resolved-extension.ts b/src/contracts/extensions/resolved-extension.ts index 6760b42..a362e20 100644 --- a/src/contracts/extensions/resolved-extension.ts +++ b/src/contracts/extensions/resolved-extension.ts @@ -7,6 +7,7 @@ * identities without coupling implementations to app configuration. */ +import type { ContentTransformer } from "./content-transformer.js"; import type { LlmProvider } from "./llm-provider.js"; import type { OutputRenderer } from "./output-renderer.js"; import type { SourceProvider } from "./source-provider.js"; @@ -18,6 +19,13 @@ export interface ResolvedSourceProvider { readonly provider: SourceProvider; } +/** Pairs a configured content transformer instance with its identity. */ +export interface ResolvedContentTransformer { + readonly name: string; + readonly type: string; + readonly transformer: ContentTransformer; +} + /** Pairs a configured LLM provider instance with its identity. */ export interface ResolvedLlmProvider { readonly name: string; diff --git a/src/engine/create-engine.ts b/src/engine/create-engine.ts index 35622f9..b89d924 100644 --- a/src/engine/create-engine.ts +++ b/src/engine/create-engine.ts @@ -7,12 +7,14 @@ * bundles, process bootstrap code, or concrete built-ins. */ +import { contentTransformerRegistryKey } from "../contracts/extensions/content-transformer.js"; import { llmProviderRegistryKey } from "../contracts/extensions/llm-provider.js"; import { outputRendererRegistryKey } from "../contracts/extensions/output-renderer.js"; import { sourceProviderRegistryKey } from "../contracts/extensions/source-provider.js"; import { PipelineOrchestrator } from "../core/pipeline/orchestrator.js"; import { PipelineRunner } from "../core/pipeline/runner.js"; import { createEngineRuntime } from "./engine-runtime.js"; +import { buildContentTransformers } from "./internal/builders/build-content-transformers.js"; import { buildLlmProviders } from "./internal/builders/build-llm-providers.js"; import { buildOutputRenderers } from "./internal/builders/build-output-renderers.js"; import { buildPipelines } from "./internal/builders/build-pipelines.js"; @@ -40,6 +42,13 @@ export async function createEngine(args: { args.descriptors.sourceProviders ); serviceBuilder.register(sourceProviderRegistryKey, sourceProviders); + const contentTransformers = buildContentTransformers( + args.config.contentTransformers, + args.tools, + args.logger, + args.descriptors.contentTransformers + ); + serviceBuilder.register(contentTransformerRegistryKey, contentTransformers); const llmProviders = buildLlmProviders( args.config.llmProviders, args.tools, @@ -65,6 +74,12 @@ export async function createEngine(args: { ); logSkipped(args.logger, "source provider", Object.keys(args.config.sourceProviders), sourceProviders.builtNames()); + logSkipped( + args.logger, + "content transformer", + Object.keys(args.config.contentTransformers), + contentTransformers.builtNames() + ); logSkipped(args.logger, "LLM provider", Object.keys(args.config.llmProviders), llmProviders.builtNames()); logSkipped(args.logger, "output renderer", Object.keys(args.config.outputRenderers), outputRenderers.builtNames()); @@ -72,7 +87,7 @@ export async function createEngine(args: { new PipelineOrchestrator({ logger: args.logger.child({ component: "orchestrator" }) }) ); const handles = new Map([...compiledPipelines].map(([name, pipeline]) => [name, runner.bindTo(pipeline)])); - const registries: EngineRegistries = { sourceProviders, llmProviders, outputRenderers }; + const registries: EngineRegistries = { sourceProviders, contentTransformers, llmProviders, outputRenderers }; return createEngineRuntime({ registries, pipelines: pipelineInfos(args.config), handles }); } diff --git a/src/engine/engine-config.ts b/src/engine/engine-config.ts index 0535506..afacd8b 100644 --- a/src/engine/engine-config.ts +++ b/src/engine/engine-config.ts @@ -32,6 +32,7 @@ export interface EnginePipelineConfig { /** Engine-only config. Excludes schema metadata and adapter configuration. */ export interface EngineConfig { readonly sourceProviders: Readonly>; + readonly contentTransformers: Readonly>; readonly llmProviders: Readonly>; readonly outputRenderers: Readonly>; readonly pipelines: Readonly>; diff --git a/src/engine/engine-descriptors.ts b/src/engine/engine-descriptors.ts index 842579b..05f078a 100644 --- a/src/engine/engine-descriptors.ts +++ b/src/engine/engine-descriptors.ts @@ -5,6 +5,7 @@ * itself. Bundle selection belongs to app assembly or another embedding host. */ +import type { ContentTransformerDescriptor } from "../contracts/extensions/content-transformer.js"; import type { LlmProviderDescriptor } from "../contracts/extensions/llm-provider.js"; import type { OutputRendererDescriptor } from "../contracts/extensions/output-renderer.js"; import type { SourceProviderDescriptor } from "../contracts/extensions/source-provider.js"; @@ -13,6 +14,7 @@ import type { PipelineStepDescriptor } from "../contracts/pipeline/step.js"; /** Descriptor records needed to construct an engine runtime. */ export interface EngineDescriptors { readonly sourceProviders: Readonly>; + readonly contentTransformers: Readonly>; readonly llmProviders: Readonly>; readonly outputRenderers: Readonly>; readonly pipelineSteps: Readonly>; diff --git a/src/engine/engine-runtime.ts b/src/engine/engine-runtime.ts index 7c7ae2c..36936dd 100644 --- a/src/engine/engine-runtime.ts +++ b/src/engine/engine-runtime.ts @@ -8,6 +8,7 @@ import { ConfigurationError } from "../shared/errors.js"; +import type { ContentTransformerRegistry } from "../contracts/extensions/content-transformer.js"; import type { LlmProviderRegistry } from "../contracts/extensions/llm-provider.js"; import type { OutputRendererRegistry } from "../contracts/extensions/output-renderer.js"; import type { SourceProviderRegistry } from "../contracts/extensions/source-provider.js"; @@ -17,6 +18,7 @@ import type { PipelineHandle, PipelineRunOutput } from "../contracts/pipeline/ha /** Registries of engine-owned configured runtime instances. */ export interface EngineRegistries { readonly sourceProviders: SourceProviderRegistry; + readonly contentTransformers: ContentTransformerRegistry; readonly llmProviders: LlmProviderRegistry; readonly outputRenderers: OutputRendererRegistry; } diff --git a/src/engine/internal/builders/build-content-transformers.ts b/src/engine/internal/builders/build-content-transformers.ts new file mode 100644 index 0000000..d83e9c9 --- /dev/null +++ b/src/engine/internal/builders/build-content-transformers.ts @@ -0,0 +1,44 @@ +/** + * Constructs configured content transformers for the engine registry. + * + * The generic registry builder handles descriptor lookup, config parsing, and + * identity wrapping. This file supplies content-transformer-specific error codes + * and logger identity fields. + */ + +import { buildNamedRegistry, type BuiltNameTracking } from "./build-named-registry.js"; + +import type { + ContentTransformerDescriptor, + ContentTransformerRegistry +} from "../../../contracts/extensions/content-transformer.js"; +import type { HostTools } from "../../../contracts/host/host-tools.js"; +import type { Logger } from "../../../shared/logger.js"; +import type { EngineConfig } from "../../engine-config.js"; + +/** Constructs content transformers from configured content transformer entries. */ +export function buildContentTransformers( + rawTransformers: EngineConfig["contentTransformers"], + tools: HostTools, + logger: Logger, + descriptors: Readonly> +): ContentTransformerRegistry & BuiltNameTracking { + return buildNamedRegistry({ + rawEntries: rawTransformers, + descriptors, + tools, + logger, + labels: { + logField: "content_transformer", + unknownTypeCode: "unknown_content_transformer_type", + unknownTypeMessage: type => `Unknown content transformer type: ${type}`, + invalidConfigCode: "invalid_content_transformer_config", + invalidConfigMessage: (name, type) => `Invalid config for content transformer '${name}' of type '${type}'`, + createFailedCode: "content_transformer_create_failed", + createFailedMessage: (name, type) => `Failed to create content transformer '${name}' of type '${type}'`, + missingCode: "unknown_content_transformer", + missingMessage: name => `Unknown content transformer: ${name}` + }, + resolve: ({ name, type }, transformer) => ({ name, type, transformer }) + }); +} diff --git a/src/shared/media-types.ts b/src/shared/media-types.ts index b397a54..8376825 100644 --- a/src/shared/media-types.ts +++ b/src/shared/media-types.ts @@ -27,3 +27,14 @@ export function isTextLike(mediaType: string): boolean { normalized.endsWith("+xml") ); } + +/** Returns whether a media type is HTML or XHTML, ignoring parameters and case. */ +export function isHtmlMediaType(mediaType: string): boolean { + const essence = mediaType.split(";", 1)[0].trim().toLowerCase(); + return essence === mediaTypes.html || essence === mediaTypes.xhtml; +} + +/** Returns whether trimmed text starts with an HTML tag or XML declaration. */ +export function looksLikeHtml(text: string): boolean { + return /^\s* { httpFetch: async () => new Response("{}") }); - expect(loaded.registries.sourceProviders.require("default-firecrawl").name).toBe("default-firecrawl"); - expect(loaded.registries.llmProviders.require("default-llm").name).toBe("default-llm"); + expect(loaded.registries.sourceProviders.require("firecrawl-markdown").name).toBe("firecrawl-markdown"); + expect(loaded.registries.llmProviders.require("llm-default").name).toBe("llm-default"); expect( loaded.registries.pipelines.find(pipeline => pipeline.name === "default")?.steps.map(entry => entry.name) ).toEqual(["fetch", "clean", "truncate"]); @@ -45,7 +45,7 @@ describe("composeApp", () => { logger: createTestLogger(), httpFetch: fetch }) - ).rejects.toThrow("Invalid config for source provider 'default-firecrawl'"); + ).rejects.toThrow("Invalid config for source provider 'firecrawl-markdown'"); const missingTemplate = await writeConfigFixture(defaultYaml(), false); await expect( @@ -110,7 +110,7 @@ describe("composeApp", () => { }); await loaded.registries.llmProviders - .require("default-llm") + .require("llm-default") .provider.chat([], { signal: new AbortController().signal }); expect(requestBody?.model).toBe("${LLM_MODEL}"); @@ -123,8 +123,8 @@ describe("composeApp", () => { .replace("stripBase64Images: true", "stripBase64Images: ${FIRECRAWL_STRIP_IMAGES:-}") .replace("includeSkipped: true", "includeSkipped: ${DEBUG_XML_INCLUDE_SKIPPED:-}") .replace( - " config:\n provider: default-llm", - " timeoutSeconds: ${CLEAN_TIMEOUT_SECONDS:-}\n config:\n provider: default-llm" + " config:\n provider: llm-default", + " timeoutSeconds: ${CLEAN_TIMEOUT_SECONDS:-}\n config:\n provider: llm-default" ) .replace("minInputChars: 1", "minInputChars: ${MIN_INPUT_CHARS:-}") .replace("maxInputChars: 100", "maxInputChars: ${MAX_INPUT_CHARS:-}") @@ -159,7 +159,7 @@ describe("composeApp", () => { ).toBe(60); await loaded.registries.sourceProviders - .require("default-firecrawl") + .require("firecrawl-markdown") .provider.load("https://example.com", { signal: new AbortController().signal }); expect(firecrawlRequestBody).toMatchObject({ onlyMainContent: false, removeBase64Images: true }); @@ -243,7 +243,7 @@ function envValues(): NodeJS.ProcessEnv { function defaultYaml(): string { return `sourceProviders: - default-firecrawl: + firecrawl-markdown: type: firecrawl config: baseUrl: \${FIRECRAWL_BASE_URL} @@ -253,7 +253,7 @@ function defaultYaml(): string { stripBase64Images: true parsePdf: true llmProviders: - default-llm: + llm-default: type: openai-chat config: baseUrl: \${LLM_BASE_URL} @@ -277,12 +277,12 @@ pipelines: name: fetch concurrencyGroup: source config: - provider: default-firecrawl + provider: firecrawl-markdown - type: llm-pass name: clean concurrencyGroup: llm config: - provider: default-llm + provider: llm-default minInputChars: 1 maxInputChars: 100 templates: diff --git a/tests/architecture/import-boundaries.test.ts b/tests/architecture/import-boundaries.test.ts index 186141f..e822a35 100644 --- a/tests/architecture/import-boundaries.test.ts +++ b/tests/architecture/import-boundaries.test.ts @@ -24,6 +24,7 @@ interface ClassifiedSource { type SourceLayer = | "adapter-http" | "bundles" + | "builtins-content-transformer" | "builtins-http-adapter" | "builtins-output-renderer" | "builtins-pipeline-step" @@ -195,6 +196,8 @@ function classifySource(sourcePath: string): ClassifiedSource { if (sourcePath.startsWith("src/adapters/http/")) return { layer: "adapter-http" }; if (sourcePath.startsWith("src/builtins/source-providers/")) return { layer: "builtins-provider", component: providerComponent(sourcePath, "source") }; + if (sourcePath.startsWith("src/builtins/content-transformers/")) + return { layer: "builtins-content-transformer", component: contentTransformerComponent(sourcePath) }; if (sourcePath.startsWith("src/builtins/llm-providers/")) return { layer: "builtins-provider", component: providerComponent(sourcePath, "llm") }; if (sourcePath.startsWith("src/builtins/pipeline-steps/")) @@ -211,6 +214,7 @@ function isAllowedImport(edge: ImportEdge, source: ClassifiedSource, target: Cla switch (source.layer) { case "bundles": return [ + "builtins-content-transformer", "builtins-output-renderer", "builtins-pipeline-step", "builtins-provider", @@ -238,6 +242,8 @@ function isAllowedImport(edge: ImportEdge, source: ClassifiedSource, target: Cla ); case "builtins-provider": return ["contracts", "shared"].includes(target.layer) || isSameComponent(source, target); + case "builtins-content-transformer": + return ["contracts", "shared"].includes(target.layer) || isSameComponent(source, target); case "builtins-pipeline-step": return ["contracts", "shared"].includes(target.layer) || isSameComponent(source, target); case "builtins-output-renderer": @@ -282,9 +288,13 @@ function groupConcreteBuiltinImportsBySource(): Map> { } function isConcreteBuiltinLayer(layer: SourceLayer): boolean { - return ["builtins-http-adapter", "builtins-output-renderer", "builtins-pipeline-step", "builtins-provider"].includes( - layer - ); + return [ + "builtins-content-transformer", + "builtins-http-adapter", + "builtins-output-renderer", + "builtins-pipeline-step", + "builtins-provider" + ].includes(layer); } function isDescriptorBundleSource(sourcePath: string): boolean { @@ -310,6 +320,11 @@ function pipelineStepComponent(sourcePath: string): string | undefined { return segments.length > 4 ? segments[3] : undefined; } +function contentTransformerComponent(sourcePath: string): string | undefined { + const segments = sourcePath.split("/"); + return segments.length > 4 ? segments[3] : undefined; +} + function outputRendererComponent(sourcePath: string): string | undefined { const segments = sourcePath.split("/"); return segments.length > 4 ? segments[3] : undefined; diff --git a/tests/builtins/content-transformers/mdream/mdream-transformer-config.test.ts b/tests/builtins/content-transformers/mdream/mdream-transformer-config.test.ts new file mode 100644 index 0000000..8485060 --- /dev/null +++ b/tests/builtins/content-transformers/mdream/mdream-transformer-config.test.ts @@ -0,0 +1,18 @@ +/** Verifies the mdream content transformer config parsing. */ +import { describe, expect, it } from "vitest"; + +import { parseMdreamTransformerConfig } from "../../../../src/builtins/content-transformers/mdream/mdream-transformer-config.js"; + +describe("parseMdreamTransformerConfig", () => { + it("applies defaults", () => { + expect(parseMdreamTransformerConfig({})).toEqual({ minimal: false, clean: true }); + }); + + it("coerces env-substituted boolean strings", () => { + expect(parseMdreamTransformerConfig({ minimal: "true", clean: "false" })).toEqual({ minimal: true, clean: false }); + }); + + it("rejects unknown keys", () => { + expect(() => parseMdreamTransformerConfig({ unexpected: true })).toThrow(); + }); +}); diff --git a/tests/builtins/content-transformers/mdream/mdream-transformer.test.ts b/tests/builtins/content-transformers/mdream/mdream-transformer.test.ts new file mode 100644 index 0000000..4d1557b --- /dev/null +++ b/tests/builtins/content-transformers/mdream/mdream-transformer.test.ts @@ -0,0 +1,130 @@ +/** Verifies the mdream content transformer support matrix and conversion. */ +import { describe, expect, it } from "vitest"; + +import { MdreamTransformer } from "../../../../src/builtins/content-transformers/mdream/mdream-transformer.js"; +import { InternalError } from "../../../../src/shared/errors.js"; +import { mediaTypes } from "../../../../src/shared/media-types.js"; +import { createTestLogger } from "../../../helpers/logger.js"; + +import type { BodyContent } from "../../../../src/contracts/pipeline/context.js"; + +function makeTransformer(config: { minimal?: boolean; clean?: boolean } = {}) { + return new MdreamTransformer( + { minimal: config.minimal ?? false, clean: config.clean ?? true }, + { + logger: createTestLogger() + } + ); +} + +const markdownRequest = { targetMediaType: mediaTypes.markdown }; + +describe("MdreamTransformer.supports", () => { + it("supports HTML and XHTML text bodies targeting markdown", () => { + const transformer = makeTransformer(); + expect(transformer.supports({ sourceKind: "text", sourceMediaType: "text/html", request: markdownRequest })).toBe( + true + ); + expect( + transformer.supports({ sourceKind: "text", sourceMediaType: "application/xhtml+xml", request: markdownRequest }) + ).toBe(true); + expect( + transformer.supports({ + sourceKind: "text", + sourceMediaType: "text/html; charset=utf-8", + request: markdownRequest + }) + ).toBe(true); + }); + + it("rejects non-HTML sources, non-text kinds, and non-markdown targets", () => { + const transformer = makeTransformer(); + expect( + transformer.supports({ sourceKind: "text", sourceMediaType: "text/markdown", request: markdownRequest }) + ).toBe(false); + expect(transformer.supports({ sourceKind: "binary", sourceMediaType: "text/html", request: markdownRequest })).toBe( + false + ); + expect( + transformer.supports({ + sourceKind: "text", + sourceMediaType: "text/html", + request: { targetMediaType: "text/html" } + }) + ).toBe(false); + }); +}); + +describe("MdreamTransformer.transform", () => { + const signal = new AbortController().signal; + + it("converts HTML to markdown, resolves relative links via origin, and preserves the title", async () => { + const transformer = makeTransformer(); + const body: BodyContent = { + kind: "text", + mediaType: "text/html", + content: '

Hello

World link

', + title: "Doc" + }; + + const result = await transformer.transform( + { url: "https://example.com", body, request: markdownRequest }, + { signal } + ); + + expect(result.outcome).toBe("transformed"); + if (result.outcome !== "transformed") throw new Error("Expected transformed outcome"); + expect(result.body.kind).toBe("text"); + if (result.body.kind !== "text") throw new Error("Expected text body"); + expect(result.body.mediaType).toBe(mediaTypes.markdown); + expect(result.body.title).toBe("Doc"); + expect(result.body.content).toContain("# Hello"); + expect(result.body.content).toContain("](https://example.com"); + }); + + it("emits a summary diagnostic and flags empty output", async () => { + const transformer = makeTransformer(); + const empty = await transformer.transform( + { + url: "https://example.com/", + body: { kind: "text", mediaType: "text/html", content: "
" }, + request: markdownRequest + }, + { signal } + ); + expect(empty.outcome).toBe("transformed"); + if (empty.outcome !== "transformed") throw new Error("Expected transformed outcome"); + expect(empty.diagnostics?.some(diagnostic => diagnostic.code === "empty_output")).toBe(true); + expect(empty.diagnostics?.some(diagnostic => diagnostic.code === "mdream")).toBe(true); + }); + + it("throws when given a non-text body", async () => { + const transformer = makeTransformer(); + await expect( + transformer.transform( + { + url: "https://example.com/", + body: { kind: "binary", mediaType: "application/pdf", bytes: new Uint8Array([1]) }, + request: markdownRequest + }, + { signal } + ) + ).rejects.toBeInstanceOf(InternalError); + }); + + it("throws if the signal is already aborted", async () => { + const transformer = makeTransformer(); + const controller = new AbortController(); + controller.abort(); + await expect( + transformer.transform( + { + url: "https://example.com/", + body: { kind: "text", mediaType: "text/html", content: "

hi

" }, + request: markdownRequest + }, + { signal: controller.signal } + ) + ).rejects.toBeDefined(); + }); +}); diff --git a/tests/builtins/content-transformers/readability/readability-transformer-config.test.ts b/tests/builtins/content-transformers/readability/readability-transformer-config.test.ts new file mode 100644 index 0000000..3f36025 --- /dev/null +++ b/tests/builtins/content-transformers/readability/readability-transformer-config.test.ts @@ -0,0 +1,50 @@ +/** Verifies the readability content transformer config parsing. */ +import { describe, expect, it } from "vitest"; + +import { parseReadabilityTransformerConfig } from "../../../../src/builtins/content-transformers/readability/readability-transformer-config.js"; + +describe("parseReadabilityTransformerConfig", () => { + it("applies defaults for all fields", () => { + expect(parseReadabilityTransformerConfig({})).toEqual({ + minContentLength: 140, + minScore: 20, + maxElements: 0 + }); + }); + + it("parses explicit numeric values", () => { + expect( + parseReadabilityTransformerConfig({ + minContentLength: "100", + minScore: "15", + maxElements: "50" + }) + ).toEqual({ + minContentLength: 100, + minScore: 15, + maxElements: 50 + }); + }); + + it("treats blank strings as undefined (env placeholder fallback)", () => { + expect( + parseReadabilityTransformerConfig({ + minContentLength: "", + minScore: "", + maxElements: "" + }) + ).toEqual({ + minContentLength: 140, + minScore: 20, + maxElements: 0 + }); + }); + + it("rejects negative minContentLength", () => { + expect(() => parseReadabilityTransformerConfig({ minContentLength: "-1" })).toThrow(); + }); + + it("rejects unknown keys", () => { + expect(() => parseReadabilityTransformerConfig({ extra: 1 })).toThrow(); + }); +}); diff --git a/tests/builtins/content-transformers/readability/readability-transformer.test.ts b/tests/builtins/content-transformers/readability/readability-transformer.test.ts new file mode 100644 index 0000000..625b055 --- /dev/null +++ b/tests/builtins/content-transformers/readability/readability-transformer.test.ts @@ -0,0 +1,172 @@ +/** Verifies the readability content transformer support matrix, decline behavior, and conversion. */ +import { describe, expect, it } from "vitest"; + +import { ReadabilityTransformer } from "../../../../src/builtins/content-transformers/readability/readability-transformer.js"; +import { InternalError } from "../../../../src/shared/errors.js"; +import { mediaTypes } from "../../../../src/shared/media-types.js"; +import { createTestLogger } from "../../../helpers/logger.js"; + +import type { BodyContent } from "../../../../src/contracts/pipeline/context.js"; + +function articleHtml(title?: string): string { + const header = title ? `${title}` : ""; + return `${header}

Article

${"Article content paragraph. ".repeat(60)}

`; +} + +function shortHtml(): string { + return "

Too short

"; +} + +function makeTransformer(config: { minContentLength?: number; minScore?: number; maxElements?: number } = {}) { + return new ReadabilityTransformer( + { + minContentLength: config.minContentLength ?? 140, + minScore: config.minScore ?? 20, + maxElements: config.maxElements ?? 0 + }, + { logger: createTestLogger() } + ); +} + +const htmlRequest = { targetMediaType: mediaTypes.html }; + +describe("ReadabilityTransformer.supports", () => { + it("supports HTML and XHTML text bodies targeting HTML", () => { + const transformer = makeTransformer(); + expect(transformer.supports({ sourceKind: "text", sourceMediaType: "text/html", request: htmlRequest })).toBe(true); + expect( + transformer.supports({ sourceKind: "text", sourceMediaType: "application/xhtml+xml", request: htmlRequest }) + ).toBe(true); + }); + + it("rejects non-HTML sources, non-text kinds, and non-HTML targets", () => { + const transformer = makeTransformer(); + expect(transformer.supports({ sourceKind: "text", sourceMediaType: "text/plain", request: htmlRequest })).toBe( + false + ); + expect(transformer.supports({ sourceKind: "binary", sourceMediaType: "text/html", request: htmlRequest })).toBe( + false + ); + expect( + transformer.supports({ + sourceKind: "text", + sourceMediaType: "text/html", + request: { targetMediaType: "text/markdown" } + }) + ).toBe(false); + }); +}); + +describe("ReadabilityTransformer.transform", () => { + const signal = new AbortController().signal; + + it("extracts article content from readerable HTML", async () => { + const transformer = makeTransformer(); + const body: BodyContent = { + kind: "text", + mediaType: "text/html", + content: articleHtml("Test Page"), + title: "Original Title" + }; + + const result = await transformer.transform({ url: "https://example.com", body, request: htmlRequest }, { signal }); + + expect(result.outcome).toBe("transformed"); + if (result.outcome !== "transformed") return; + expect(result.body.kind).toBe("text"); + if (result.body.kind !== "text") return; + expect(result.body.mediaType).toBe(mediaTypes.html); + expect(result.body.content).toContain("Article content"); + expect(result.body.title).toBe("Original Title"); + }); + + it("falls back to the article title when the incoming body has no title", async () => { + const transformer = makeTransformer(); + const body: BodyContent = { + kind: "text", + mediaType: "text/html", + content: articleHtml("Page Title") + }; + + const result = await transformer.transform({ url: "https://example.com", body, request: htmlRequest }, { signal }); + + expect(result.outcome).toBe("transformed"); + if (result.outcome !== "transformed") return; + expect(result.body.title).toBe("Page Title"); + }); + + it("adds a readability diagnostic with char counts", async () => { + const transformer = makeTransformer(); + const body: BodyContent = { + kind: "text", + mediaType: "text/html", + content: articleHtml() + }; + + const result = await transformer.transform({ url: "https://example.com", body, request: htmlRequest }, { signal }); + + expect(result.outcome).toBe("transformed"); + if (result.outcome !== "transformed") return; + expect(result.diagnostics?.some(d => d.code === "readability")).toBe(true); + }); + + it("declines not_readerable for short boilerplate HTML", async () => { + const transformer = makeTransformer(); + const body: BodyContent = { + kind: "text", + mediaType: "text/html", + content: shortHtml() + }; + + const result = await transformer.transform({ url: "https://example.com", body, request: htmlRequest }, { signal }); + + expect(result.outcome).toBe("declined"); + if (result.outcome !== "declined") return; + expect(result.reason).toBe("not_readerable"); + }); + + it("declines parse_empty for a document with no document element", async () => { + const transformer = makeTransformer(); + const body: BodyContent = { + kind: "text", + mediaType: "text/html", + content: "" + }; + + const result = await transformer.transform({ url: "https://example.com", body, request: htmlRequest }, { signal }); + + expect(result.outcome).toBe("declined"); + if (result.outcome !== "declined") return; + expect(result.reason).toBe("parse_empty"); + }); + + it("throws when given a non-text body", async () => { + const transformer = makeTransformer(); + await expect( + transformer.transform( + { + url: "https://example.com/", + body: { kind: "binary", mediaType: "application/pdf", bytes: new Uint8Array([1]) }, + request: htmlRequest + }, + { signal } + ) + ).rejects.toBeInstanceOf(InternalError); + }); + + it("throws if the signal is already aborted", async () => { + const transformer = makeTransformer(); + const controller = new AbortController(); + controller.abort(); + await expect( + transformer.transform( + { + url: "https://example.com/", + body: { kind: "text", mediaType: "text/html", content: articleHtml() }, + request: htmlRequest + }, + { signal: controller.signal } + ) + ).rejects.toThrow(); + }); +}); diff --git a/tests/builtins/pipeline-steps/llm-pass/llm-pass-step-config.test.ts b/tests/builtins/pipeline-steps/llm-pass/llm-pass-step-config.test.ts index 2f45a6c..c96384a 100644 --- a/tests/builtins/pipeline-steps/llm-pass/llm-pass-step-config.test.ts +++ b/tests/builtins/pipeline-steps/llm-pass/llm-pass-step-config.test.ts @@ -4,7 +4,7 @@ import { describe, expect, it } from "vitest"; import { parseLlmPassStepConfig } from "../../../../src/builtins/pipeline-steps/llm-pass/llm-pass-step-config.js"; const baseConfig = { - provider: "default-llm", + provider: "llm-default", templates: { system: "system", user: "user" } }; diff --git a/tests/builtins/pipeline-steps/transform/transform-step-config.test.ts b/tests/builtins/pipeline-steps/transform/transform-step-config.test.ts new file mode 100644 index 0000000..847a3eb --- /dev/null +++ b/tests/builtins/pipeline-steps/transform/transform-step-config.test.ts @@ -0,0 +1,46 @@ +/** Verifies the transform pipeline step config parsing. */ +import { describe, expect, it } from "vitest"; + +import { parseTransformStepConfig } from "../../../../src/builtins/pipeline-steps/transform/transform-step-config.js"; + +describe("parseTransformStepConfig", () => { + it("applies defaults for optional fields", () => { + expect(parseTransformStepConfig({ transformer: "mdream-convert", target: "text/markdown" })).toEqual({ + transformer: "mdream-convert", + target: "text/markdown", + onUnsupported: "skip", + onDeclined: "skip", + emitDiagnostics: false + }); + }); + + it("requires transformer and target", () => { + expect(() => parseTransformStepConfig({ target: "text/markdown" })).toThrow(); + expect(() => parseTransformStepConfig({ transformer: "x" })).toThrow(); + }); + + it("validates the onUnsupported and onDeclined enums and coerces emitDiagnostics", () => { + expect(() => + parseTransformStepConfig({ transformer: "x", target: "text/markdown", onUnsupported: "explode" }) + ).toThrow(); + expect(() => + parseTransformStepConfig({ transformer: "x", target: "text/markdown", onDeclined: "explode" }) + ).toThrow(); + expect( + parseTransformStepConfig({ transformer: "x", target: "text/markdown", emitDiagnostics: "true" }).emitDiagnostics + ).toBe(true); + }); + + it("parses explicit onDeclined values", () => { + expect(parseTransformStepConfig({ transformer: "x", target: "text/markdown", onDeclined: "fail" }).onDeclined).toBe( + "fail" + ); + expect(parseTransformStepConfig({ transformer: "x", target: "text/markdown", onDeclined: "skip" }).onDeclined).toBe( + "skip" + ); + }); + + it("rejects unknown keys", () => { + expect(() => parseTransformStepConfig({ transformer: "x", target: "text/markdown", extra: 1 })).toThrow(); + }); +}); diff --git a/tests/builtins/pipeline-steps/transform/transform-step.test.ts b/tests/builtins/pipeline-steps/transform/transform-step.test.ts new file mode 100644 index 0000000..fa7eea2 --- /dev/null +++ b/tests/builtins/pipeline-steps/transform/transform-step.test.ts @@ -0,0 +1,216 @@ +/** Verifies the transform pipeline step routing, honesty gate, and diagnostics. */ +import { describe, expect, it } from "vitest"; + +import { TransformStep } from "../../../../src/builtins/pipeline-steps/transform/transform-step.js"; +import { createTestLogger } from "../../../helpers/logger.js"; +import { makeStepContext } from "../utils.js"; + +import type { + ContentTransformer, + ContentTransformResult +} from "../../../../src/contracts/extensions/content-transformer.js"; +import type { TransformStepOptions } from "../../../../src/builtins/pipeline-steps/transform/transform-step-config.js"; + +const baseConfig: TransformStepOptions = { + target: "text/markdown", + onUnsupported: "skip", + onDeclined: "skip", + emitDiagnostics: false +}; +const markdownResult: ContentTransformResult = { + outcome: "transformed", + body: { kind: "text", mediaType: "text/markdown", content: "# md" } +}; + +function declinedResult(reason?: string): ContentTransformResult { + return { outcome: "declined", reason }; +} + +function fakeTransformer(overrides: Partial = {}): ContentTransformer { + return { + supports: () => true, + transform: async () => markdownResult, + ...overrides + }; +} + +function makeStep(config: TransformStepOptions, transformer: ContentTransformer): TransformStep { + return new TransformStep(config, { transformer, logger: createTestLogger() }); +} + +const htmlBody = { content: "

md

", mediaType: "text/html" }; + +describe("TransformStep", () => { + it("skips when there is no body", async () => { + const step = makeStep( + baseConfig, + fakeTransformer({ + supports: () => { + throw new Error("supports must not be called without a body"); + } + }) + ); + + await expect(step.run(makeStepContext())).resolves.toMatchObject({ status: "skipped", reason: "no_body" }); + }); + + it("skips an unsupported body when onUnsupported is skip", async () => { + const step = makeStep(baseConfig, fakeTransformer({ supports: () => false })); + + await expect(step.run(makeStepContext({ body: htmlBody }))).resolves.toMatchObject({ + status: "skipped", + reason: "unsupported" + }); + }); + + it("fails an unsupported body when onUnsupported is fail", async () => { + const step = makeStep({ ...baseConfig, onUnsupported: "fail" }, fakeTransformer({ supports: () => false })); + + await expect(step.run(makeStepContext({ body: htmlBody }))).resolves.toMatchObject({ + status: "failed", + reason: "unsupported" + }); + }); + + it("applies the transformed body on success without diagnostics by default", async () => { + const step = makeStep(baseConfig, fakeTransformer()); + + const result = await step.run(makeStepContext({ body: htmlBody })); + + expect(result.status).toBe("ok"); + expect(result.effects?.body).toEqual({ kind: "text", mediaType: "text/markdown", content: "# md" }); + expect(result.diagnostics).toBeUndefined(); + }); + + it("passes the url and requested target to the transformer", async () => { + let captured: { url: string; request: { targetMediaType: string } } | undefined; + const step = makeStep( + baseConfig, + fakeTransformer({ + transform: async input => { + captured = { url: input.url, request: input.request }; + return markdownResult; + } + }) + ); + + await step.run(makeStepContext({ body: htmlBody, url: "https://example.com/page" })); + + expect(captured).toEqual({ url: "https://example.com/page", request: { targetMediaType: "text/markdown" } }); + }); + + it("surfaces transformer diagnostics as children when emitDiagnostics is true", async () => { + const step = makeStep( + { ...baseConfig, emitDiagnostics: true }, + fakeTransformer({ + transform: async () => ({ + outcome: "transformed", + body: markdownResult.body, + diagnostics: [{ code: "mdream", message: "10 -> 4" }, { code: "empty_output" }] + }) + }) + ); + + const result = await step.run(makeStepContext({ body: htmlBody })); + + expect(result.diagnostics).toEqual({ + children: [ + { name: "mdream", attributes: { message: "10 -> 4" } }, + { name: "empty_output", attributes: undefined } + ] + }); + }); + + it("fails when the transformer returns a different media type", async () => { + const step = makeStep( + baseConfig, + fakeTransformer({ + transform: async () => ({ + outcome: "transformed", + body: { kind: "text", mediaType: "text/plain", content: "x" } + }) + }) + ); + + await expect(step.run(makeStepContext({ body: htmlBody }))).resolves.toMatchObject({ + status: "failed", + reason: "wrong_output_type" + }); + }); + + it("fails when the transformer returns the wrong representation for a text target", async () => { + const step = makeStep( + baseConfig, + fakeTransformer({ + transform: async () => ({ + outcome: "transformed", + body: { kind: "binary", mediaType: "text/markdown", bytes: new Uint8Array([1]) } + }) + }) + ); + + await expect(step.run(makeStepContext({ body: htmlBody }))).resolves.toMatchObject({ + status: "failed", + reason: "wrong_output_type" + }); + }); + + it("maps an abort during transform to a timeout failure", async () => { + const step = makeStep( + baseConfig, + fakeTransformer({ + transform: async () => { + throw new DOMException("Aborted", "AbortError"); + } + }) + ); + + await expect(step.run(makeStepContext({ body: htmlBody }))).resolves.toMatchObject({ + status: "failed", + reason: "timeout" + }); + }); + + it("skips when the transformer declines with onDeclined default skip", async () => { + const step = makeStep(baseConfig, fakeTransformer({ transform: async () => declinedResult("not_readerable") })); + + await expect(step.run(makeStepContext({ body: htmlBody }))).resolves.toMatchObject({ + status: "skipped", + reason: "not_readerable" + }); + }); + + it("fails when the transformer declines with onDeclined fail", async () => { + const step = makeStep( + { ...baseConfig, onDeclined: "fail" }, + fakeTransformer({ transform: async () => declinedResult("not_readerable") }) + ); + + await expect(step.run(makeStepContext({ body: htmlBody }))).resolves.toMatchObject({ + status: "failed", + reason: "not_readerable" + }); + }); + + it("surfaces a default declined reason when the transformer omits the reason", async () => { + const step = makeStep(baseConfig, fakeTransformer({ transform: async () => declinedResult() })); + + await expect(step.run(makeStepContext({ body: htmlBody }))).resolves.toMatchObject({ + status: "skipped", + reason: "declined" + }); + }); + + it("rethrows non-abort transformer errors", async () => { + const step = makeStep( + baseConfig, + fakeTransformer({ + transform: async () => { + throw new Error("boom"); + } + }) + ); + + await expect(step.run(makeStepContext({ body: htmlBody }))).rejects.toThrow("boom"); + }); +}); diff --git a/tests/builtins/source-providers/firecrawl/firecrawl-provider.test.ts b/tests/builtins/source-providers/firecrawl/firecrawl-provider.test.ts index 10734e2..af99e36 100644 --- a/tests/builtins/source-providers/firecrawl/firecrawl-provider.test.ts +++ b/tests/builtins/source-providers/firecrawl/firecrawl-provider.test.ts @@ -69,6 +69,210 @@ describe("FirecrawlProvider", () => { expect(document).toEqual({ kind: "text", content: "Raw", mediaType: "text/html" }); }); + it("labels rawHtml PDF content as text/plain", async () => { + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "rawHtml" }), + { + httpFetch: async () => + jsonResponse({ + success: true, + data: { + rawHtml: "Dummy PDF file", + metadata: { contentType: "application/pdf" } + } + }), + logger: createTestLogger() + } + ); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document).toEqual({ + kind: "text", + content: "Dummy PDF file", + mediaType: "text/plain" + }); + }); + + it("labels rawHtml image viewer as text/html", async () => { + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "rawHtml" }), + { + httpFetch: async () => + jsonResponse({ + success: true, + data: { + rawHtml: + '', + metadata: { contentType: "image/jpeg" } + } + }), + logger: createTestLogger() + } + ); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.mediaType).toBe("text/html"); + }); + + it("labels html PDF wrapper as text/html", async () => { + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "html" }), + { + httpFetch: async () => + jsonResponse({ + success: true, + data: { + html: "Dummy PDF file", + metadata: { contentType: "application/pdf" } + } + }), + logger: createTestLogger() + } + ); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document).toEqual({ + kind: "text", + content: "Dummy PDF file", + mediaType: "text/html" + }); + }); + + it("labels markdown PDF content as text/markdown", async () => { + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "markdown" }), + { + httpFetch: async () => + jsonResponse({ + success: true, + data: { + markdown: "Dummy PDF file", + metadata: { contentType: "application/pdf" } + } + }), + logger: createTestLogger() + } + ); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document).toEqual({ + kind: "text", + content: "Dummy PDF file", + mediaType: "text/markdown" + }); + }); + + it("returns binary body when parsePdf is false for a PDF", async () => { + const base64Payload = Buffer.from("%PDF-1.4 test document").toString("base64"); + const fetchFn: typeof fetch = async (_input, init) => { + return jsonResponse({ + success: true, + data: { markdown: base64Payload, metadata: { contentType: "application/pdf" } } + }); + }; + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "markdown", parsePdf: false }), + { httpFetch: fetchFn, logger: createTestLogger() } + ); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.kind).toBe("binary"); + if (document.kind === "binary") { + expect(document.mediaType).toBe("application/pdf"); + expect(new TextDecoder().decode(document.bytes)).toBe("%PDF-1.4 test document"); + } + }); + + it("strips html wrapper from binary base64 when parsePdf is false and output is html", async () => { + const payload = "JVBERi0xLjQK"; + const fetchFn: typeof fetch = async () => { + return jsonResponse({ + success: true, + data: { html: payload, metadata: { contentType: "application/pdf" } } + }); + }; + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "html", parsePdf: false }), + { httpFetch: fetchFn, logger: createTestLogger() } + ); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.kind).toBe("binary"); + if (document.kind === "binary") { + expect(new TextDecoder().decode(document.bytes)).toBe("%PDF-1.4\n"); + } + }); + + it("returns text body when parsePdf is false for a non-PDF (HTML) source", async () => { + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "markdown", parsePdf: false }), + { + httpFetch: async () => + jsonResponse({ + success: true, + data: { + markdown: "# Hello from HTML", + metadata: { contentType: "text/html; charset=UTF-8" } + } + }), + logger: createTestLogger() + } + ); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document).toEqual({ + kind: "text", + content: "# Hello from HTML", + mediaType: "text/markdown" + }); + }); + + it("returns text body when parsePdf is false for an image source", async () => { + const base64Payload = Buffer.from("RIFF fake").toString("base64"); + const fetchFn: typeof fetch = async () => { + return jsonResponse({ + success: true, + data: { + markdown: `![](https://example.com/img.jpg)`, + metadata: { contentType: "image/png" } + } + }); + }; + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", parsePdf: false }), + { httpFetch: fetchFn, logger: createTestLogger() } + ); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.kind).toBe("text"); + }); + + it("rejects empty base64 when parsePdf is false for PDF", async () => { + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", parsePdf: false }), + { + httpFetch: async () => + jsonResponse({ + success: true, + data: { markdown: "", metadata: { contentType: "application/pdf" } } + }), + logger: createTestLogger() + } + ); + + await expect(provider.load("https://example.com", { signal: new AbortController().signal })).rejects.toMatchObject({ + upstreamCode: "empty" + }); + }); + it("omits parsers when parsePdf is false", async () => { let requestBody: Record | undefined; const fetchFn: typeof fetch = async (_input, init) => { diff --git a/tests/bundles/default-engine-descriptors.test.ts b/tests/bundles/default-engine-descriptors.test.ts index 8614c2f..6584623 100644 --- a/tests/bundles/default-engine-descriptors.test.ts +++ b/tests/bundles/default-engine-descriptors.test.ts @@ -4,7 +4,12 @@ import { describe, expect, it } from "vitest"; import type { LlmProvider, LlmProviderRegistry } from "../../src/contracts/extensions/llm-provider.js"; import type { ResourceLoader } from "../../src/contracts/host/host-tools.js"; import type { SourceProvider, SourceProviderRegistry } from "../../src/contracts/extensions/source-provider.js"; +import type { + ContentTransformer, + ContentTransformerRegistry +} from "../../src/contracts/extensions/content-transformer.js"; import { DEFAULT_ENGINE_DESCRIPTOR_BUNDLE } from "../../src/bundles/default-engine-descriptors.js"; +import { contentTransformerRegistryKey } from "../../src/contracts/extensions/content-transformer.js"; import { llmProviderRegistryKey } from "../../src/contracts/extensions/llm-provider.js"; import { sourceProviderRegistryKey } from "../../src/contracts/extensions/source-provider.js"; import { createExtensionServicesBuilder } from "../../src/engine/internal/extension-services.js"; @@ -16,7 +21,7 @@ describe("default engine descriptor bundle", () => { const firecrawl = DEFAULT_ENGINE_DESCRIPTOR_BUNDLE.sourceProviders.firecrawl; const firecrawlConfig = firecrawl.parseConfig({ baseUrl: "https://firecrawl.example" }); const firecrawlProvider = await firecrawl.create({ - name: "default-firecrawl", + name: "firecrawl-markdown", config: firecrawlConfig, deps: { tools: createTestHostTools({ httpFetch: async () => new Response("{}") }), logger: createTestLogger() } }); @@ -24,7 +29,7 @@ describe("default engine descriptor bundle", () => { const openAi = DEFAULT_ENGINE_DESCRIPTOR_BUNDLE.llmProviders["openai-chat"]; const openAiConfig = openAi.parseConfig({ baseUrl: "https://llm.example/v1", model: "model" }); const openAiProvider = await openAi.create({ - name: "default-llm", + name: "llm-default", config: openAiConfig, deps: { tools: createTestHostTools({ httpFetch: async () => new Response("{}") }), logger: createTestLogger() } }); @@ -33,6 +38,32 @@ describe("default engine descriptor bundle", () => { expect(openAiProvider.chat).toEqual(expect.any(Function)); }); + it("round-trips the mdream content transformer descriptor", async () => { + const mdream = DEFAULT_ENGINE_DESCRIPTOR_BUNDLE.contentTransformers.mdream; + const config = mdream.parseConfig({}); + const transformer = await mdream.create({ + name: "mdream-convert", + config, + deps: { tools: createTestHostTools(), logger: createTestLogger() } + }); + + expect(transformer.supports).toEqual(expect.any(Function)); + expect(transformer.transform).toEqual(expect.any(Function)); + }); + + it("round-trips the readability content transformer descriptor", async () => { + const readability = DEFAULT_ENGINE_DESCRIPTOR_BUNDLE.contentTransformers.readability; + const config = readability.parseConfig({ minContentLength: "200", minScore: "15", maxElements: "0" }); + const transformer = await readability.create({ + name: "readability-default", + config, + deps: { tools: createTestHostTools(), logger: createTestLogger() } + }); + + expect(transformer.supports).toEqual(expect.any(Function)); + expect(transformer.transform).toEqual(expect.any(Function)); + }); + it("round-trips step descriptors through parse and create", async () => { const services = servicesWithProviders(sourceProvider(), llmProvider()); const deps = { @@ -72,10 +103,23 @@ describe("default engine descriptor bundle", () => { services, deps }); + const transformConfig = DEFAULT_ENGINE_DESCRIPTOR_BUNDLE.pipelineSteps.transform.parseConfig({ + transformer: "content-transformer", + target: "text/markdown" + }); + const transformStep = await DEFAULT_ENGINE_DESCRIPTOR_BUNDLE.pipelineSteps.transform.create({ + name: "transform", + type: "transform", + timeoutSeconds: 60, + config: transformConfig, + services, + deps + }); expect(loadSourceStep.run).toEqual(expect.any(Function)); expect(llmStep.run).toEqual(expect.any(Function)); expect(truncateStep.run).toEqual(expect.any(Function)); + expect(transformStep.run).toEqual(expect.any(Function)); }); }); @@ -91,9 +135,34 @@ function servicesWithProviders(sourceProvider: SourceProvider, llmProvider: LlmP const builder = createExtensionServicesBuilder(); builder.register(sourceProviderRegistryKey, providerRegistry("source-provider", sourceProvider)); builder.register(llmProviderRegistryKey, providerRegistry("llm-provider", llmProvider)); + builder.register( + contentTransformerRegistryKey, + contentTransformerRegistry("content-transformer", contentTransformer()) + ); return builder.build(); } +function contentTransformer(): ContentTransformer { + return { + supports: () => true, + transform: async () => ({ + outcome: "transformed", + body: { kind: "text", mediaType: "text/markdown", content: "md" } + }) + }; +} + +function contentTransformerRegistry(expectedName: string, transformer: ContentTransformer): ContentTransformerRegistry { + const wrap = { name: expectedName, type: "test", transformer }; + return { + require: (name: string) => { + if (name !== expectedName) throw new Error(`Unknown transformer: ${name}`); + return wrap; + }, + tryGet: (name: string) => (name === expectedName ? wrap : undefined) + }; +} + function providerRegistry( expectedName: string, provider: TProvider diff --git a/tests/engine/activation-lazy.test.ts b/tests/engine/activation-lazy.test.ts index b393a98..46d3798 100644 --- a/tests/engine/activation-lazy.test.ts +++ b/tests/engine/activation-lazy.test.ts @@ -18,6 +18,7 @@ describe("lazy provider/registry activation", () => { sourceProviders: { "bad-provider": { type: "validated-source", config: { requiredField: "" } } }, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: { type: "markdown", config: {} } }, pipelines: { @@ -41,6 +42,7 @@ describe("lazy provider/registry activation", () => { } } satisfies SourceProviderDescriptor }, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: markdownRendererDescriptor }, pipelineSteps: { @@ -60,6 +62,7 @@ describe("lazy provider/registry activation", () => { sourceProviders: { "bad-provider": { type: "validated-source", config: { requiredField: "" } } }, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: { type: "markdown", config: {} } }, pipelines: { @@ -83,6 +86,7 @@ describe("lazy provider/registry activation", () => { } } satisfies SourceProviderDescriptor }, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: markdownRendererDescriptor }, pipelineSteps: { diff --git a/tests/engine/activation-pipeline.test.ts b/tests/engine/activation-pipeline.test.ts index 84d7848..b8ff127 100644 --- a/tests/engine/activation-pipeline.test.ts +++ b/tests/engine/activation-pipeline.test.ts @@ -30,6 +30,7 @@ describe("pipeline activation", () => { const engine = await createEngine({ config: { sourceProviders: {}, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: { type: "markdown", config: {} } }, pipelines: { @@ -49,6 +50,7 @@ describe("pipeline activation", () => { }, descriptors: { sourceProviders: {}, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: markdownRendererDescriptor }, pipelineSteps: { "static-body": staticBodyStepDescriptor } @@ -75,12 +77,14 @@ describe("pipeline activation", () => { await createEngine({ config: { sourceProviders: { "never-used": { type: "never-built", config: {} } }, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: { type: "markdown", config: {} } }, pipelines: {} }, descriptors: { sourceProviders: { "never-built": sourceProv }, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: markdownRendererDescriptor }, pipelineSteps: {} diff --git a/tests/engine/create-engine.test.ts b/tests/engine/create-engine.test.ts index 807a995..46f8369 100644 --- a/tests/engine/create-engine.test.ts +++ b/tests/engine/create-engine.test.ts @@ -13,6 +13,7 @@ describe("createEngine", () => { const engine = await createEngine({ config: { sourceProviders: {}, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: { type: "markdown", config: {} } }, pipelines: { @@ -26,6 +27,7 @@ describe("createEngine", () => { }, descriptors: { sourceProviders: {}, + contentTransformers: {}, llmProviders: {}, outputRenderers: { markdown: markdownRendererDescriptor }, pipelineSteps: { "static-body": staticBodyStepDescriptor } @@ -54,8 +56,14 @@ describe("createEngine", () => { it("reports unknown pipelines clearly", async () => { const engine = await createEngine({ - config: { sourceProviders: {}, llmProviders: {}, outputRenderers: {}, pipelines: {} }, - descriptors: { sourceProviders: {}, llmProviders: {}, outputRenderers: {}, pipelineSteps: {} }, + config: { sourceProviders: {}, contentTransformers: {}, llmProviders: {}, outputRenderers: {}, pipelines: {} }, + descriptors: { + sourceProviders: {}, + contentTransformers: {}, + llmProviders: {}, + outputRenderers: {}, + pipelineSteps: {} + }, tools: createTestHostTools(), logger: createTestLogger() }); diff --git a/tests/shared/media-types.test.ts b/tests/shared/media-types.test.ts index 432a715..0055440 100644 --- a/tests/shared/media-types.test.ts +++ b/tests/shared/media-types.test.ts @@ -1,7 +1,7 @@ /** Verifies media-type predicates and constants. */ import { describe, expect, it } from "vitest"; -import { isTextLike, mediaTypes } from "../../src/shared/media-types.js"; +import { isTextLike, looksLikeHtml, mediaTypes } from "../../src/shared/media-types.js"; describe("mediaTypes", () => { it("exports expected text media type constants", () => { @@ -48,3 +48,37 @@ describe("isTextLike", () => { expect(isTextLike("")).toBe(false); }); }); + +describe("looksLikeHtml", () => { + it("returns true for text starting with ", () => { + expect(looksLikeHtml("Hello")).toBe(true); + }); + + it("returns true for text starting with { + expect(looksLikeHtml("\n")).toBe(true); + }); + + it("returns true with leading whitespace", () => { + expect(looksLikeHtml(" \n")).toBe(true); + }); + + it("returns false for bare plain text", () => { + expect(looksLikeHtml("Dummy PDF file")).toBe(false); + }); + + it("returns false for markdown image reference", () => { + expect(looksLikeHtml("![](https://example.com/img.jpg)")).toBe(false); + }); + + it("returns false for empty string", () => { + expect(looksLikeHtml("")).toBe(false); + }); + + it("returns true for linkime-type HTML", () => { + expect(looksLikeHtml('')).toBe(true); + }); + + it("returns false for markdown heading", () => { + expect(looksLikeHtml("# Hello world\n\nSome text")).toBe(false); + }); +});