diff --git a/.env.example b/.env.example index 33b9ce3..8931826 100644 --- a/.env.example +++ b/.env.example @@ -22,6 +22,9 @@ LOG_LEVEL=info LOG_PRETTY=auto # --- Pipeline ------------------------------------------------------------ +# Active pipeline selected by HTTP adapters. Default: truncate (load-source + truncate) +DEFAULT_PIPELINE=truncate + # Maximum concurrent source-loading groups. Default: 1 SOURCE_CONCURRENCY=4 @@ -38,20 +41,29 @@ DEBUG_XML_INCLUDE_SKIPPED=false OUTPUT_TARGET_CHARS=35000 # --- Source provider ----------------------------------------------------- -# Firecrawl base URL used by the default source provider. Required, no default. +# Active source provider name used by pipelines (truncate and clean-llm). Default: default-http (native HTTP fetch, no external service needed) +SOURCE_PROVIDER=default-http + +# Firecrawl base URL. Required only when a referenced provider uses it (e.g. default-firecrawl). FIRECRAWL_BASE_URL=https://firecrawl.example # Optional Firecrawl bearer token. Default: empty (no token) FIRECRAWL_API_KEY= +# Docling Serve base URL. Required only when a referenced provider uses it (e.g. default-docling). +DOCLING_BASE_URL=https://docling.example + +# Optional Docling API key (X-Api-Key header). Default: empty (no token) +DOCLING_API_KEY= + # --- LLM provider -------------------------------------------------------- -# OpenAI-compatible Chat Completions /v1 base URL. Required, no default. +# OpenAI-compatible Chat Completions /v1 base URL. Required only when a referenced provider uses it (e.g. default-llm). LLM_BASE_URL=https://openai-compatible.example/v1 # Optional LLM bearer token. Default: empty (no token) LLM_API_KEY= -# Model identifier sent to the chat-completions endpoint. Required, no default. +# Model identifier sent to the chat-completions endpoint. Required only when a referenced provider uses it (e.g. default-llm). LLM_MODEL=model-name # Model context window in tokens. Default: empty (disables the context-fit gate) diff --git a/AGENTS.md b/AGENTS.md index d02298b..38c1297 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,6 +22,7 @@ Read the row that matches your task. Each focused doc is self-contained for its | Run or configure the service, or change the environment surface | [docs/CONFIGURATION.md](docs/CONFIGURATION.md), [README.md](README.md) | | Work on logging, request context, errors, or diagnostics | [docs/agents/logging-and-errors.md](docs/agents/logging-and-errors.md) | | Touch auth, URL validation, upstream parsing, XML diagnostics, or other security paths | [docs/agents/security-boundaries.md](docs/agents/security-boundaries.md) | +| Run smoke tests: start/stop, interpret footers, avoid common traps | [docs/agents/smoke-testing.md](docs/agents/smoke-testing.md) | | Change code style, comments, TypeScript conventions, or tests | [docs/agents/coding-conventions.md](docs/agents/coding-conventions.md), [docs/TESTING.md](docs/TESTING.md) | | Commit, push, release, or change versioning and deployment | [docs/agents/git-workflow.md](docs/agents/git-workflow.md), [docs/RELEASING.md](docs/RELEASING.md) | | Check scope before adding a feature | [docs/ROADMAP.md](docs/ROADMAP.md) | @@ -34,3 +35,5 @@ Read the row that matches your task. Each focused doc is self-contained for its - Do not edit `package.json`'s `version` field by hand. Version bumps use `npm version` only when explicitly requested. - When changing the configuration surface, update the source schema, [config/llm-context-loader.yaml](config/llm-context-loader.yaml), [.env.example](.env.example), Compose files when relevant, and the configuration and customization docs in the same change. - When changing behavior, keep the docs that describe it accurate in the same change. +- No gratuitous comments. Inline comments that narrate a change are prohibited ([coding-conventions](docs/agents/coding-conventions.md)). Before adding a comment, ask: will it still be useful after the transient context is gone? +- Don't over-document trivial changes. A new knob, helper, or internal refactor does not warrant updates to `ARCHITECTURE.md` or `extension-authoring.md`. Update only docs that directly describe the changed surface. diff --git a/README.md b/README.md index 76fcc1b..ebf15cb 100644 --- a/README.md +++ b/README.md @@ -7,18 +7,14 @@ [![license](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) [![node](https://img.shields.io/badge/node-22+-brightgreen.svg)](https://nodejs.org/) -LLM Context Loader is a small HTTP service that turns URLs into markdown suitable for language-model context. The default bundle exposes an Open WebUI external web loader endpoint and a limited Jina Reader-style endpoint, fetches pages through Firecrawl, can run OpenAI-compatible Chat Completions passes, and renders either plain markdown or markdown with an XML diagnostic footer. +LLM Context Loader is a small HTTP service that turns URLs into markdown for LLM context. It exposes Open WebUI and Jina Reader-style HTTP endpoints, fetches pages through built-in or external providers, can run LLM passes, and renders markdown with an optional XML diagnostic footer. It handles single URLs — crawling, search, model serving, storage, and retrieval stay outside this tool. ## Current Shape -- HTTP adapters: Open WebUI `POST /`, Jina-style `GET /r/` and `GET /r?url=`, plus open `GET /health`. -- Source provider: Firecrawl `/v2/scrape` with markdown output. -- LLM provider: lowest-common-denominator OpenAI-compatible `/chat/completions` using `system` and `user` messages. -- Default pipeline: load the page, clean it with an LLM pass, summarize it when it is too long, cap the output size. -- LLM passes are guarded by URL-hallucination detection that rolls back to trusted source content. -- Output renderers: `debug-xml` and `passthrough`, selected per pipeline in YAML. - -It fetches and shapes single URLs; crawling, search, model serving, storage, and retrieval stay outside this tool. +- **Pipelines:** `truncate` (default, no LLM) and `clean-llm` (Firecrawl + LLM), selected via `DEFAULT_PIPELINE`. +- **Source providers:** native HTTP fetch, Firecrawl, Docling. +- **LLM provider:** OpenAI-compatible `/chat/completions`. +- **Output renderers:** `debug-xml` and `passthrough`, selected per pipeline in YAML. ## Quick Start @@ -27,7 +23,6 @@ Requires Node 22+ for local runs. ```bash npm install cp .env.example .env -# edit .env for FIRECRAWL_BASE_URL, LLM_BASE_URL, LLM_MODEL, and keys if needed npm run dev ``` @@ -68,7 +63,7 @@ docker compose -f compose.deploy.yaml up -d The deploy compose file requires `LLMC_IMAGE_TAG`. The example env file uses `latest`, but repeatable deployments should pin it to an immutable release tag. -The provider values in `.env.example` are placeholders. Set `FIRECRAWL_BASE_URL`, `LLM_BASE_URL`, and `LLM_MODEL` before starting the service; the Compose files fail early if they are missing. If Firecrawl or the LLM server runs on the host machine, remember that `localhost` inside a container means the container itself. Use a LAN address or `host.docker.internal` when appropriate. +The out-of-box `truncate` pipeline needs no provider config. Switch to `clean-llm` with `DEFAULT_PIPELINE=clean-llm` and the appropriate provider vars. The Compose files pass all variables through without failing early; the service validates only the active pipeline's providers at startup. If Firecrawl or the LLM server runs on the host, use a LAN address or `host.docker.internal` instead of `localhost`. ## API diff --git a/compose.deploy.yaml b/compose.deploy.yaml index 25bd21a..8ea88b6 100644 --- a/compose.deploy.yaml +++ b/compose.deploy.yaml @@ -10,6 +10,9 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l LOG_LEVEL: "${LOG_LEVEL:-info}" LOG_PRETTY: "${LOG_PRETTY:-auto}" + # Active pipeline selected by HTTP adapters. + DEFAULT_PIPELINE: "${DEFAULT_PIPELINE:-truncate}" + # Pipeline concurrency and renderer selection. SOURCE_CONCURRENCY: "${SOURCE_CONCURRENCY:-1}" LLM_CONCURRENCY: "${LLM_CONCURRENCY:-1}" @@ -18,13 +21,16 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l OUTPUT_TARGET_CHARS: "${OUTPUT_TARGET_CHARS:-25000}" # Source provider. - FIRECRAWL_BASE_URL: "${FIRECRAWL_BASE_URL:?Set FIRECRAWL_BASE_URL in .env or the shell}" + SOURCE_PROVIDER: "${SOURCE_PROVIDER:-default-http}" + FIRECRAWL_BASE_URL: "${FIRECRAWL_BASE_URL:-}" FIRECRAWL_API_KEY: "${FIRECRAWL_API_KEY:-}" + DOCLING_BASE_URL: "${DOCLING_BASE_URL:-}" + DOCLING_API_KEY: "${DOCLING_API_KEY:-}" # LLM provider. - LLM_BASE_URL: "${LLM_BASE_URL:?Set LLM_BASE_URL in .env or the shell}" + LLM_BASE_URL: "${LLM_BASE_URL:-}" LLM_API_KEY: "${LLM_API_KEY:-}" - LLM_MODEL: "${LLM_MODEL:?Set LLM_MODEL in .env or the shell}" + LLM_MODEL: "${LLM_MODEL:-}" LLM_CONTEXT_TOKENS: "${LLM_CONTEXT_TOKENS:-}" LLM_CHARS_PER_TOKEN: "${LLM_CHARS_PER_TOKEN:-3.5}" LLM_SAFETY_MARGIN_TOKENS: "${LLM_SAFETY_MARGIN_TOKENS:-128}" diff --git a/compose.yaml b/compose.yaml index dec7f9e..7dbb7c6 100644 --- a/compose.yaml +++ b/compose.yaml @@ -10,6 +10,9 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l LOG_LEVEL: "${LOG_LEVEL:-info}" LOG_PRETTY: "${LOG_PRETTY:-auto}" + # Active pipeline selected by HTTP adapters. + DEFAULT_PIPELINE: "${DEFAULT_PIPELINE:-truncate}" + # Pipeline concurrency and renderer selection. SOURCE_CONCURRENCY: "${SOURCE_CONCURRENCY:-1}" LLM_CONCURRENCY: "${LLM_CONCURRENCY:-1}" @@ -18,13 +21,16 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l OUTPUT_TARGET_CHARS: "${OUTPUT_TARGET_CHARS:-25000}" # Source provider. - FIRECRAWL_BASE_URL: "${FIRECRAWL_BASE_URL:?Set FIRECRAWL_BASE_URL in .env or the shell}" + SOURCE_PROVIDER: "${SOURCE_PROVIDER:-default-http}" + FIRECRAWL_BASE_URL: "${FIRECRAWL_BASE_URL:-}" FIRECRAWL_API_KEY: "${FIRECRAWL_API_KEY:-}" + DOCLING_BASE_URL: "${DOCLING_BASE_URL:-}" + DOCLING_API_KEY: "${DOCLING_API_KEY:-}" # LLM provider. - LLM_BASE_URL: "${LLM_BASE_URL:?Set LLM_BASE_URL in .env or the shell}" + LLM_BASE_URL: "${LLM_BASE_URL:-}" LLM_API_KEY: "${LLM_API_KEY:-}" - LLM_MODEL: "${LLM_MODEL:?Set LLM_MODEL in .env or the shell}" + LLM_MODEL: "${LLM_MODEL:-}" LLM_CONTEXT_TOKENS: "${LLM_CONTEXT_TOKENS:-}" LLM_CHARS_PER_TOKEN: "${LLM_CHARS_PER_TOKEN:-3.5}" LLM_SAFETY_MARGIN_TOKENS: "${LLM_SAFETY_MARGIN_TOKENS:-128}" diff --git a/config/llm-context-loader.yaml b/config/llm-context-loader.yaml index 70b3921..2de37f9 100644 --- a/config/llm-context-loader.yaml +++ b/config/llm-context-loader.yaml @@ -1,16 +1,16 @@ schemaVersion: 1 -# HTTP adapters expose the default pipeline through client-specific contracts. +# HTTP adapters expose the selected pipeline through client-specific contracts. httpAdapters: open-webui: type: open-webui - pipeline: default + pipeline: ${DEFAULT_PIPELINE:-truncate} config: auth: bearerToken: ${OWUI_AUTH_TOKEN:-} jina: type: jina - pipeline: default + pipeline: ${DEFAULT_PIPELINE:-truncate} config: auth: bearerToken: ${JINA_AUTH_TOKEN:-} @@ -28,50 +28,78 @@ outputRenderers: # Source providers turn an input URL into source markdown. sourceProviders: + default-http: + type: http + config: + userAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" + maxBytes: 5000000 + titleFromHtml: true default-firecrawl: type: firecrawl config: - baseUrl: ${FIRECRAWL_BASE_URL} + baseUrl: ${FIRECRAWL_BASE_URL:-} apiKey: ${FIRECRAWL_API_KEY:-} + output: markdown onlyMainContent: true - formats: [markdown] - maxAge: 0 + stripBase64Images: true + parsePdf: true + default-docling: + type: docling + config: + baseUrl: ${DOCLING_BASE_URL:-} + apiKey: ${DOCLING_API_KEY:-} + output: markdown + doOcr: true + tableMode: accurate # LLM providers run prompt-rendered chat-completions passes. llmProviders: default-llm: type: openai-chat config: - baseUrl: ${LLM_BASE_URL} + baseUrl: ${LLM_BASE_URL:-} apiKey: ${LLM_API_KEY:-} - model: ${LLM_MODEL} + model: ${LLM_MODEL:-} contextTokens: ${LLM_CONTEXT_TOKENS:-} charsPerToken: ${LLM_CHARS_PER_TOKEN:-3.5} safetyMarginTokens: ${LLM_SAFETY_MARGIN_TOKENS:-128} extraBody: {} -# The default pipeline fetches, cleans, verifies URLs, summarizes, and caps output size. +# Shipped pipelines. DEFAULT_PIPELINE selects which is active (truncate by default). pipelines: - default: + truncate: + outputRenderer: ${DEFAULT_OUTPUT_RENDERER:-debug-xml} + limiters: + source: ${SOURCE_CONCURRENCY:-1} + steps: + - type: load-source + name: fetch + concurrencyGroup: source + timeoutSeconds: ${LOAD_SOURCE_TIMEOUT_SECONDS:-20} + config: + provider: ${SOURCE_PROVIDER:-default-http} + - type: truncate + name: truncate + config: + targetChars: ${OUTPUT_TARGET_CHARS:-25000} + + clean-llm: outputRenderer: ${DEFAULT_OUTPUT_RENDERER:-debug-xml} limiters: source: ${SOURCE_CONCURRENCY:-1} llm: ${LLM_CONCURRENCY:-1} steps: - # Fetch source markdown and keep source work in the source limiter group. - type: load-source - name: firecrawl + name: fetch concurrencyGroup: source timeoutSeconds: ${LOAD_SOURCE_TIMEOUT_SECONDS:-20} config: - provider: default-firecrawl - # Capture trusted source URLs before any LLM pass can alter the body. + provider: ${SOURCE_PROVIDER:-default-firecrawl} - type: capture-urls name: capture_source_urls concurrencyGroup: source config: artifact: trusted-urls - # Clean readable source markdown while preserving roughly the original budget. - type: llm-pass name: clean concurrencyGroup: llm @@ -83,7 +111,6 @@ pipelines: templates: system: ../templates/clean.system.md user: ../templates/clean.user.md - # Roll back to source content when the clean pass introduces new URLs. - type: verify-urls name: verify_after_clean concurrencyGroup: llm @@ -91,7 +118,6 @@ pipelines: artifact: trusted-urls onHallucination: rollback maxReportedUrls: 0 - # Summarize only once content exceeds the target output size. - type: llm-pass name: summarize concurrencyGroup: llm @@ -105,7 +131,6 @@ pipelines: user: ../templates/summarize.user.md vars: targetChars: ${OUTPUT_TARGET_CHARS:-25000} - # Report summary URL drift without replacing the current body. - type: verify-urls name: verify_after_summarize concurrencyGroup: llm @@ -113,7 +138,6 @@ pipelines: artifact: trusted-urls onHallucination: report maxReportedUrls: 50 - # Enforce the final hard character budget. - type: truncate name: truncate config: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0c09e95..b9fc5e1 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -66,7 +66,7 @@ scripts/ developer smoke helpers The current built-ins are: - HTTP adapters: `open-webui`, `jina`. -- Source providers: `firecrawl`. +- Source providers: `http`, `firecrawl`, `docling`. - LLM providers: `openai-chat`. - Pipeline steps: `load-source`, `llm-pass`, `truncate`, `capture-urls`, `verify-urls`. - Output renderers: `debug-xml`, `passthrough`. @@ -99,6 +99,15 @@ One important boundary is explicit in the architecture test: `src/core/` must no `src/core/` is the framework-free runtime engine. It does not know about Firecrawl, OpenAI, Open WebUI, Jina, or provider categories. +The pipeline body is a **discriminated union** (`TextBody | BinaryBody`), discriminated on `kind`: + +- **Text bodies** (`kind: "text"`) carry `content` as a `string` and are the payload text-only steps read and write. +- **Binary bodies** (`kind: "binary"`) carry `bytes` as a `Uint8Array` and are carried through the pipeline for conversion by a later step. + +A binary body that reaches the end of the pipeline without being converted is treated as a **failed run** with the error `unconverted_binary: `. + +Run reports use representation-neutral length diagnostics: `initial_length` and `final_length` are characters for text bodies and bytes for binary bodies. `ratio` is reported only when the first and final body have the same representation. + Key pieces: - `PipelineStep` (`src/contracts/pipeline/step.ts`) is the executable step port: `run(ctx): Promise`. @@ -158,10 +167,12 @@ Bootstrap environment is intentionally small and parsed by `src/config/env-confi The YAML document is translated to `AppConfig` before runtime construction: 1. `src/config/yaml/yaml-config.ts` validates the coarse YAML shape: `httpAdapters`, `outputRenderers`, `sourceProviders`, `llmProviders`, and `pipelines`. -2. `src/config/yaml/yaml-app-config.ts` maps engine-owned sections to `AppConfig.engineConfig`, HTTP adapter declarations to `AppConfig.adapters.http`, and `schemaVersion` to app metadata. +2. `src/config/yaml/yaml-app-config.ts` maps engine-owned sections to `AppConfig.engineConfig`, HTTP adapter declarations to `AppConfig.adapters.http`, and `schemaVersion` to app metadata. Pipeline activation follows a tri-state `enabled`: `true` compiles, `false` parks it, **omitted** activates only when an HTTP adapter references it. 3. Each built-in descriptor parses its own `config` block with a local schema during engine or adapter construction. -YAML environment substitution happens before schema validation. See [CONFIGURATION.md](CONFIGURATION.md) for default operation and [CUSTOMIZATION.md](CUSTOMIZATION.md) for YAML structure. +YAML environment substitution happens before schema validation. A missing `${VAR}` with no `:-` default resolves to `""` so the env layer never throws for unused config. Required-ness is owned by reachable built-in schemas, not by substitution. + +**Lazy validation**: providers and output renderers are built and validated only when an active pipeline references them. An unused provider with missing env vars does not fail startup. `EngineConfig` excludes adapter declarations and schema metadata. `AppConfig` is the app-level envelope that pairs `engineConfig` with adapter configuration such as `adapters.http`. diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index fa9fcee..77947f8 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -47,51 +47,71 @@ These are consumed by helper scripts and ignored by the Node app. The default YAML file uses environment substitution for provider URLs, tokens, concurrency, timeouts, and renderer selection. Most of these values are shown in [.env.example](../.env.example) and passed through the Compose files. -The default pipeline cannot run without a Firecrawl base URL, an OpenAI-compatible LLM base URL, and a model name. Compose enforces those three values up front with required-variable interpolation. Direct Node startup enforces them during YAML substitution and provider config parsing. +The default (`truncate`) pipeline needs no provider configuration — it fetches and truncates. The `clean-llm` pipeline uses an LLM and Firecrawl; those env vars are +required only when `clean-llm` is active (set `DEFAULT_PIPELINE=clean-llm`). + +Validation is lazy: the service validates and constructs only the providers and pipelines that +are reachable from the active configuration. An unused provider with missing env vars will not +fail startup. The placeholders below are grouped by the part of the pipeline they configure. +### Source provider selection + +| Variable | Default / behavior | Purpose | +| ----------------- | ------------------ | -------------------------------------------------------------- | +| `SOURCE_PROVIDER` | `default-http` | Named source provider used by the `load-source` pipeline step. | + +The built-in `http` provider has no environment variables — it fetches the input URL directly. + ### Source provider (Firecrawl) -| Variable | Default / behavior | Purpose | -| -------------------- | ------------------ | -------------------------------- | -| `FIRECRAWL_BASE_URL` | Required | Firecrawl base URL. | -| `FIRECRAWL_API_KEY` | empty | Optional Firecrawl bearer token. | +| Variable | Default / behavior | Purpose | +| -------------------- | ------------------ | -------------------------------------------------------------------------------------------- | +| `FIRECRAWL_BASE_URL` | empty | Firecrawl base URL. Required only when a pipeline referencing `default-firecrawl` is active. | +| `FIRECRAWL_API_KEY` | empty | Optional Firecrawl bearer token. | + +### Source provider (Docling) + +| Variable | Default / behavior | Purpose | +| ------------------ | ------------------ | ---------------------------------------------------------------------------------------------- | +| `DOCLING_BASE_URL` | empty | Docling Serve base URL. Required only when a pipeline referencing `default-docling` is active. | +| `DOCLING_API_KEY` | empty | Optional Docling API key. | ### LLM provider (OpenAI-compatible) -| Variable | Default / behavior | Purpose | -| -------------------------- | ------------------ | ----------------------------------------------------------------------------- | -| `LLM_BASE_URL` | Required | OpenAI-compatible API base URL, usually ending in `/v1`. | -| `LLM_API_KEY` | empty | Optional LLM bearer token. | -| `LLM_MODEL` | Required | Model identifier sent to chat completions. | -| `LLM_CONTEXT_TOKENS` | empty in YAML | Optional model context window in tokens. Empty disables the context-fit gate. | -| `LLM_CHARS_PER_TOKEN` | `3.5` | Conservative chars-per-token estimator used for the context-fit gate. | -| `LLM_SAFETY_MARGIN_TOKENS` | `128` | Extra tokens reserved for chat-template framing and estimator drift. | +| Variable | Default / behavior | Purpose | +| -------------------------- | ------------------ | --------------------------------------------------------------------------------------------------------------------------- | +| `LLM_BASE_URL` | empty | OpenAI-compatible API base URL, usually ending in `/v1`. Required only when a pipeline referencing `default-llm` is active. | +| `LLM_API_KEY` | empty | Optional LLM bearer token. | +| `LLM_MODEL` | empty | Model identifier sent to chat completions. Required only when a pipeline referencing `default-llm` is active. | +| `LLM_CONTEXT_TOKENS` | empty in YAML | Optional model context window in tokens. Empty disables the context-fit gate. | +| `LLM_CHARS_PER_TOKEN` | `3.5` | Conservative chars-per-token estimator used for the context-fit gate. | +| `LLM_SAFETY_MARGIN_TOKENS` | `128` | Extra tokens reserved for chat-template framing and estimator drift. | ### Pipeline output and step thresholds -| Variable | Default / behavior | Purpose | -| ----------------------------- | ------------------ | ------------------------------------------------------------ | -| `OUTPUT_TARGET_CHARS` | `25000` | Desired maximum characters returned by the default pipeline. | -| `LOAD_SOURCE_TIMEOUT_SECONDS` | `20` | Per-call timeout for the source-loading step. | -| `CLEAN_MIN_INPUT_CHARS` | `1000` | Minimum body characters before the clean LLM pass runs. | -| `CLEAN_TIMEOUT_SECONDS` | `60` | Per-call timeout for the clean LLM pass. | -| `SUMMARIZE_TIMEOUT_SECONDS` | `60` | Per-call timeout for the summarize LLM pass. | +| Variable | Default / behavior | Purpose | +| ----------------------------- | ------------------ | ----------------------------------------------------------- | +| `OUTPUT_TARGET_CHARS` | `25000` | Desired maximum characters returned by the active pipeline. | +| `LOAD_SOURCE_TIMEOUT_SECONDS` | `20` | Per-call timeout for the source-loading step. | +| `CLEAN_MIN_INPUT_CHARS` | `1000` | Minimum body characters before the clean LLM pass runs. | +| `CLEAN_TIMEOUT_SECONDS` | `60` | Per-call timeout for the clean LLM pass. | +| `SUMMARIZE_TIMEOUT_SECONDS` | `60` | Per-call timeout for the summarize LLM pass. | ### Concurrency -| Variable | Default / behavior | Purpose | -| -------------------- | ------------------ | --------------------------------------------------------------- | -| `SOURCE_CONCURRENCY` | `1` | Maximum concurrent source-loading groups. | -| `LLM_CONCURRENCY` | `1` | Maximum concurrent LLM workflow groups in the default pipeline. | +| Variable | Default / behavior | Purpose | +| -------------------- | ------------------ | ------------------------------------------------------------- | +| `SOURCE_CONCURRENCY` | `1` | Maximum concurrent source-loading groups. | +| `LLM_CONCURRENCY` | `1` | Maximum concurrent LLM workflow groups (used by `clean-llm`). | ### Output rendering -| Variable | Default / behavior | Purpose | -| --------------------------- | ------------------ | ---------------------------------------------------------------------------- | -| `DEFAULT_OUTPUT_RENDERER` | `debug-xml` | Output renderer name for the default pipeline when set to a non-empty value. | -| `DEBUG_XML_INCLUDE_SKIPPED` | `false` | Include skipped steps in the debug-xml footer (`true`/`false`). | +| Variable | Default / behavior | Purpose | +| --------------------------- | ------------------ | --------------------------------------------------------------------------- | +| `DEFAULT_OUTPUT_RENDERER` | `debug-xml` | Output renderer name for the active pipeline when set to a non-empty value. | +| `DEBUG_XML_INCLUDE_SKIPPED` | `false` | Include skipped steps in the debug-xml footer (`true`/`false`). | ### Inbound authentication @@ -106,17 +126,17 @@ The placeholders below are grouped by the part of the pipeline they configure. YAML substitution happens before schema validation. -| Syntax | Meaning | -| ------------------ | ------------------------------------------------------------------------- | -| `${VAR}` | Required. Startup fails if `VAR` is not set. | -| `${VAR:-fallback}` | Optional. Uses `VAR` when set to a non-empty value, otherwise `fallback`. | -| `$${VAR}` | Literal escape. Produces `${VAR}` in the parsed config. | +| Syntax | Meaning | +| ------------------ | --------------------------------------------------------------- | +| `${VAR}` | Resolves to the value of `VAR`, or `""` if unset. | +| `${VAR:-fallback}` | Uses `VAR` when set to a non-empty value, otherwise `fallback`. | +| `$${VAR}` | Literal escape. Produces `${VAR}` in the parsed config. | Substitution applies recursively to YAML string values. Non-string YAML values are left as YAML values and then parsed by schemas. -Substitution itself is string-only. Numeric and boolean fields recover their types during schema parsing, and blank env values use the field's schema default when that field has one. Blank strings remain meaningful for token fields such as `FIRECRAWL_API_KEY`, `LLM_API_KEY`, `OWUI_AUTH_TOKEN`, and `JINA_AUTH_TOKEN`, where empty means no token. +Substitution itself is string-only. Numeric and boolean fields recover their types during schema parsing, and blank env values use the field's schema default when that field has one. Blank strings remain meaningful for token fields such as `FIRECRAWL_API_KEY`, `DOCLING_API_KEY`, `LLM_API_KEY`, `OWUI_AUTH_TOKEN`, and `JINA_AUTH_TOKEN`, where empty means no token. -Docker Compose performs its own interpolation before the container starts. The shipped Compose files use `${VAR:?message}` for values that must be supplied by `.env` or the shell, so missing or blank provider values fail before the container is created. +Docker Compose performs its own interpolation before the container starts. The shipped Compose files pass all provider variables through with empty defaults (`${VAR:-}`) so the container always starts and the Node process validates only the active pipeline's providers at startup. ## Authentication @@ -129,6 +149,7 @@ Inbound authentication is configured per HTTP adapter in YAML. The default YAML Outbound authentication is configured per provider: - `FIRECRAWL_API_KEY` is sent as bearer auth to Firecrawl when set. +- `DOCLING_API_KEY` is sent via the `X-Api-Key` header to Docling Serve when set. - `LLM_API_KEY` is sent as bearer auth to the OpenAI-compatible endpoint when set. ## Local And Container Networking @@ -139,8 +160,7 @@ When running in Docker, `localhost` inside the container means the container its ## Common Startup Failures -- Missing required environment variable: a YAML placeholder such as `${FIRECRAWL_BASE_URL}` or `${LLM_MODEL}` was not provided and has no fallback in the running environment. -- Missing required Compose variable: a Compose placeholder such as `${FIRECRAWL_BASE_URL:?Set FIRECRAWL_BASE_URL in .env or the shell}` was unset or blank. +- Provider config validation: a reachable provider (referenced by an active pipeline) has an empty required field, e.g. `"firecrawl baseUrl is required"`. The error names the config field, not the env var. - Invalid YAML shape: the top-level YAML structure failed the coarse schema in `src/config/yaml/yaml-config.ts`. - Unknown type: a YAML `type` does not exist in the selected built-in descriptor bundles. - Unknown reference: a pipeline, output renderer, provider, or concurrency group name references an instance that was not declared. diff --git a/docs/CUSTOMIZATION.md b/docs/CUSTOMIZATION.md index be0c0da..6dcb386 100644 --- a/docs/CUSTOMIZATION.md +++ b/docs/CUSTOMIZATION.md @@ -34,10 +34,17 @@ HTTP adapters also name the pipeline they expose: ```yaml some-adapter: type: open-webui - pipeline: default + pipeline: ${DEFAULT_PIPELINE:-truncate} config: {} ``` +Each pipeline has an optional `enabled` field (tri-state: `true`, `false`, or omitted). When +omitted, a pipeline is active only when at least one HTTP adapter references it. Setting +`enabled: false` parks it. Setting `enabled: true` pins it active regardless of adapter +references. The shipped pipelines leave `enabled` omitted, so `DEFAULT_PIPELINE` alone drives +the active set. Active-set pipelines are compiled and validated at startup; inactive ones are +skipped (their providers are never built). + Pipeline steps include common orchestration fields plus a type-specific `config` block: ```yaml @@ -93,7 +100,7 @@ config: includeSkipped: false ``` -Appends an XML diagnostic footer to the body. If the pipeline produced no body, the footer is returned by itself. `includeSkipped` defaults to `false`; set it to `true` to list skipped steps in the footer. +Appends an XML diagnostic footer to the body. If the pipeline produced no body, or the final body is binary, the footer is returned by itself. Body lengths in the footer are characters for text and bytes for binary. `includeSkipped` defaults to `false`; set it to `true` to list skipped steps in the footer. ### `passthrough` @@ -101,22 +108,68 @@ Appends an XML diagnostic footer to the body. If the pipeline produced no body, config: {} ``` -Returns the current body without a diagnostic footer. If the pipeline failed before producing a body, returns the pipeline error message. +Returns the current text body without a diagnostic footer. If the pipeline failed before producing a text body, returns the pipeline error message. ## Source Providers +### `http` + +```yaml +config: + userAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36" + maxBytes: 5000000 + titleFromHtml: true +``` + +| Knob | Values | Default | Purpose | +| --------------- | ------ | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `userAgent` | string | Chrome 124 Linux UA | User-Agent header sent with the fetch request. | +| `maxBytes` | int | `5000000` | Response body size cap. The body is read as a stream and the connection is cancelled once the cap is reached, so memory stays bounded. When the cap truncates the body, the document is flagged truncated and the load-source step reports `degraded`. | +| `titleFromHtml` | bool | `true` | When enabled, extracts the first `` tag content from the HTML response. | + +**Security caveat — testing only.** This provider fetches the input URL directly with no SSRF protection. It is intended as a zero-dependency testing fallback — no `baseUrl`, no `apiKey`, no external service required. + +The provider preserves textual response media types from `Content-Type` (`text/*`, JSON, XML, and structured `+json`/`+xml` types). Missing `Content-Type` defaults to `text/plain` when the body passes the text sniff. Binary responses are returned as binary bodies; mislabeled text-like responses containing NUL bytes are reclassified as `application/octet-stream`. If a binary body reaches the end of the pipeline without a converter, the run fails with `unconverted_binary: <mediaType>` instead of returning raw bytes. + ### `firecrawl` ```yaml config: baseUrl: ${FIRECRAWL_BASE_URL} apiKey: ${FIRECRAWL_API_KEY:-} + output: markdown onlyMainContent: true - formats: [markdown] - maxAge: 0 + stripBase64Images: true + parsePdf: true ``` -The provider calls `/v2/scrape` and returns markdown plus an optional title. Upstream HTTP, parse, empty, and network failures are converted to degradable upstream errors. +| Knob | Values | Default | Purpose | +| ------------------- | --------------------------------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `output` | `markdown` \| `html` \| `rawHtml` | `markdown` | Which format Firecrawl returns. `html` is cleaned main-content HTML. `rawHtml` is the full JS-rendered DOM (use for Readability testing). | +| `onlyMainContent` | bool | `true` | When enabled Firecrawl extracts the main page content and strips headers, nav, footers. No-op for `rawHtml`. | +| `stripBase64Images` | bool | `true` | Maps to `removeBase64Images` — replaces inline data URIs with short placeholders in markdown output. No-op for `html`/`rawHtml`. | +| `parsePdf` | bool | `true` | When enabled Firecrawl parses PDF files to markdown via `parsers: ["pdf"]`. | + +The provider calls `/v2/scrape`. Upstream HTTP, parse, empty, and network failures are converted to degradable upstream errors. Title is read from `data.metadata.title` (format-independent). `output: markdown` returns `text/markdown`; `output: html` and `output: rawHtml` return `text/html`. + +### `docling` + +```yaml +config: + baseUrl: ${DOCLING_BASE_URL} + apiKey: ${DOCLING_API_KEY:-} + output: markdown + doOcr: true + tableMode: accurate +``` + +| Knob | Values | Default | Purpose | +| ----------- | -------------------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `output` | `markdown` \| `html` | `markdown` | Which format Docling returns. `html` is the Docling HTML serializer output (polished semantic HTML with inline CSS — does NOT provide raw/rough HTML). | +| `doOcr` | bool | `true` | Run OCR on scanned documents and images within PDFs. | +| `tableMode` | `fast` \| `accurate` | `accurate` | Table extraction quality. `accurate` is slower but better for complex layouts. | + +The provider calls `POST /v1/convert/source`. Title is read from `document.json_content.name` (the JSON format is always requested internally regardless of the `output` setting). Upstream HTTP, parse, empty, and network failures are converted to degradable upstream errors. `output: markdown` returns `text/markdown`; `output: html` returns `text/html`. ## LLM Providers @@ -171,7 +224,7 @@ Adjacent steps that share the same `concurrencyGroup` share one limiter acquisit ```yaml config: - provider: default-firecrawl + provider: ${SOURCE_PROVIDER:-default-http} ``` Loads the initial body from a named source provider. If a body already exists, the step skips with `body_present`. @@ -192,9 +245,10 @@ config: targetChars: 25000 ``` -Runs a prompt-rendered LLM transformation against the current body. The step skips when: +Runs a prompt-rendered LLM transformation against the current text body. The step skips when: - there is no body (`no_body`) +- the current body is binary (`unsupported_media_type`) - input is shorter than `minInputChars`, when set (`too_short`) - input is longer than `maxInputChars`, when set (`too_long`) - the provider reports the rendered prompt plus the reserved output cannot fit the model context window (`context_overflow`) @@ -212,7 +266,7 @@ Both templates are required. `templates.vars` is an optional map of literal valu | ------------------ | ------------------------------------------------- | | `url` | Input URL for this pipeline run. | | `title` | Current body title, when available. | -| `content` | Current markdown body content. | +| `content` | Current text body content. | | `templates.vars.*` | Any keys declared under `templates.vars` in YAML. | Mustache escaping is disabled for prompt templates so markdown is passed through as-is. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 38b3382..66b24d6 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -38,21 +38,11 @@ fetch (HTML or Markdown) The URL-in / clean-Markdown-within-a-budget contract does not change. What changes is how the middle is done: established extraction and conversion tools instead of a model. We own the plumbing — content-type routing, budgets, honesty gates — and the surfaces — HTTP/MCP/CLI, observability, caching. We do not write our own readability heuristics or HTML parser. -Three cross-cutting changes enable the rest: - -- **Content-typed body.** The pipeline body and `SourceDocument` carry a media type (Markdown and HTML now; PDF, DOCX, and images later). Steps act or skip based on it. -- **Fetch providers declare an output format.** Providers that can emit more than one format (Firecrawl, Docling) are configured for the format the pipeline wants, and tag the document they produce so later steps know what they received. -- **Deterministic transform steps.** Main-content extraction and HTML-to-Markdown conversion become composable steps that run only on a matching content type. - ## Coming soon -The pivot above, sequenced so each step delivers value on its own and de-risks the next. - -1. **Docling fetch provider (Markdown).** A drop-in `SourceProvider` over `docling-serve` (`POST /v1/convert/source`). No core changes; it immediately adds a higher-fidelity, document-capable (PDF, Office) alternative to Firecrawl. -2. **Content-typed body and fetch output formats.** Add a media type to the body and `SourceDocument`; let Firecrawl and Docling return HTML and tag it. This is the keystone everything else builds on. -3. **Deterministic extract and convert steps.** First implementations: main-content extraction with Mozilla Readability (in-process, needs only a DOM such as linkedom) and HTML-to-Markdown with node-html-markdown or Turndown. Composed, these replace the LLM clean pass on the HTML path. -4. **Demote the LLM clean pass.** Once deterministic clean matches or beats it on representative URLs, make the clean stage optional and off by default. Keep summarize. -5. **Keep summaries honest.** Extend the existing URL quality gate from all-or-nothing rejection into a deterministic repair pass, and add fenced-code-block verification/repair for the summarize stage. +1. **Deterministic extract and convert steps.** First implementations: main-content extraction with Mozilla Readability (in-process, needs only a DOM such as linkedom) and HTML-to-Markdown with node-html-markdown or Turndown. Composed, these replace the LLM clean pass on the HTML path. +2. **Demote the LLM clean pass.** Once deterministic clean matches or beats it on representative URLs, make the clean stage optional and only execute it for URLs where deterministic clean did not produce desired results. Keep summarize. +3. **Keep summaries honest.** Extend the existing URL quality gate from all-or-nothing rejection into a deterministic repair pass, and implement fenced-code-block verification/repair. A couple of polish items ride along: @@ -63,9 +53,8 @@ A couple of polish items ride along: Near-term additions once the pivot above settles. -- **Playwright fetch provider (HTML/DOM)** to remove the hard dependency on a running Firecrawl or Docling instance for HTML pages. -- **Handling non-HTML, non-Markdown content in the pipeline** for configurations without a conversion-capable fetch provider. -- **`ContentTransformer` provider category.** Once a second extractor or converter implementation exists, promote the transform steps into a named provider category so the choice is plug-and-play in YAML. Candidate implementations: Readability, node-html-markdown, Turndown, and Trafilatura behind a small service. +- **Playwright fetch provider (HTML/DOM)** to remove the hard dependency on a running Firecrawl instance for HTML pages. +- **Docling for document conversion.** Use Docling to convert PDFs, Office documents, and other document-type files to markdown. ## Mid term diff --git a/docs/agents/logging-and-errors.md b/docs/agents/logging-and-errors.md index 8f400a4..398c36f 100644 --- a/docs/agents/logging-and-errors.md +++ b/docs/agents/logging-and-errors.md @@ -43,7 +43,7 @@ Logging uses pino. Common identity fields include `component`, `source_provider`, `llm_provider`, `http_adapter`, `output_renderer`, `pipeline`, `step`, `type`, and `step_index`. -Common per-call fields include `duration_ms`, `input_chars`, `output_chars`, `upstream_code`, `upstream_status`, `url_count`, `ok_count`, `degraded_count`, `failed_count`, and `err`. +Common per-call fields include `duration_ms`, `input_length`, `output_length`, `upstream_code`, `upstream_status`, `url_count`, `ok_count`, `degraded_count`, `failed_count`, and `err`. Use snake_case for structured log fields. diff --git a/docs/agents/smoke-testing.md b/docs/agents/smoke-testing.md new file mode 100644 index 0000000..d00e86a --- /dev/null +++ b/docs/agents/smoke-testing.md @@ -0,0 +1,117 @@ +# Agent Smoke Testing Guide + +Agent-only rule sheet for the smoke scripts under `scripts/`. Kept terse — each script's header comment is the primary reference. This guide exists to keep agents out of the recurring traps, not to document the pipeline. + +## Scripts + +| Script | Purpose | +| ---------------------- | ----------------------------------------------------- | +| `gather-urls.ps1` | Collect candidate URLs from a SearXNG instance. | +| `smoke-jina.ps1` | Exercise the Jina-style `GET /r/<url>` surface. | +| `smoke-owui.ps1` | Exercise the Open WebUI `POST /` batch surface. | +| `inspect-suspects.ps1` | Compare raw source against loader output for one URL. | +| `shared-lib.ps1` | Shared helpers; dot-sourced, not run directly. | + +Artifacts land in `scripts/out/` (gitignored). + +## Picking URLs + +- Keep the set small — **5 URLs or fewer**. Each provider × surface × environment combination reruns the whole set, so it adds up fast. +- Cover a couple of content shapes: at least one ordinary HTML article, and at least one real document (a true PDF, confirmed by content type rather than a `.pdf` in the path). +- Skip raw images; they only produce binary-rejection results and teach nothing new. + +```powershell +# gather-urls.ps1 reads SEARX_BASE from the environment and does NOT load .env. +$env:SEARX_BASE = 'http://<your-searxng-host>:<port>' +./scripts/gather-urls.ps1 -Count 5 -OutFile scripts/out/urls-smoke.txt + +# Or pass known URLs inline (no SearXNG needed): +./scripts/smoke-jina.ps1 -Urls @('https://example.com/article', 'https://example.com/file.pdf') +``` + +## Choosing a Source Provider + +`SOURCE_PROVIDER` selects one of the source providers declared in the YAML config. Read the config for the current names, then run the set against each backend you care about. The provider categories behave differently, which is the usual "is this broken or expected?" question: + +- **Native fetch** — requests the URL directly, no external service. Content it cannot turn into text (PDFs, images) returns a failed result with a binary-rejection error. +- **External scraper** — hands fetching and cleanup to an external service that handles many content types, including PDFs. Reads its base URL from the environment. +- **Document converter** — converts uploaded documents (PDF, Office, images) to markdown. It does not scrape generic HTML pages; those come back as an empty-source failure. Pair it with document URLs. + +Providers that depend on an external service read the hostname from the environment — never hard-code hosts in the doc or scripts. + +## Running Locally (Node) + +```powershell +# SOURCE_PROVIDER is read once at startup — set it BEFORE starting the server. +$env:SOURCE_PROVIDER = '<provider-name>' +npm run dev + +# In a second terminal: +./scripts/smoke-jina.ps1 -UrlFile scripts/out/urls-smoke.txt -Concurrency 2 +./scripts/smoke-owui.ps1 -UrlFile scripts/out/urls-smoke.txt -Concurrency 2 -BatchSize 3 + +# Stop the server before switching providers so the port frees up: +Get-Process -Name node -ErrorAction SilentlyContinue | + Where-Object { $_.MainWindowTitle -eq '' -and $_.Id -ne $pid } | + Stop-Process -Force -ErrorAction SilentlyContinue +``` + +Changing `$env:SOURCE_PROVIDER` while the server runs has no effect — restart it. + +## Running in Docker + +```powershell +# Compose reads variables from the .env FILE. Terminal $env: vars are NOT passed in. +# Edit SOURCE_PROVIDER in .env first, then: +docker compose build # only when source code changed +docker compose up -d +./scripts/smoke-jina.ps1 -UrlFile scripts/out/urls-smoke.txt -HealthTimeoutSec 30 -Concurrency 2 +./scripts/smoke-owui.ps1 -UrlFile scripts/out/urls-smoke.txt -HealthTimeoutSec 30 -Concurrency 2 -BatchSize 3 +docker compose down +``` + +Give the container time to pass its healthcheck — `-HealthTimeoutSec 30` covers startup. + +Run the full set against each provider, on each surface, in each environment you need to cover. + +## Validating Configuration Without Starting + +```powershell +npm run dev -- --check +``` + +`--check` composes the app and validates the active configuration (pipelines, providers, +templates) without starting the HTTP listener. Exits 0 on success, 1 on failure. Useful in CI +or when iterating on YAML changes without needing a running server. + +## Reading the Output + +Each run prints a per-URL block and then aggregate breakdowns. When the `debug-xml` renderer is active, the per-URL footer is the source of truth: + +- `result` — overall outcome: `ok`, `degraded`, or `failed`. +- `error` — present on failures, with a short machine reason (for example a binary-rejection or empty-source reason). +- The footer also carries per-step entries plus length and ratio fields. Treat the step names as whatever the current pipeline defines — do not assume a fixed set. + +The summary block tallies `ok` / `fail` and per-step status counts (`ok`, `skipped`, `degraded`, `failed`) across the set. `skipped` is normal: a step opts out when it does not apply to that URL. + +For the OWUI surface the per-URL time column is the **batch** time shared by every URL in the batch, not a per-URL figure. Jina times are per request. + +When the script output is not enough, read the server logs: pipeline start/finish lines carry the URL, outcome, and duration, and step warnings carry a `reason`. To dig into a single URL, use `inspect-suspects.ps1`. + +## Common Traps + +- [ ] `SEARX_BASE` exported before `gather-urls.ps1`? It falls back to localhost otherwise. +- [ ] `SOURCE_PROVIDER` set **before** `npm run dev`? It is read once at startup. +- [ ] `DEFAULT_PIPELINE` set **before** `npm run dev`? It is also read once at startup. +- [ ] For Docker, edited `.env` rather than a terminal `$env:` var? Compose only reads the file. +- [ ] Previous server stopped before switching providers? Otherwise the port stays taken. +- [ ] `-HealthTimeoutSec 30` on Docker runs so the healthcheck can pass first? +- [ ] Low `-Concurrency` (e.g. 2) for providers that call an LLM or remote service? They are slow per URL. +- [ ] OWUI time column read as a batch time, not per URL? +- [ ] "PDF" URLs confirmed as real PDFs by content type, not just by their path? + +## Cleanup + +- Restore `.env` to its original `SOURCE_PROVIDER`. +- `docker compose down` if you started a container. +- Stop any leftover dev server with the kill command above. diff --git a/scripts/inspect-suspects.ps1 b/scripts/inspect-suspects.ps1 index f6613e5..44d609b 100644 --- a/scripts/inspect-suspects.ps1 +++ b/scripts/inspect-suspects.ps1 @@ -82,14 +82,7 @@ if (-not $SkipHealthCheck) { $outDir = Join-Path $PSScriptRoot 'out' New-Item -ItemType Directory -Force -Path $outDir | Out-Null -# Footer attributes worth surfacing in the console preview. -$footerHighlights = @( - 'returned', 'final_chars', - 'fetch_status', 'fetch_chars', - 'clean_status', 'clean_ratio', 'clean_reason', - 'summarize_status', 'summarize_ratio', 'summarize_reason', - 'truncate_status' -) +$footerRootHighlights = @('returned', 'result', 'final_length', 'ratio', 'duration_ms') Write-Host ("=== inspect {0} URLs ===" -f $Urls.Count) -ForegroundColor Cyan Write-Host ("loader : {0}" -f $loader.LoaderBase) -ForegroundColor DarkGray @@ -141,11 +134,19 @@ foreach ($url in $Urls) { if ($footerLine) { $attrs = Get-LoaderFooterAttrMap -Footer $footerLine Write-Host ' --- FOOTER ---' - foreach ($name in $footerHighlights) { + foreach ($name in $footerRootHighlights) { if ($attrs.ContainsKey($name)) { Write-Host (" {0,-18} {1}" -f $name, $attrs[$name]) } } + $steps = Get-LoaderFooterSteps -Footer $footerLine + foreach ($step in $steps) { + if ($step.Status) { + $line = " {0,-18} status={1}" -f $step.Name, $step.Status + if ($step.Reason) { $line += " reason={0}" -f $step.Reason } + Write-Host $line + } + } } else { Write-Host ' --- FOOTER --- <none>' diff --git a/scripts/shared-lib.ps1 b/scripts/shared-lib.ps1 index 4504aef..8035060 100644 --- a/scripts/shared-lib.ps1 +++ b/scripts/shared-lib.ps1 @@ -9,19 +9,27 @@ # Smoke scripts are best-effort: a single bad URL must not abort the run. $ErrorActionPreference = 'Continue' -# Standard set of footer attributes printed by Write-SmokeSummary. Kept here -# so smoke-owui and smoke-jina produce byte-identical breakdown sections. -$script:SmokeFooterAttrs = @( - @{ Attr = 'returned'; Label = 'Returned breakdown' }, - @{ Attr = 'result'; Label = 'Pipeline result breakdown' }, - @{ Node = 'firecrawl'; Attr = 'status'; LegacyAttr = 'fetch_status'; Label = 'Fetch status breakdown' }, - @{ Node = 'capture_source_urls'; Attr = 'status'; Label = 'Capture-urls status breakdown' }, - @{ Node = 'clean'; Attr = 'status'; LegacyAttr = 'clean_status'; Label = 'Clean status breakdown' }, - @{ Node = 'verify_after_clean'; Attr = 'status'; Label = 'Verify-after-clean status breakdown' }, - @{ Node = 'summarize'; Attr = 'status'; LegacyAttr = 'summarize_status'; Label = 'Summarize status breakdown' }, - @{ Node = 'verify_after_summarize'; Attr = 'status'; Label = 'Verify-after-summarize status breakdown' }, - @{ Node = 'truncate'; Attr = 'status'; LegacyAttr = 'truncate_status'; Label = 'Truncate status breakdown' } -) +# Return an ordered list of direct-child step elements from a diagnostic footer +# as [pscustomobject]@{ Name, Status, Reason }. Returns empty list on parse +# failure so callers degrade gracefully to root-attribute-only output. +function Get-LoaderFooterSteps { + param([string] $Footer) + if (-not $Footer) { return @() } + $steps = New-Object System.Collections.Generic.List[object] + try { + $xml = [xml]$Footer + foreach ($child in $xml.DocumentElement.ChildNodes) { + if ($child.NodeType -ne 'Element') { continue } + $steps.Add([pscustomobject]@{ + Name = $child.LocalName + Status = $child.GetAttribute('status') + Reason = $child.GetAttribute('reason') + }) + } + } + catch { } + return ,$steps +} # Read the loader base URL + optional bearer from the environment. function Get-LoaderDefaults { @@ -209,22 +217,19 @@ function ConvertTo-UrlSlug { return $slug } -# Tally one footer attribute across rows and print a sorted breakdown block. -# `$Rows` items must expose a `.footer` string property. +# Tally one root-level footer attribute across rows and print a sorted breakdown +# block. `$Rows` items must expose a `.footer` string property. Only root-level +# attributes (e.g. `returned`, `result`) are read; per-step status is handled by +# Write-StepStatusBreakdown via Get-LoaderFooterSteps. function Write-FooterBreakdown { param( [object[]] $Rows, [string] $AttrName, - [string] $Label, - [string] $NodeName, - [string] $LegacyAttrName + [string] $Label ) $byAttr = @{} foreach ($row in $Rows) { - $value = '' - if ($NodeName) { $value = Get-LoaderFooterNodeAttr -Footer $row.footer -NodeName $NodeName -AttrName $AttrName } - if (-not $value -and $LegacyAttrName) { $value = Get-LoaderFooterAttr -Footer $row.footer -AttrName $LegacyAttrName } - if (-not $value) { $value = Get-LoaderFooterAttr -Footer $row.footer -AttrName $AttrName } + $value = Get-LoaderFooterAttr -Footer $row.footer -AttrName $AttrName if (-not $value) { continue } if (-not $byAttr.ContainsKey($value)) { $byAttr[$value] = 0 } $byAttr[$value]++ @@ -236,9 +241,40 @@ function Write-FooterBreakdown { } } -# Print the standard per-URL block, an aggregate summary line, and the five -# footer attribute breakdowns. Used by every smoke script so their output is -# identical down to the wording. `$Results` items must expose: +# Tally the `status` attribute of each direct-child step element across rows, +# printing one breakdown block per discovered step name. Step names are +# discovered in first-seen order across rows. Grandchildren with a `status` +# attribute are never enumerated. +function Write-StepStatusBreakdown { + param([Parameter(Mandatory)] [object[]] $Rows) + # Collect ordered union of step names and tally status per step across rows. + $firstSeen = New-Object System.Collections.Generic.List[string] + $seen = @{} + # Map<name, Map<status, count>> + $byStep = @{} + foreach ($row in $Rows) { + $steps = Get-LoaderFooterSteps -Footer $row.footer + foreach ($step in $steps) { + $n = $step.Name + $s = $step.Status + if (-not $n -or -not $s) { continue } + if (-not $seen.ContainsKey($n)) { $seen[$n] = $true; $firstSeen.Add($n) } + if (-not $byStep.ContainsKey($n)) { $byStep[$n] = @{} } + if (-not $byStep[$n].ContainsKey($s)) { $byStep[$n][$s] = 0 } + $byStep[$n][$s]++ + } + } + foreach ($n in $firstSeen) { + Write-Output ("{0} status breakdown:" -f $n) + $byStep[$n].GetEnumerator() | Sort-Object Key | ForEach-Object { + Write-Output (" {0,-22} {1}" -f $_.Key, $_.Value) + } + } +} + +# Print the standard per-URL block, an aggregate summary line, root-level footer +# breakdowns, and per-step status breakdowns. Used by every smoke script so +# their output is identical down to the wording. `$Results` items must expose: # .url, .ok, .len, .ms, .footer, .error function Write-SmokeSummary { param( @@ -274,12 +310,9 @@ function Write-SmokeSummary { Write-Output ("{0} urls: {1} ok: {2} fail: {3} wall: {4}ms" -f $Header, $Results.Count, $ok, $fail, $WallMs) $footerRows = $Results | Where-Object { $_.footer } - foreach ($entry in $script:SmokeFooterAttrs) { - Write-FooterBreakdown ` - -Rows $footerRows ` - -AttrName $entry['Attr'] ` - -Label $entry['Label'] ` - -NodeName $entry['Node'] ` - -LegacyAttrName $entry['LegacyAttr'] + if ($footerRows.Count -gt 0) { + Write-FooterBreakdown -Rows $footerRows -AttrName 'returned' -Label 'Returned breakdown' + Write-FooterBreakdown -Rows $footerRows -AttrName 'result' -Label 'Pipeline result breakdown' + Write-StepStatusBreakdown -Rows $footerRows } } diff --git a/src/builtins/llm-providers/openai-chat/openai-chat-provider-config.ts b/src/builtins/llm-providers/openai-chat/openai-chat-provider-config.ts index 7e27467..4d2a7c5 100644 --- a/src/builtins/llm-providers/openai-chat/openai-chat-provider-config.ts +++ b/src/builtins/llm-providers/openai-chat/openai-chat-provider-config.ts @@ -12,9 +12,9 @@ import { emptyStringAsUndefined } from "../../../shared/config-coercion.js"; const openAiChatConfigSchema = z .object({ - baseUrl: z.string().url(), + baseUrl: z.string().min(1, "openai-chat baseUrl is required").url("openai-chat baseUrl must be a valid URL"), apiKey: z.string().default(""), - model: z.string().min(1), + model: z.string().min(1, "openai-chat model is required"), extraBody: z.record(z.unknown()).default({}), contextTokens: z.preprocess(emptyStringAsUndefined, z.coerce.number().int().positive().optional()), charsPerToken: z.preprocess(emptyStringAsUndefined, z.coerce.number().positive().default(3.5)), diff --git a/src/builtins/output-renderers/debug-xml/debug-xml-renderer.ts b/src/builtins/output-renderers/debug-xml/debug-xml-renderer.ts index d952b5b..4739503 100644 --- a/src/builtins/output-renderers/debug-xml/debug-xml-renderer.ts +++ b/src/builtins/output-renderers/debug-xml/debug-xml-renderer.ts @@ -1,12 +1,13 @@ /** - * Appends an XML diagnostic footer to the final markdown body. + * Appends an XML diagnostic footer to the final body content. * - * When a pipeline produced no body, the renderer returns the footer alone so - * Open WebUI and markdown clients still receive visible diagnostics instead of - * an empty document. + * When a pipeline produced no body, or the final body is binary (which cannot + * be rendered as text), the renderer returns the footer alone so Open WebUI + * and markdown clients still receive visible diagnostics. */ import { serializeFooter } from "./footer-serializer.js"; +import { isTextBody } from "../../../contracts/pipeline/context.js"; import type { OutputRenderer, @@ -30,7 +31,7 @@ export class DebugXmlRenderer implements OutputRenderer { /** Returns the body followed by an XML diagnostic footer. */ render(input: OutputRendererInput): OutputRendererResult { - const content = input.body?.content ?? ""; + const content = input.body && isTextBody(input.body) ? input.body.content : ""; const footer = serializeFooter(input.report, { rootElement: this.config.rootElement, includeSkipped: this.config.includeSkipped diff --git a/src/builtins/output-renderers/debug-xml/footer-serializer.ts b/src/builtins/output-renderers/debug-xml/footer-serializer.ts index 3f7de40..f05be39 100644 --- a/src/builtins/output-renderers/debug-xml/footer-serializer.ts +++ b/src/builtins/output-renderers/debug-xml/footer-serializer.ts @@ -22,8 +22,8 @@ export function serializeFooter(report: PipelineReport, options: FooterSerialize assertDiagnosticName(options.rootElement); const rootAttrs: Record<string, DiagnosticValue | undefined> = { url: report.url, - initial_chars: report.initialChars, - final_chars: report.finalChars, + initial_length: report.initialLength, + final_length: report.finalLength, ratio: report.ratio === undefined ? undefined : report.ratio.toFixed(3), duration_ms: report.durationMs, returned: report.returned, @@ -47,8 +47,8 @@ function renderStep(step: StepReport, indent: number): string { status: step.status, reason: step.reason, duration_ms: step.durationMs, - input_chars: step.inputChars, - output_chars: step.outputChars, + input_length: step.inputLength, + output_length: step.outputLength, ...step.diagnostics?.attributes }; return renderNode({ name: step.name, attributes: attrs, children: step.diagnostics?.children }, indent); diff --git a/src/builtins/output-renderers/passthrough/passthrough-renderer.ts b/src/builtins/output-renderers/passthrough/passthrough-renderer.ts index 6e39aa3..497c644 100644 --- a/src/builtins/output-renderers/passthrough/passthrough-renderer.ts +++ b/src/builtins/output-renderers/passthrough/passthrough-renderer.ts @@ -1,8 +1,9 @@ /** - * Returns the final markdown body without decoration. + * Returns the final body content without decoration. * - * When a pipeline failed before producing a body, this renderer returns the - * pipeline error message so clients do not receive a silent empty document. + * When a pipeline failed before producing a body, or when the final body is + * binary and no converter handled it, this renderer returns the pipeline error + * message so clients do not receive a silent empty document. */ import type { @@ -10,12 +11,14 @@ import type { OutputRendererInput, OutputRendererResult } from "../../../contracts/extensions/output-renderer.js"; +import { isTextBody } from "../../../contracts/pipeline/context.js"; /** Returns the final body content unchanged. */ export class PassthroughRenderer implements OutputRenderer { /** Returns the body content as markdown. */ render(input: OutputRendererInput): OutputRendererResult { - if (!input.body && input.report.result === "failed") return { markdown: input.report.error ?? "Pipeline failed." }; - return { markdown: input.body?.content ?? "" }; + if (input.body && isTextBody(input.body)) return { markdown: input.body.content }; + if (input.report.result === "failed") return { markdown: input.report.error ?? "Pipeline failed." }; + return { markdown: "" }; } } diff --git a/src/builtins/pipeline-steps/capture-urls/capture-urls-step.ts b/src/builtins/pipeline-steps/capture-urls/capture-urls-step.ts index ce6c0e3..2c864ee 100644 --- a/src/builtins/pipeline-steps/capture-urls/capture-urls-step.ts +++ b/src/builtins/pipeline-steps/capture-urls/capture-urls-step.ts @@ -1,16 +1,18 @@ /** * Captures canonical URLs from the current body into a pipeline artifact. * - * The captured artifact is the trusted inventory used by verify-urls quality - * gates after LLM transformations. It also seeds a sibling marker artifact - * (`<artifact>:checked-content`) with the verbatim source body, so verify-urls - * skips until a later step actually changes the body. Diagnostics expose only - * the artifact name and URL count; later steps read the Set through the bag. + * Skips when the current body is binary — URLs cannot be extracted from raw + * bytes. The captured artifact is the trusted inventory used by verify-urls + * quality gates after LLM transformations. It also seeds a sibling marker + * artifact (`<artifact>:checked-content`) with the verbatim source body, so + * verify-urls skips until a later step actually changes the body. Diagnostics + * expose only the artifact name and URL count; later steps read the Set + * through the bag. */ import { collectCanonicalUrls } from "../../../shared/markdown-urls.js"; -import type { PipelineContext } from "../../../contracts/pipeline/context.js"; +import { isTextBody, type PipelineContext } from "../../../contracts/pipeline/context.js"; import type { PipelineStep, StepResult } from "../../../contracts/pipeline/step.js"; import type { Logger } from "../../../shared/logger.js"; import type { CaptureUrlsStepOptions } from "./capture-urls-step-config.js"; @@ -27,6 +29,7 @@ export class CaptureUrlsStep implements PipelineStep { async run(ctx: PipelineContext): Promise<StepResult> { const body = ctx.body.current(); if (!body) return { status: "skipped", reason: "no_body" }; + if (!isTextBody(body)) return { status: "skipped", reason: "unsupported_media_type" }; const urls = collectCanonicalUrls(body.content); return { diff --git a/src/builtins/pipeline-steps/llm-pass/llm-pass-step.ts b/src/builtins/pipeline-steps/llm-pass/llm-pass-step.ts index ccf9e8d..cb01f03 100644 --- a/src/builtins/pipeline-steps/llm-pass/llm-pass-step.ts +++ b/src/builtins/pipeline-steps/llm-pass/llm-pass-step.ts @@ -1,13 +1,16 @@ /** - * Runs prompt-rendered LLM transformations against the current markdown body. + * Runs prompt-rendered LLM transformations against the current text body. * - * The step owns provider failure classification for LLM calls, template-render + * Skips without error when the current body is binary — the step operates on + * text only. Owns provider failure classification for LLM calls, template-render * failure handling, input length gates, and character-based context-fit checks. - * It preserves the current body title when the LLM returns replacement content. + * Preserves the current body title when the LLM returns replacement content. */ import { UpstreamError, isAbortError } from "../../../shared/errors.js"; +import { isTextBody } from "../../../contracts/pipeline/context.js"; + import type { LlmProvider } from "../../../contracts/extensions/llm-provider.js"; import type { PipelineContext } from "../../../contracts/pipeline/context.js"; import type { PipelineStep, StepResult } from "../../../contracts/pipeline/step.js"; @@ -27,6 +30,7 @@ export class LlmPassStep implements PipelineStep { async run(ctx: PipelineContext): Promise<StepResult> { const body = ctx.body.current(); if (!body) return { status: "skipped", reason: "no_body" }; + if (!isTextBody(body)) return { status: "skipped", reason: "unsupported_media_type" }; if (this.config.minInputChars !== undefined && body.content.length < this.config.minInputChars) return { status: "skipped", reason: "too_short" }; if (this.config.maxInputChars !== undefined && body.content.length > this.config.maxInputChars) @@ -61,7 +65,10 @@ export class LlmPassStep implements PipelineStep { ); const text = result.text.trim(); if (!text) return { status: "failed", reason: "empty_response" }; - return { status: "ok", effects: { body: { content: text, title: body.title } } }; + return { + status: "ok", + effects: { body: { kind: "text", content: text, mediaType: body.mediaType, title: body.title } } + }; } catch (error) { return classifyLlmPassFailure(error); } diff --git a/src/builtins/pipeline-steps/load-source/load-source-step.ts b/src/builtins/pipeline-steps/load-source/load-source-step.ts index 0fb3607..2a3717c 100644 --- a/src/builtins/pipeline-steps/load-source/load-source-step.ts +++ b/src/builtins/pipeline-steps/load-source/load-source-step.ts @@ -1,9 +1,10 @@ /** - * Loads the initial markdown body for a configured pipeline run. + * Loads the initial body for a configured pipeline run. * * Provider-specific upstream failures are classified here, not in core. The - * step emits the first body version and optional title diagnostics, or skips if - * an earlier step already produced a body. + * step emits the first body version (text or binary, carrying the provider's + * media type) and optional title diagnostics, or skips if an earlier step + * already produced a body. */ import { UpstreamError, isAbortError } from "../../../shared/errors.js"; @@ -13,23 +14,38 @@ import type { PipelineContext } from "../../../contracts/pipeline/context.js"; import type { PipelineStep, StepResult } from "../../../contracts/pipeline/step.js"; import type { Logger } from "../../../shared/logger.js"; -/** Fetches source markdown and writes the initial pipeline body. */ +/** Fetches source content and writes the initial pipeline body. */ export class LoadSourceStep implements PipelineStep { /** Creates a load-source step instance. */ constructor(private readonly deps: { readonly sourceProvider: SourceProvider; readonly logger: Logger }) {} - /** Loads markdown for the requested URL unless a body already exists. */ + /** Loads content for the requested URL unless a body already exists. */ async run(ctx: PipelineContext): Promise<StepResult> { if (ctx.body.current()) return { status: "skipped", reason: "body_present" }; try { const document = await this.deps.sourceProvider.load(ctx.input.url, { signal: ctx.signal }); - const content = document.content.trim(); - if (!content) return { status: "failed", reason: "empty" }; + + if (document.kind === "text") { + const content = document.content.trim(); + if (!content) return { status: "failed", reason: "empty" }; + + return { + status: document.truncated ? "degraded" : "ok", + reason: document.truncated ? "truncated" : undefined, + diagnostics: document.title ? { attributes: { title: document.title } } : undefined, + effects: { body: { kind: "text", content, mediaType: document.mediaType, title: document.title } } + }; + } + + if (document.bytes.byteLength === 0) return { status: "failed", reason: "empty" }; + return { - status: "ok", - diagnostics: document.title ? { attributes: { title: document.title } } : undefined, - effects: { body: { content, title: document.title } } + status: document.truncated ? "degraded" : "ok", + reason: document.truncated ? "truncated" : undefined, + effects: { + body: { kind: "binary", bytes: document.bytes, mediaType: document.mediaType, title: document.title } + } }; } catch (error) { return classifySourceLoadFailure(error); diff --git a/src/builtins/pipeline-steps/truncate/truncate-step.ts b/src/builtins/pipeline-steps/truncate/truncate-step.ts index 653dcfd..053b60c 100644 --- a/src/builtins/pipeline-steps/truncate/truncate-step.ts +++ b/src/builtins/pipeline-steps/truncate/truncate-step.ts @@ -1,12 +1,12 @@ /** - * Applies the final markdown body size budget for a pipeline run. + * Applies the final body size budget for a pipeline run. * * The step preserves the current title and writes a new body version only when - * content exceeds the configured target. The truncation suffix is counted inside - * the target budget. + * content exceeds the configured target. Skips when the current body is binary. + * The truncation suffix is counted inside the target budget. */ -import type { PipelineContext } from "../../../contracts/pipeline/context.js"; +import { isTextBody, type PipelineContext } from "../../../contracts/pipeline/context.js"; import type { PipelineStep, StepResult } from "../../../contracts/pipeline/step.js"; import type { Logger } from "../../../shared/logger.js"; import type { TruncateStepOptions } from "./truncate-step-config.js"; @@ -25,6 +25,7 @@ export class TruncateStep implements PipelineStep { async run(ctx: PipelineContext): Promise<StepResult> { const body = ctx.body.current(); if (!body) return { status: "skipped", reason: "no_body" }; + if (!isTextBody(body)) return { status: "skipped", reason: "unsupported_media_type" }; const inputChars = body.content.length; if (this.config.targetChars === 0 || inputChars <= this.config.targetChars) return { status: "skipped", reason: "under_target" }; @@ -32,7 +33,7 @@ export class TruncateStep implements PipelineStep { const truncated = truncateToBudget(body.content, this.config.targetChars); return { status: "ok", - effects: { body: { content: truncated, title: body.title } } + effects: { body: { kind: "text", content: truncated, mediaType: body.mediaType, title: body.title } } }; } } diff --git a/src/builtins/pipeline-steps/verify-urls/verify-urls-step.ts b/src/builtins/pipeline-steps/verify-urls/verify-urls-step.ts index f60eac9..cb3d9a9 100644 --- a/src/builtins/pipeline-steps/verify-urls/verify-urls-step.ts +++ b/src/builtins/pipeline-steps/verify-urls/verify-urls-step.ts @@ -11,7 +11,7 @@ import { collectCanonicalUrlCounts } from "../../../shared/markdown-urls.js"; -import type { PipelineContext } from "../../../contracts/pipeline/context.js"; +import { isTextBody, type PipelineContext } from "../../../contracts/pipeline/context.js"; import type { ChildReportNode, StepDiagnostics } from "../../../contracts/pipeline/diagnostics.js"; import type { PipelineStep, StepResult } from "../../../contracts/pipeline/step.js"; import type { Logger } from "../../../shared/logger.js"; @@ -35,6 +35,7 @@ export class VerifyUrlsStep implements PipelineStep { async run(ctx: PipelineContext): Promise<StepResult> { const body = ctx.body.current(); if (!body) return { status: "skipped", reason: "no_body" }; + if (!isTextBody(body)) return { status: "skipped", reason: "unsupported_media_type" }; const inventory = readInventory(ctx, this.options.artifact); if (!inventory) return { status: "skipped", reason: "no_inventory" }; @@ -82,7 +83,21 @@ export class VerifyUrlsStep implements PipelineStep { return { status: "degraded", reason: "hallucinated_urls", - effects: { body: { content: rollbackTarget.content, title: rollbackTarget.title } }, + effects: { + body: isTextBody(rollbackTarget) + ? { + kind: "text", + content: rollbackTarget.content, + mediaType: rollbackTarget.mediaType, + title: rollbackTarget.title + } + : { + kind: "binary", + bytes: rollbackTarget.bytes, + mediaType: rollbackTarget.mediaType, + title: rollbackTarget.title + } + }, diagnostics }; } diff --git a/src/builtins/source-providers/docling/docling-provider-config.ts b/src/builtins/source-providers/docling/docling-provider-config.ts new file mode 100644 index 0000000..2bc9e31 --- /dev/null +++ b/src/builtins/source-providers/docling/docling-provider-config.ts @@ -0,0 +1,28 @@ +/** + * Parses YAML configuration for the built-in Docling source provider. + * + * Environment substitution runs before this schema, so boolean fields use + * shared preprocessors to treat blank placeholders as omitted values. + */ + +import { z } from "zod"; + +import { booleanStringAsBooleanOrUndefined, emptyStringAsUndefined } from "../../../shared/config-coercion.js"; + +const doclingConfigSchema = z + .object({ + baseUrl: z.string().min(1, "docling baseUrl is required").url("docling baseUrl must be a valid URL"), + apiKey: z.string().default(""), + output: z.preprocess(emptyStringAsUndefined, z.enum(["markdown", "html"]).default("markdown")), + doOcr: z.preprocess(booleanStringAsBooleanOrUndefined, z.boolean().default(true)), + tableMode: z.enum(["fast", "accurate"]).default("accurate") + }) + .strict(); + +/** Represents parsed Docling provider configuration. */ +export type DoclingConfig = z.infer<typeof doclingConfigSchema>; + +/** Parses Docling provider configuration from YAML. */ +export function parseDoclingConfig(raw: unknown): DoclingConfig { + return Object.freeze(doclingConfigSchema.parse(raw)); +} diff --git a/src/builtins/source-providers/docling/docling-provider-descriptor.ts b/src/builtins/source-providers/docling/docling-provider-descriptor.ts new file mode 100644 index 0000000..0415575 --- /dev/null +++ b/src/builtins/source-providers/docling/docling-provider-descriptor.ts @@ -0,0 +1,25 @@ +/** + * Exposes the Docling source provider descriptor to engine descriptor bundles. + * + * The descriptor resolves the host HTTP fetch tool at construction time and + * returns a behavior-only provider instance; configured identity is added later + * by engine registry construction. + */ + +import { httpFetchKey } from "../../../contracts/host/host-tools.js"; +import { parseDoclingConfig } from "./docling-provider-config.js"; +import { DoclingProvider } from "./docling-provider.js"; + +import type { SourceProviderDescriptor } from "../../../contracts/extensions/source-provider.js"; +import type { DoclingConfig } from "./docling-provider-config.js"; + +/** Defines the built-in Docling provider type. */ +export const doclingProviderDescriptor = { + type: "docling", + parseConfig: parseDoclingConfig, + create: args => + new DoclingProvider(args.config, { + httpFetch: args.deps.tools.require(httpFetchKey), + logger: args.deps.logger + }) +} satisfies SourceProviderDescriptor<DoclingConfig>; diff --git a/src/builtins/source-providers/docling/docling-provider.ts b/src/builtins/source-providers/docling/docling-provider.ts new file mode 100644 index 0000000..5f44717 --- /dev/null +++ b/src/builtins/source-providers/docling/docling-provider.ts @@ -0,0 +1,133 @@ +/** + * Implements typed text loading through Docling Serve's sync convert endpoint. + * + * Provider responses are untrusted external content. Upstream HTTP, JSON parse, + * response-shape, empty-body, and network failures are translated to + * `UpstreamError`; its constructor owns the sanitizeUpstreamCode security + * boundary before failures reach step diagnostics or logs. + */ + +import { z } from "zod"; + +import { UpstreamError, isAbortError } from "../../../shared/errors.js"; +import { mediaTypes } from "../../../shared/media-types.js"; +import { joinUrl } from "../../../shared/urls.js"; + +import type { SourceDocument, SourceProvider } from "../../../contracts/extensions/source-provider.js"; +import type { Logger } from "../../../shared/logger.js"; +import type { DoclingConfig } from "./docling-provider-config.js"; + +const acceptedStatuses = new Set(["success", "partial_success"]); + +const doclingResponseSchema = z + .object({ + document: z + .object({ + md_content: z.string().nullable().optional(), + html_content: z.string().nullable().optional(), + json_content: z + .object({ + name: z.string().optional() + }) + .passthrough() + .nullable() + .optional() + }) + .passthrough() + .optional(), + status: z.string().optional(), + errors: z + .array( + z + .object({ + error_message: z.string().optional() + }) + .passthrough() + ) + .optional() + }) + .passthrough(); + +/** Implements the Docling source provider using the sync convert endpoint. */ +export class DoclingProvider implements SourceProvider { + /** Creates a Docling provider. */ + constructor( + private readonly config: DoclingConfig, + private readonly deps: { readonly httpFetch: typeof globalThis.fetch; readonly logger: Logger } + ) {} + + /** Loads a URL through Docling and returns configured text content. */ + async load(url: string, opts: { readonly signal: AbortSignal }): Promise<SourceDocument> { + try { + const response = await this.deps.httpFetch(joinUrl(this.config.baseUrl, "/v1/convert/source"), { + method: "POST", + signal: opts.signal, + headers: this.headers(), + body: JSON.stringify(this.buildRequestBody(url)) + }); + + const raw: unknown = await response.json().catch((error: unknown) => { + if (isAbortError(error)) throw error; + throw new UpstreamError("Docling response was not valid JSON", "parse_error", { + upstreamStatus: response.status, + cause: error + }); + }); + const parsed = doclingResponseSchema.safeParse(raw); + if (!parsed.success) + throw new UpstreamError("Docling response shape was not recognized", "parse_error", { + upstreamStatus: response.status, + cause: parsed.error.flatten() + }); + + const data = parsed.data; + const status = data.status ?? (response.ok ? "success" : "failure"); + + if (!response.ok || !acceptedStatuses.has(status)) { + const errors = data.errors?.map(e => e.error_message).filter(Boolean) ?? []; + const message = errors.length > 0 ? errors.join("; ") : `Docling returned HTTP ${response.status}`; + throw new UpstreamError(message, status === "failure" ? status : `http_${response.status}`, { + upstreamStatus: response.status || 502, + cause: raw + }); + } + + const doc = data.document; + const rawContent = this.config.output === "html" ? doc?.html_content : doc?.md_content; + const content = rawContent?.trim() ?? ""; + if (!content) + throw new UpstreamError(`Docling returned empty ${this.config.output}`, "empty", { + upstreamStatus: response.status + }); + + const name = data.document?.json_content?.name; + const title = typeof name === "string" && name.length > 0 ? name : undefined; + const mediaType: string = this.config.output === "html" ? mediaTypes.html : mediaTypes.markdown; + + return { kind: "text", content, mediaType, title }; + } catch (error) { + if (error instanceof UpstreamError || isAbortError(error)) throw error; + throw new UpstreamError(error instanceof Error ? error.message : String(error), "network", { cause: error }); + } + } + + /** Builds the request body for the Docling sync convert endpoint. */ + private buildRequestBody(url: string): unknown { + return { + sources: [{ kind: "http", url }], + options: { + to_formats: this.config.output === "html" ? ["html", "json"] : ["md", "json"], + image_export_mode: "placeholder", + do_ocr: this.config.doOcr, + table_mode: this.config.tableMode + } + }; + } + + /** Builds request headers for the upstream Docling endpoint. */ + private headers(): HeadersInit { + const headers: Record<string, string> = { "content-type": "application/json" }; + if (this.config.apiKey) headers["x-api-key"] = this.config.apiKey; + return headers; + } +} diff --git a/src/builtins/source-providers/firecrawl/firecrawl-provider-config.ts b/src/builtins/source-providers/firecrawl/firecrawl-provider-config.ts index 637d6c5..768cfbe 100644 --- a/src/builtins/source-providers/firecrawl/firecrawl-provider-config.ts +++ b/src/builtins/source-providers/firecrawl/firecrawl-provider-config.ts @@ -7,15 +7,16 @@ import { z } from "zod"; -import { booleanStringAsBooleanOrUndefined, emptyStringAsUndefined } from "../../../shared/config-coercion.js"; +import { booleanStringAsBooleanOrUndefined } from "../../../shared/config-coercion.js"; const firecrawlConfigSchema = z .object({ - baseUrl: z.string().url(), + baseUrl: z.string().min(1, "firecrawl baseUrl is required").url("firecrawl baseUrl must be a valid URL"), apiKey: z.string().default(""), + output: z.enum(["markdown", "html", "rawHtml"]).default("markdown"), onlyMainContent: z.preprocess(booleanStringAsBooleanOrUndefined, z.boolean().default(true)), - formats: z.array(z.string().min(1)).default(["markdown"]), - maxAge: z.preprocess(emptyStringAsUndefined, z.coerce.number().int().nonnegative().default(0)) + stripBase64Images: z.preprocess(booleanStringAsBooleanOrUndefined, z.boolean().default(true)), + parsePdf: z.preprocess(booleanStringAsBooleanOrUndefined, z.boolean().default(true)) }) .strict(); diff --git a/src/builtins/source-providers/firecrawl/firecrawl-provider.ts b/src/builtins/source-providers/firecrawl/firecrawl-provider.ts index d2fab58..9c595c3 100644 --- a/src/builtins/source-providers/firecrawl/firecrawl-provider.ts +++ b/src/builtins/source-providers/firecrawl/firecrawl-provider.ts @@ -1,5 +1,5 @@ /** - * Implements URL-to-markdown loading through Firecrawl's v2 scrape endpoint. + * Implements typed text loading through Firecrawl's v2 scrape endpoint. * * Provider responses are untrusted external content. Upstream HTTP, JSON parse, * response-shape, empty-body, and network failures are translated to @@ -10,6 +10,7 @@ import { z } from "zod"; import { UpstreamError, isAbortError } from "../../../shared/errors.js"; +import { mediaTypes } from "../../../shared/media-types.js"; import { joinUrl } from "../../../shared/urls.js"; import type { SourceDocument, SourceProvider } from "../../../contracts/extensions/source-provider.js"; @@ -24,11 +25,15 @@ const firecrawlResponseSchema = z data: z .object({ markdown: z.string().optional(), + html: z.string().optional(), + rawHtml: z.string().optional(), title: z.string().optional(), metadata: z.record(z.unknown()).optional() }) .optional(), markdown: z.string().optional(), + html: z.string().optional(), + rawHtml: z.string().optional(), title: z.string().optional(), metadata: z.record(z.unknown()).optional() }) @@ -42,7 +47,7 @@ export class FirecrawlProvider implements SourceProvider { private readonly deps: { readonly httpFetch: typeof globalThis.fetch; readonly logger: Logger } ) {} - /** Loads a URL through Firecrawl and returns markdown content. */ + /** Loads a URL through Firecrawl and returns configured text content. */ async load(url: string, opts: { readonly signal: AbortSignal }): Promise<SourceDocument> { try { const response = await this.deps.httpFetch(joinUrl(this.config.baseUrl, "/v2/scrape"), { @@ -51,9 +56,10 @@ export class FirecrawlProvider implements SourceProvider { headers: this.headers(), body: JSON.stringify({ url, - formats: this.config.formats, + formats: [this.config.output], onlyMainContent: this.config.onlyMainContent, - maxAge: this.config.maxAge + removeBase64Images: this.config.stripBase64Images, + parsers: this.config.parsePdf ? ["pdf"] : [] }) }); @@ -80,11 +86,15 @@ export class FirecrawlProvider implements SourceProvider { const data = parsed.data.data ?? parsed.data; const metadata = data.metadata ?? {}; - const content = data.markdown?.trim() ?? ""; + const contentField = data[this.config.output] as string | undefined; + const content = contentField?.trim() ?? ""; if (!content) - throw new UpstreamError("Firecrawl returned empty markdown", "empty", { upstreamStatus: response.status }); + throw new UpstreamError(`Firecrawl returned empty ${this.config.output}`, "empty", { + upstreamStatus: response.status + }); const title = data.title ?? (typeof metadata.title === "string" ? metadata.title : undefined); - return { content, title }; + const mediaType: string = this.config.output === "markdown" ? mediaTypes.markdown : mediaTypes.html; + return { kind: "text", content, mediaType, title }; } catch (error) { if (error instanceof UpstreamError || isAbortError(error)) throw error; throw new UpstreamError(error instanceof Error ? error.message : String(error), "network", { cause: error }); diff --git a/src/builtins/source-providers/http/http-provider-config.ts b/src/builtins/source-providers/http/http-provider-config.ts new file mode 100644 index 0000000..80d9e98 --- /dev/null +++ b/src/builtins/source-providers/http/http-provider-config.ts @@ -0,0 +1,31 @@ +/** + * Parses YAML configuration for the built-in HTTP source provider. + * + * Environment substitution runs before this schema, so numeric and boolean + * fields use shared preprocessors to treat blank placeholders as omitted values. + * + * The HTTP provider is a testing-only fallback that fetches the input URL + * directly with no SSRF protection. See the security caveat in CUSTOMIZATION.md. + */ + +import { z } from "zod"; + +import { booleanStringAsBooleanOrUndefined, emptyStringAsUndefined } from "../../../shared/config-coercion.js"; + +const httpConfigSchema = z + .object({ + userAgent: z + .string() + .default("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"), + maxBytes: z.preprocess(emptyStringAsUndefined, z.coerce.number().int().positive().default(5_000_000)), + titleFromHtml: z.preprocess(booleanStringAsBooleanOrUndefined, z.boolean().default(true)) + }) + .strict(); + +/** Represents parsed HTTP provider configuration. */ +export type HttpConfig = z.infer<typeof httpConfigSchema>; + +/** Parses HTTP provider configuration from YAML. */ +export function parseHttpConfig(raw: unknown): HttpConfig { + return Object.freeze(httpConfigSchema.parse(raw)); +} diff --git a/src/builtins/source-providers/http/http-provider-descriptor.ts b/src/builtins/source-providers/http/http-provider-descriptor.ts new file mode 100644 index 0000000..2be3652 --- /dev/null +++ b/src/builtins/source-providers/http/http-provider-descriptor.ts @@ -0,0 +1,27 @@ +/** + * Exposes the HTTP source provider descriptor to engine descriptor bundles. + * + * The descriptor resolves the host HTTP fetch tool at construction time and + * returns a behavior-only provider instance; configured identity is added later + * by engine registry construction. + * + * Security: this provider has no SSRF protection — see http-provider.ts + */ + +import { httpFetchKey } from "../../../contracts/host/host-tools.js"; +import { parseHttpConfig } from "./http-provider-config.js"; +import { HttpProvider } from "./http-provider.js"; + +import type { SourceProviderDescriptor } from "../../../contracts/extensions/source-provider.js"; +import type { HttpConfig } from "./http-provider-config.js"; + +/** Defines the built-in HTTP provider type. */ +export const httpProviderDescriptor = { + type: "http", + parseConfig: parseHttpConfig, + create: args => + new HttpProvider(args.config, { + httpFetch: args.deps.tools.require(httpFetchKey), + logger: args.deps.logger + }) +} satisfies SourceProviderDescriptor<HttpConfig>; diff --git a/src/builtins/source-providers/http/http-provider.ts b/src/builtins/source-providers/http/http-provider.ts new file mode 100644 index 0000000..56cf275 --- /dev/null +++ b/src/builtins/source-providers/http/http-provider.ts @@ -0,0 +1,149 @@ +/** + * Implements URL-to-content loading through Node-native HTTP fetch. + * + * This provider fetches the input URL directly and returns the response body + * as a content-typed `SourceDocument`. Text-like responses (determined by + * `isTextLike` and the NUL-byte sniff) return the text arm; everything else + * returns the binary arm. + * + * Text declared as HTML with embedded NUL bytes is treated as + * `application/octet-stream` binary rather than decoded as text, preventing + * lossy byte-to-character conversion from reaching downstream steps. + * + * - NO SSRF protection — the request goes wherever the input URL points. + * - NO JavaScript rendering — returns server-returned source HTML only. + * + * Security: the host adapter's parseHttpUrl guards protocol (http/https only) + * and absolute form, but does NOT filter private IPs or metadata endpoints. + * This is accepted as a testing tool; do not deploy as a user-facing provider. + * + * Provider responses are untrusted external content. Upstream HTTP, empty-body, + * and network failures are translated to `UpstreamError`. Oversized bodies are + * capped at maxBytes during a streaming read and flagged as truncated. + */ + +import { UpstreamError, isAbortError } from "../../../shared/errors.js"; +import { isTextLike, mediaTypes } from "../../../shared/media-types.js"; + +import type { SourceDocument, SourceProvider } from "../../../contracts/extensions/source-provider.js"; +import type { Logger } from "../../../shared/logger.js"; +import type { HttpConfig } from "./http-provider-config.js"; + +/** Extracts the first <title> tag content from raw HTML. */ +function extractTitle(html: string): string | undefined { + const match = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html); + if (!match) return undefined; + const title = match[1].replace(/\s+/g, " ").trim(); + return title.length > 0 ? title : undefined; +} + +/** Carries the bytes read from a response body and whether the byte cap forced truncation. */ +interface LimitedBody { + readonly bytes: Uint8Array; + readonly truncated: boolean; +} + +/** Reads up to maxBytes from a fetch response body, cancelling the stream when the cap is reached. */ +async function readLimitedBody(response: Response, maxBytes: number): Promise<LimitedBody> { + if (!response.body) { + const full = new Uint8Array(await response.arrayBuffer()); + if (full.byteLength > maxBytes) return { bytes: full.subarray(0, maxBytes), truncated: true }; + return { bytes: full, truncated: false }; + } + + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let totalBytes = 0; + let truncated = false; + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + if (!value) continue; + + const needed = maxBytes - totalBytes; + if (needed <= 0) { + truncated = true; + await reader.cancel(); + break; + } + + if (value.byteLength <= needed) { + chunks.push(value); + totalBytes += value.byteLength; + } else { + chunks.push(new Uint8Array(value.buffer, value.byteOffset, needed)); + totalBytes = maxBytes; + truncated = true; + await reader.cancel(); + break; + } + } + } catch (error: unknown) { + if (isAbortError(error)) throw error; + throw new UpstreamError("HTTP fetch response body read failed", "network", { cause: error }); + } finally { + reader.releaseLock(); + } + + const output = new Uint8Array(totalBytes); + let offset = 0; + for (const chunk of chunks) { + output.set(chunk, offset); + offset += chunk.byteLength; + } + return { bytes: output, truncated }; +} + +/** Implements the HTTP source provider using native fetch. */ +export class HttpProvider implements SourceProvider { + /** Creates an HTTP provider. */ + constructor( + private readonly config: HttpConfig, + private readonly deps: { readonly httpFetch: typeof globalThis.fetch; readonly logger: Logger } + ) {} + + /** Loads a URL through native HTTP fetch and returns a typed source document. */ + async load(url: string, opts: { readonly signal: AbortSignal }): Promise<SourceDocument> { + try { + const response = await this.deps.httpFetch(url, { + signal: opts.signal, + redirect: "follow", + headers: { "user-agent": this.config.userAgent } + }); + + if (!response.ok) { + throw new UpstreamError(`HTTP fetch returned HTTP ${response.status}`, `http_${response.status}`, { + upstreamStatus: response.status || 502, + cause: undefined + }); + } + + const reported = (response.headers.get("content-type") ?? "").split(";")[0].trim().toLowerCase(); + const mediaType = reported || mediaTypes.plainText; + + const limited = await readLimitedBody(response, this.config.maxBytes); + + if (isTextLike(mediaType) && !limited.bytes.includes(0)) { + const html = new TextDecoder().decode(limited.bytes); + const content = html.trim(); + + if (!content) { + throw new UpstreamError("HTTP fetch returned empty body", "empty", { + upstreamStatus: response.status + }); + } + + const title = this.config.titleFromHtml ? extractTitle(html) : undefined; + return { kind: "text", content, mediaType, title, truncated: limited.truncated }; + } + + const binaryMediaType = limited.bytes.includes(0) && isTextLike(mediaType) ? mediaTypes.octetStream : mediaType; + return { kind: "binary", bytes: limited.bytes, mediaType: binaryMediaType, truncated: limited.truncated }; + } catch (error) { + if (error instanceof UpstreamError || isAbortError(error)) throw error; + throw new UpstreamError(error instanceof Error ? error.message : String(error), "network", { cause: error }); + } + } +} diff --git a/src/bundles/default-engine-descriptors.ts b/src/bundles/default-engine-descriptors.ts index 1c863c2..d0b48a8 100644 --- a/src/bundles/default-engine-descriptors.ts +++ b/src/bundles/default-engine-descriptors.ts @@ -14,7 +14,9 @@ import { llmPassStepDescriptor } from "../builtins/pipeline-steps/llm-pass/llm-p import { loadSourceStepDescriptor } from "../builtins/pipeline-steps/load-source/load-source-step-descriptor.js"; import { truncateStepDescriptor } from "../builtins/pipeline-steps/truncate/truncate-step-descriptor.js"; import { verifyUrlsStepDescriptor } from "../builtins/pipeline-steps/verify-urls/verify-urls-step-descriptor.js"; +import { httpProviderDescriptor } from "../builtins/source-providers/http/http-provider-descriptor.js"; import { firecrawlProviderDescriptor } from "../builtins/source-providers/firecrawl/firecrawl-provider-descriptor.js"; +import { doclingProviderDescriptor } from "../builtins/source-providers/docling/docling-provider-descriptor.js"; import { createDescriptorRecord } from "../shared/descriptors.js"; import type { LlmProviderDescriptor } from "../contracts/extensions/llm-provider.js"; @@ -32,7 +34,10 @@ export interface EngineDescriptorBundle { /** Bundles the built-in engine descriptors without HTTP adapter descriptors. */ export const DEFAULT_ENGINE_DESCRIPTOR_BUNDLE: EngineDescriptorBundle = Object.freeze({ - sourceProviders: createDescriptorRecord([firecrawlProviderDescriptor], descriptor => descriptor.type), + sourceProviders: createDescriptorRecord( + [httpProviderDescriptor, firecrawlProviderDescriptor, doclingProviderDescriptor], + descriptor => descriptor.type + ), llmProviders: createDescriptorRecord([openAiChatProviderDescriptor], descriptor => descriptor.type), outputRenderers: createDescriptorRecord( [debugXmlRendererDescriptor, passthroughRendererDescriptor], diff --git a/src/config/yaml/env-substitution.ts b/src/config/yaml/env-substitution.ts index 5ef1ce5..63f497e 100644 --- a/src/config/yaml/env-substitution.ts +++ b/src/config/yaml/env-substitution.ts @@ -4,24 +4,24 @@ * Substitution is string-only and runs before Zod parsing. Later schemas recover * numbers and booleans where fields define those types, while opaque payloads * such as provider `extraBody` preserve YAML values except for string leaves. + * + * A missing `${VAR}` with no `:-` default resolves to `""` so that the env layer + * never throws for unused config. Required-ness is owned by reachable built-in + * schemas, not by substitution. */ -import { ConfigurationError } from "../../shared/errors.js"; - /** Recursively substitutes environment placeholders in YAML scalar strings. */ -export function substituteEnv(value: unknown, env: NodeJS.ProcessEnv, path: ReadonlyArray<string> = []): unknown { - if (typeof value === "string") return substituteString(value, env, path.join(".")); - if (Array.isArray(value)) return value.map((item, index) => substituteEnv(item, env, [...path, String(index)])); +export function substituteEnv(value: unknown, env: NodeJS.ProcessEnv): unknown { + if (typeof value === "string") return substituteString(value, env); + if (Array.isArray(value)) return value.map((item, index) => substituteEnv(item, env)); if (value && typeof value === "object") { - return Object.fromEntries( - Object.entries(value).map(([key, child]) => [key, substituteEnv(child, env, [...path, key])]) - ); + return Object.fromEntries(Object.entries(value).map(([key, child]) => [key, substituteEnv(child, env)])); } return value; } /** Resolves one YAML string using the supported environment substitution grammar. */ -function substituteString(value: string, env: NodeJS.ProcessEnv, path: string): string { +function substituteString(value: string, env: NodeJS.ProcessEnv): string { const literalMarker = "\u0000LITERAL_ENV_OPEN\u0000"; const protectedValue = value.replace(/\$\$\{/g, literalMarker); const substituted = protectedValue.replace( @@ -29,8 +29,7 @@ function substituteString(value: string, env: NodeJS.ProcessEnv, path: string): (_match, key: string, _defaultExpr: string | undefined, fallback: string | undefined) => { const envValue = env[key]; if (fallback !== undefined) return envValue ? envValue : fallback; - if (envValue === undefined) throw new ConfigurationError(`${path}: required env ${key} not set`, "missing_env"); - return envValue; + return envValue ?? ""; } ); return substituted.replaceAll(literalMarker, "${"); diff --git a/src/config/yaml/yaml-app-config.ts b/src/config/yaml/yaml-app-config.ts index 8424bf8..921dc98 100644 --- a/src/config/yaml/yaml-app-config.ts +++ b/src/config/yaml/yaml-app-config.ts @@ -14,6 +14,7 @@ import { readYaml } from "./read-yaml.js"; import { rawYamlConfigSchema, type RawYamlConfig } from "./yaml-config.js"; import type { AppConfig } from "../app-config.js"; +import type { EngineConfig } from "../../engine/engine-config.js"; /** Loads, substitutes, validates, and translates one YAML configuration file. */ export async function loadYamlAppConfig(args: { @@ -30,12 +31,27 @@ export async function loadYamlAppConfig(args: { /** Translates parsed YAML into the app-level configuration envelope. */ export function yamlToAppConfig(yamlConfig: RawYamlConfig): AppConfig { + const referencedPipelines = new Set(Object.values(yamlConfig.httpAdapters).map(a => a.pipeline)); + const pipelines: Record<string, EngineConfig["pipelines"][string]> = {}; + for (const [name, p] of Object.entries(yamlConfig.pipelines)) { + if (referencedPipelines.has(name) && p.enabled === false) + throw new ConfigurationError( + `Pipeline '${name}' is referenced by an HTTP adapter but disabled (enabled: false)`, + "disabled_pipeline_referenced" + ); + pipelines[name] = { + outputRenderer: p.outputRenderer, + limiters: p.limiters, + steps: p.steps as EngineConfig["pipelines"][string]["steps"], + enabled: p.enabled === undefined ? referencedPipelines.has(name) : p.enabled + }; + } return { engineConfig: { sourceProviders: yamlConfig.sourceProviders, llmProviders: yamlConfig.llmProviders, outputRenderers: yamlConfig.outputRenderers, - pipelines: yamlConfig.pipelines + pipelines }, adapters: { http: yamlConfig.httpAdapters }, metadata: { schemaVersion: yamlConfig.schemaVersion } diff --git a/src/config/yaml/yaml-config.ts b/src/config/yaml/yaml-config.ts index 00ab707..4982c99 100644 --- a/src/config/yaml/yaml-config.ts +++ b/src/config/yaml/yaml-config.ts @@ -8,7 +8,7 @@ import { z } from "zod"; -import { emptyStringAsUndefined } from "../../shared/config-coercion.js"; +import { booleanStringAsBooleanOrUndefined, emptyStringAsUndefined } from "../../shared/config-coercion.js"; const providerEntrySchema = z .object({ @@ -45,6 +45,7 @@ export const rawYamlConfigSchema = z pipelines: z.record( z .object({ + enabled: z.preprocess(booleanStringAsBooleanOrUndefined, z.boolean().optional()), outputRenderer: z.string().min(1).default("passthrough"), limiters: z.record(z.preprocess(emptyStringAsUndefined, z.coerce.number().int().positive())).default({}), steps: z.array(stepSchema).default([]) diff --git a/src/contracts/extensions/llm-provider.ts b/src/contracts/extensions/llm-provider.ts index 86058e1..2688843 100644 --- a/src/contracts/extensions/llm-provider.ts +++ b/src/contracts/extensions/llm-provider.ts @@ -61,7 +61,7 @@ export interface LlmProviderCreateArgs<TConfig = unknown> { export interface LlmProviderDescriptor<TConfig = unknown> { readonly type: string; parseConfig(raw: unknown): TConfig; - create(args: LlmProviderCreateArgs<TConfig>): LlmProvider | Promise<LlmProvider>; + create(args: LlmProviderCreateArgs<TConfig>): LlmProvider; } /** Identifies the LLM provider registry extension service. */ diff --git a/src/contracts/extensions/output-renderer.ts b/src/contracts/extensions/output-renderer.ts index 763e8b1..c3b81c1 100644 --- a/src/contracts/extensions/output-renderer.ts +++ b/src/contracts/extensions/output-renderer.ts @@ -56,7 +56,7 @@ export interface OutputRendererCreateArgs<TConfig = unknown> { export interface OutputRendererDescriptor<TConfig = unknown> { readonly type: string; parseConfig(raw: unknown): TConfig; - create(args: OutputRendererCreateArgs<TConfig>): OutputRenderer | Promise<OutputRenderer>; + create(args: OutputRendererCreateArgs<TConfig>): OutputRenderer; } /** Identifies the output renderer registry extension service. */ diff --git a/src/contracts/extensions/source-provider.ts b/src/contracts/extensions/source-provider.ts index 078041c..825e5de 100644 --- a/src/contracts/extensions/source-provider.ts +++ b/src/contracts/extensions/source-provider.ts @@ -1,7 +1,7 @@ /** * Defines the source provider extension contract and registry service key. * - * Source providers turn an already validated URL into markdown. Provider + * Source providers turn an already validated URL into a typed document. Provider * implementations handle upstream response parsing and translate degradable * external failures into `UpstreamError`; steps decide how those failures affect * pipeline status. @@ -15,14 +15,25 @@ import type { NamedRegistry } from "./named-registry.js"; import type { ResolvedSourceProvider } from "./resolved-extension.js"; /** Represents a source URL document before pipeline processing. */ -export interface SourceDocument { - readonly content: string; - readonly title?: string; -} +export type SourceDocument = + | { + readonly kind: "text"; + readonly mediaType: string; + readonly content: string; + readonly title?: string; + readonly truncated?: boolean; + } + | { + readonly kind: "binary"; + readonly mediaType: string; + readonly bytes: Uint8Array; + readonly title?: string; + readonly truncated?: boolean; + }; -/** Defines the provider port for URL-to-markdown retrieval. */ +/** Defines the provider port for URL-to-document retrieval. */ export interface SourceProvider { - /** Loads markdown for one absolute HTTP(S) URL. */ + /** Loads a typed source document for one absolute HTTP(S) URL. */ load(url: string, opts: { readonly signal: AbortSignal }): Promise<SourceDocument>; } @@ -46,7 +57,7 @@ export interface SourceProviderCreateArgs<TConfig = unknown> { export interface SourceProviderDescriptor<TConfig = unknown> { readonly type: string; parseConfig(raw: unknown): TConfig; - create(args: SourceProviderCreateArgs<TConfig>): SourceProvider | Promise<SourceProvider>; + create(args: SourceProviderCreateArgs<TConfig>): SourceProvider; } /** Identifies the source provider registry extension service. */ diff --git a/src/contracts/pipeline/context.ts b/src/contracts/pipeline/context.ts index 039e3fe..2d37eac 100644 --- a/src/contracts/pipeline/context.ts +++ b/src/contracts/pipeline/context.ts @@ -18,15 +18,44 @@ import type { StepOutcome } from "./report.js"; */ export type ScalarValue = string | number | boolean; -/** Represents markdown body content and its optional title. */ -export interface BodyContent { - readonly content: string; +/** Fields shared by every body representation. */ +export interface BodyBase { readonly title?: string; } -/** Describes one version of the orchestrator-owned body. */ -export interface BodyVersion extends BodyContent { - readonly stepName: string; +/** A text body carried as a string. */ +export interface TextBody extends BodyBase { + readonly kind: "text"; + readonly mediaType: string; + readonly content: string; +} + +/** A binary body carried as bytes. */ +export interface BinaryBody extends BodyBase { + readonly kind: "binary"; + readonly mediaType: string; + readonly bytes: Uint8Array; +} + +/** The pipeline body in either representation. */ +export type BodyContent = TextBody | BinaryBody; + +/** One version of the orchestrator-owned body. */ +export type BodyVersion = BodyContent & { readonly stepName: string }; + +/** Narrows a body to its text representation. */ +export function isTextBody(body: BodyContent): body is TextBody { + return body.kind === "text"; +} + +/** Narrows a body to its binary representation. */ +export function isBinaryBody(body: BodyContent): body is BinaryBody { + return body.kind === "binary"; +} + +/** Returns the body length: characters for text, bytes for binary. */ +export function bodyLength(body: BodyContent): number { + return body.kind === "text" ? body.content.length : body.bytes.byteLength; } /** Carries the immutable caller input for one configured pipeline invocation. */ @@ -36,7 +65,7 @@ export interface PipelineInput { /** Exposes read-only access to orchestrator-owned body versions during step execution. */ export interface BodyView { - /** Returns the current markdown body, if any. */ + /** Returns the current content-typed body, if any. */ current(): BodyContent | undefined; /** Returns an immutable snapshot of all body versions. */ diff --git a/src/contracts/pipeline/report.ts b/src/contracts/pipeline/report.ts index 88eaf4d..6184765 100644 --- a/src/contracts/pipeline/report.ts +++ b/src/contracts/pipeline/report.ts @@ -23,8 +23,8 @@ export interface StepOutcome { readonly name: string; readonly type: string; readonly status: StepStatus; - readonly inputChars?: number; - readonly outputChars?: number; + readonly inputLength?: number; + readonly outputLength?: number; readonly reason?: string; } @@ -40,8 +40,8 @@ export interface PipelineReport { readonly url: string; readonly startedAt: number; readonly durationMs: number; - readonly initialChars: number; - readonly finalChars: number; + readonly initialLength: number; + readonly finalLength: number; readonly ratio?: number; readonly returned: string; readonly result: PipelineRunStatus; diff --git a/src/core/pipeline/body.ts b/src/core/pipeline/body.ts index 72afde5..9ac0192 100644 --- a/src/core/pipeline/body.ts +++ b/src/core/pipeline/body.ts @@ -14,17 +14,30 @@ export class BodyStore implements BodyView { /** Appends a new body version and makes it current. */ append(version: BodyVersion): void { - this.bodyVersions.push({ ...version }); + this.bodyVersions.push(cloneBodyVersion(version)); } - /** Returns the current markdown body, if any. */ + /** Returns the current content-typed body, if any. */ current(): BodyContent | undefined { const latest = this.bodyVersions.at(-1); - return latest ? { content: latest.content, title: latest.title } : undefined; + if (!latest) return undefined; + return cloneBodyContent(latest); } /** Returns an immutable snapshot of all body versions. */ versions(): ReadonlyArray<BodyVersion> { - return this.bodyVersions.map(version => ({ ...version })); + return this.bodyVersions.map(cloneBodyVersion); } } + +/** Copies one body while preserving the representation arm. */ +function cloneBodyContent(body: BodyContent): BodyContent { + if (body.kind === "text") + return { kind: "text", mediaType: body.mediaType, content: body.content, title: body.title }; + return { kind: "binary", mediaType: body.mediaType, bytes: new Uint8Array(body.bytes), title: body.title }; +} + +/** Copies one stored body version while preserving the step that produced it. */ +function cloneBodyVersion(version: BodyVersion): BodyVersion { + return { ...cloneBodyContent(version), stepName: version.stepName }; +} diff --git a/src/core/pipeline/effects.ts b/src/core/pipeline/effects.ts index 36cb425..3dbffd7 100644 --- a/src/core/pipeline/effects.ts +++ b/src/core/pipeline/effects.ts @@ -7,7 +7,7 @@ * report; `skipped` and `failed` results never mutate state. */ -import type { ScalarValue } from "../../contracts/pipeline/context.js"; +import { bodyLength, type ScalarValue } from "../../contracts/pipeline/context.js"; import type { StepResult } from "../../contracts/pipeline/step.js"; import type { BodyStore } from "./body.js"; @@ -20,7 +20,7 @@ export interface EffectState { /** Summarizes state changes applied after an ok or failed step result. */ export interface AppliedEffectsSummary { - readonly outputChars?: number; + readonly outputLength?: number; readonly wroteBody: boolean; } @@ -50,7 +50,7 @@ export function applyStepEffects(stepName: string, result: StepResult, state: Ef } return { - outputChars: result.effects.body?.content.length, + outputLength: result.effects.body ? bodyLength(result.effects.body) : undefined, wroteBody: Boolean(result.effects.body) }; } diff --git a/src/core/pipeline/orchestrator.ts b/src/core/pipeline/orchestrator.ts index 6be770a..e95ba0e 100644 --- a/src/core/pipeline/orchestrator.ts +++ b/src/core/pipeline/orchestrator.ts @@ -16,7 +16,13 @@ import { ReadonlyArtifactBag, ReadonlySignalBag } from "./context.js"; import { applyStepEffects } from "./effects.js"; import { finalizeReport } from "./report.js"; -import type { PipelineContext, PipelineInput, ScalarValue } from "../../contracts/pipeline/context.js"; +import { + bodyLength, + type PipelineContext, + type PipelineInput, + type ScalarValue +} from "../../contracts/pipeline/context.js"; + import type { PipelineRunResult, StepOutcome, StepReport } from "../../contracts/pipeline/report.js"; import type { StepResult } from "../../contracts/pipeline/step.js"; import type { Logger } from "../../shared/logger.js"; @@ -99,8 +105,8 @@ export class PipelineOrchestrator { pipeline: pipeline.name, duration_ms: durationMs, result: report.result, - final_chars: report.finalChars, - initial_chars: report.initialChars, + final_length: report.finalLength, + initial_length: report.initialLength, returned: report.returned }, "Pipeline finished." @@ -159,14 +165,15 @@ export class PipelineOrchestrator { startedAt: number ): Promise<void> { const { entry, index } = step; - const inputChars = state.body.current()?.content.length; + const current = state.body.current(); + const inputLength = current ? bodyLength(current) : undefined; const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), entry.timeoutSeconds * 1000); const stepStartedAt = this.clock.now(); let result: StepResult; this.logger.debug( - { pipeline: pipeline.name, step: entry.name, type: entry.type, step_index: index, input_chars: inputChars }, + { pipeline: pipeline.name, step: entry.name, type: entry.type, step_index: index, input_length: inputLength }, "Step starting..." ); @@ -195,7 +202,7 @@ export class PipelineOrchestrator { clearTimeout(timer); } - this.recordStepResult(step, pipeline, result, state, stepStartedAt, inputChars); + this.recordStepResult(step, pipeline, result, state, stepStartedAt, inputLength); } /** Applies a step result and appends its outcome and report records. */ @@ -205,7 +212,7 @@ export class PipelineOrchestrator { result: StepResult, state: RuntimeState, stepStartedAt: number, - inputChars = state.body.current()?.content.length + inputLength?: number ): void { const { entry, index } = step; const applied = applyStepEffects(entry.name, result, state); @@ -214,8 +221,8 @@ export class PipelineOrchestrator { type: entry.type, status: result.status, reason: result.reason, - inputChars, - outputChars: applied.outputChars + inputLength, + outputLength: applied.outputLength }; const durationMs = this.clock.now() - stepStartedAt; const report: StepReport = { ...outcome, startedAt: stepStartedAt, durationMs, diagnostics: result.diagnostics }; @@ -230,7 +237,7 @@ export class PipelineOrchestrator { status: result.status, reason: result.reason, duration_ms: durationMs, - output_chars: applied.outputChars, + output_length: applied.outputLength, upstream_code: result.diagnostics?.attributes?.["upstream_code"], upstream_status: result.diagnostics?.attributes?.["upstream_status"] }; diff --git a/src/core/pipeline/report.ts b/src/core/pipeline/report.ts index 2d63cb0..0a8b361 100644 --- a/src/core/pipeline/report.ts +++ b/src/core/pipeline/report.ts @@ -4,8 +4,12 @@ * The report rollup is intentionally derived after all effects are applied: a * failed or degraded step with a body yields `degraded`, a failed run without a * body yields `failed`, and an all-ok run with a body yields `ok`. + * + * A binary body that reaches the end of the pipeline without conversion produces + * a failed run with an `unconverted_binary` error. */ +import { bodyLength } from "../../contracts/pipeline/context.js"; import type { BodyContent } from "../../contracts/pipeline/context.js"; import type { PipelineReport, StepReport } from "../../contracts/pipeline/report.js"; import type { BodyStore } from "./body.js"; @@ -21,25 +25,28 @@ export function finalizeReport(args: { const versions = args.body.versions(); const first = versions[0]; const last = versions.at(-1); - const initialChars = first?.content.length ?? 0; - const finalChars = last?.content.length ?? 0; + const initialLength = first ? bodyLength(first) : 0; + const finalLength = last ? bodyLength(last) : 0; + const terminalBinary = last?.kind === "binary"; const degradedStep = args.steps.find(step => step.status === "failed" || step.status === "degraded"); - const result = finalChars === 0 ? "failed" : degradedStep ? "degraded" : "ok"; - const ratio = initialChars > 0 && finalChars > 0 ? Number((finalChars / initialChars).toFixed(3)) : undefined; + const result = finalLength === 0 || terminalBinary ? "failed" : degradedStep ? "degraded" : "ok"; + const sameKind = first?.kind === last?.kind; + const ratio = + sameKind && initialLength > 0 && finalLength > 0 ? Number((finalLength / initialLength).toFixed(3)) : undefined; return { url: args.url, startedAt: args.startedAt, durationMs: args.durationMs, - initialChars, - finalChars, + initialLength, + finalLength, ratio, returned: last?.stepName ?? "none", result, steps: args.steps, bodyProducedBy: findBodyProducer(versions, last), bodyChangedBy: last?.stepName, - error: result === "failed" ? describePipelineFailure(args.steps) : undefined + error: result === "failed" ? (describePipelineFailure(args.steps) ?? terminalBinaryError(last)) : undefined }; } @@ -49,11 +56,29 @@ function describePipelineFailure(steps: ReadonlyArray<StepReport>): string | und return failed ? `${failed.name}: ${failed.reason ?? "failed"}` : undefined; } -/** Finds the step that first produced the final body bytes. */ +/** Returns the error message when a binary body reaches the end of the pipeline. */ +function terminalBinaryError(last: BodyContent | undefined): string | undefined { + if (last?.kind === "binary") return `unconverted_binary: ${last.mediaType}`; + return undefined; +} + +/** Finds the step that first produced the final body, comparing by representation kind. */ function findBodyProducer( versions: ReturnType<BodyStore["versions"]>, last: BodyContent | undefined ): string | undefined { if (!last) return undefined; - return versions.find(version => version.content === last.content && version.title === last.title)?.stepName; + return versions.find(version => { + if (version.kind !== last.kind || version.mediaType !== last.mediaType || version.title !== last.title) + return false; + if (last.kind === "text" && version.kind === "text") return version.content === last.content; + if (last.kind === "binary" && version.kind === "binary") return bytesEqual(version.bytes, last.bytes); + return false; + })?.stepName; +} + +/** Compares binary payloads by value so copied body snapshots still match their producer. */ +function bytesEqual(left: Uint8Array, right: Uint8Array): boolean { + if (left.byteLength !== right.byteLength) return false; + return left.every((value, index) => value === right[index]); } diff --git a/src/core/pipeline/runner.ts b/src/core/pipeline/runner.ts index 5970987..7dbe62b 100644 --- a/src/core/pipeline/runner.ts +++ b/src/core/pipeline/runner.ts @@ -86,8 +86,8 @@ function synthesizeFailureReport(url: string, error: unknown): PipelineReport { url, startedAt, durationMs: 0, - initialChars: 0, - finalChars: 0, + initialLength: 0, + finalLength: 0, returned: "none", result: "failed", steps: [], diff --git a/src/engine/create-engine.ts b/src/engine/create-engine.ts index 2180240..35622f9 100644 --- a/src/engine/create-engine.ts +++ b/src/engine/create-engine.ts @@ -33,21 +33,21 @@ export async function createEngine(args: { readonly logger: Logger; }): Promise<EngineRuntime> { const serviceBuilder = createExtensionServicesBuilder(); - const sourceProviders = await buildSourceProviders( + const sourceProviders = buildSourceProviders( args.config.sourceProviders, args.tools, args.logger, args.descriptors.sourceProviders ); serviceBuilder.register(sourceProviderRegistryKey, sourceProviders); - const llmProviders = await buildLlmProviders( + const llmProviders = buildLlmProviders( args.config.llmProviders, args.tools, args.logger, args.descriptors.llmProviders ); serviceBuilder.register(llmProviderRegistryKey, llmProviders); - const outputRenderers = await buildOutputRenderers( + const outputRenderers = buildOutputRenderers( args.config.outputRenderers, args.tools, args.logger, @@ -63,6 +63,11 @@ export async function createEngine(args: { args.logger, args.descriptors.pipelineSteps ); + + logSkipped(args.logger, "source provider", Object.keys(args.config.sourceProviders), sourceProviders.builtNames()); + logSkipped(args.logger, "LLM provider", Object.keys(args.config.llmProviders), llmProviders.builtNames()); + logSkipped(args.logger, "output renderer", Object.keys(args.config.outputRenderers), outputRenderers.builtNames()); + const runner = new PipelineRunner( new PipelineOrchestrator({ logger: args.logger.child({ component: "orchestrator" }) }) ); @@ -71,16 +76,25 @@ export async function createEngine(args: { return createEngineRuntime({ registries, pipelines: pipelineInfos(args.config), handles }); } -/** Builds stable pipeline discovery info from engine config. */ +/** Builds stable pipeline discovery info from engine config (enabled only). */ function pipelineInfos(config: EngineConfig): ReadonlyArray<PipelineInfo> { - return Object.entries(config.pipelines).map(([name, pipeline]) => ({ - name, - outputRenderer: pipeline.outputRenderer, - steps: pipeline.steps.map(step => ({ - name: step.name, - type: step.type, - timeoutSeconds: step.timeoutSeconds, - concurrencyGroup: step.concurrencyGroup - })) - })); + return Object.entries(config.pipelines) + .filter(([, pipeline]) => pipeline.enabled) + .map(([name, pipeline]) => ({ + name, + outputRenderer: pipeline.outputRenderer, + steps: pipeline.steps.map(step => ({ + name: step.name, + type: step.type, + timeoutSeconds: step.timeoutSeconds, + concurrencyGroup: step.concurrencyGroup + })) + })); +} + +/** Logs defined-but-unreferenced leaves for visibility (D7). */ +function logSkipped(logger: Logger, label: string, defined: ReadonlyArray<string>, built: ReadonlySet<string>): void { + const skipped = defined.filter(name => !built.has(name)); + if (skipped.length > 0) + logger.info({ skipped }, `${label}s defined but not referenced by any active pipeline; skipped`); } diff --git a/src/engine/engine-config.ts b/src/engine/engine-config.ts index 6b4f06b..0535506 100644 --- a/src/engine/engine-config.ts +++ b/src/engine/engine-config.ts @@ -23,6 +23,7 @@ export interface EngineStepConfig { /** Configures one pipeline before runtime construction. */ export interface EnginePipelineConfig { + readonly enabled: boolean; readonly outputRenderer: string; readonly limiters: Readonly<Record<string, number>>; readonly steps: ReadonlyArray<EngineStepConfig>; diff --git a/src/engine/internal/builders/build-llm-providers.ts b/src/engine/internal/builders/build-llm-providers.ts index a146666..9da0d5e 100644 --- a/src/engine/internal/builders/build-llm-providers.ts +++ b/src/engine/internal/builders/build-llm-providers.ts @@ -6,7 +6,7 @@ * logger identity fields. */ -import { buildNamedRegistry } from "./build-named-registry.js"; +import { buildNamedRegistry, type BuiltNameTracking } from "./build-named-registry.js"; import type { LlmProviderDescriptor, LlmProviderRegistry } from "../../../contracts/extensions/llm-provider.js"; import type { HostTools } from "../../../contracts/host/host-tools.js"; @@ -14,12 +14,12 @@ import type { Logger } from "../../../shared/logger.js"; import type { EngineConfig } from "../../engine-config.js"; /** Constructs LLM providers from configured LLM provider entries. */ -export async function buildLlmProviders( +export function buildLlmProviders( rawProviders: EngineConfig["llmProviders"], tools: HostTools, logger: Logger, descriptors: Readonly<Record<string, LlmProviderDescriptor>> -): Promise<LlmProviderRegistry> { +): LlmProviderRegistry & BuiltNameTracking { return buildNamedRegistry({ rawEntries: rawProviders, descriptors, diff --git a/src/engine/internal/builders/build-named-registry.ts b/src/engine/internal/builders/build-named-registry.ts index 0caadff..be6afa2 100644 --- a/src/engine/internal/builders/build-named-registry.ts +++ b/src/engine/internal/builders/build-named-registry.ts @@ -22,11 +22,7 @@ interface RawDescriptorEntry { /** Minimal descriptor shape needed for registry construction. */ interface MinimalDescriptor<TInstance> { parseConfig(raw: unknown): unknown; - create(args: { - name: string; - config: unknown; - deps: { tools: HostTools; logger: Logger }; - }): TInstance | Promise<TInstance>; + create(args: { name: string; config: unknown; deps: { tools: HostTools; logger: Logger } }): TInstance; } /** Labels and log-field name for descriptor-driven registry construction. */ @@ -42,6 +38,11 @@ interface NamedRegistryLabels { readonly missingMessage: (name: string) => string; } +/** Built-name tracking for engine-internal skipped-leaf logging. */ +export interface BuiltNameTracking { + builtNames(): ReadonlySet<string>; +} + /** Inputs to a name-keyed registry build. */ interface BuildNamedRegistryOptions<TInstance, TResolved> { readonly rawEntries: Readonly<Record<string, RawDescriptorEntry>>; @@ -52,13 +53,15 @@ interface BuildNamedRegistryOptions<TInstance, TResolved> { readonly resolve: (id: { readonly name: string; readonly type: string }, instance: TInstance) => TResolved; } -/** Builds a name-keyed registry by instantiating each configured entry's descriptor. */ -export async function buildNamedRegistry<TInstance, TResolved>( +/** Builds a name-keyed registry with lazy (first-require) construction. */ +export function buildNamedRegistry<TInstance, TResolved>( options: BuildNamedRegistryOptions<TInstance, TResolved> -): Promise<NamedRegistry<TResolved>> { +): NamedRegistry<TResolved> & BuiltNameTracking { const { rawEntries, descriptors, tools, logger, labels, resolve } = options; - const resolved = new Map<string, TResolved>(); - for (const [name, raw] of Object.entries(rawEntries)) { + const cache = new Map<string, TResolved>(); + + function build(name: string): TResolved { + const raw = rawEntries[name]; const descriptor = descriptors[raw.type]; if (!descriptor) throw new ConfigurationError(labels.unknownTypeMessage(raw.type), labels.unknownTypeCode); let parsed: unknown; @@ -74,7 +77,7 @@ export async function buildNamedRegistry<TInstance, TResolved>( } let instance: TInstance; try { - instance = await descriptor.create({ + instance = descriptor.create({ name, config: parsed, deps: { tools, logger: logger.child({ [labels.logField]: name, type: raw.type }) } @@ -87,16 +90,25 @@ export async function buildNamedRegistry<TInstance, TResolved>( type: raw.type }); } - resolved.set(name, resolve({ name, type: raw.type }, instance)); + const resolved = resolve({ name, type: raw.type }, instance); + cache.set(name, resolved); + return resolved; } + return Object.freeze({ require(name: string): TResolved { - const entry = resolved.get(name); - if (!entry) throw new ConfigurationError(labels.missingMessage(name), labels.missingCode); - return entry; + if (cache.has(name)) return cache.get(name)!; + if (!Object.hasOwn(rawEntries, name)) + throw new ConfigurationError(labels.missingMessage(name), labels.missingCode); + return build(name); }, tryGet(name: string): TResolved | undefined { - return resolved.get(name); + if (cache.has(name)) return cache.get(name)!; + if (!Object.hasOwn(rawEntries, name)) return undefined; + return build(name); + }, + builtNames(): ReadonlySet<string> { + return new Set(cache.keys()); } }); } diff --git a/src/engine/internal/builders/build-output-renderers.ts b/src/engine/internal/builders/build-output-renderers.ts index 3deccdb..c8875bc 100644 --- a/src/engine/internal/builders/build-output-renderers.ts +++ b/src/engine/internal/builders/build-output-renderers.ts @@ -6,7 +6,7 @@ * logger identity fields. */ -import { buildNamedRegistry } from "./build-named-registry.js"; +import { buildNamedRegistry, type BuiltNameTracking } from "./build-named-registry.js"; import type { OutputRendererDescriptor, @@ -17,12 +17,12 @@ import type { Logger } from "../../../shared/logger.js"; import type { EngineConfig } from "../../engine-config.js"; /** Constructs output renderers from configured output renderer entries. */ -export async function buildOutputRenderers( +export function buildOutputRenderers( rawRenderers: EngineConfig["outputRenderers"], tools: HostTools, logger: Logger, descriptors: Readonly<Record<string, OutputRendererDescriptor>> -): Promise<OutputRendererRegistry> { +): OutputRendererRegistry & BuiltNameTracking { return buildNamedRegistry({ rawEntries: rawRenderers, descriptors, diff --git a/src/engine/internal/builders/build-pipelines.ts b/src/engine/internal/builders/build-pipelines.ts index 570db42..00d1667 100644 --- a/src/engine/internal/builders/build-pipelines.ts +++ b/src/engine/internal/builders/build-pipelines.ts @@ -29,6 +29,7 @@ export async function buildPipelines( const rendererRegistry = services.require(outputRendererRegistryKey); const pipelines = new Map<string, CompiledPipeline>(); for (const [pipelineName, pipeline] of Object.entries(rawPipelines)) { + if (!pipeline.enabled) continue; const renderer = rendererRegistry.tryGet(pipeline.outputRenderer); if (!renderer) throw new ConfigurationError( diff --git a/src/engine/internal/builders/build-source-providers.ts b/src/engine/internal/builders/build-source-providers.ts index 0633790..67216f5 100644 --- a/src/engine/internal/builders/build-source-providers.ts +++ b/src/engine/internal/builders/build-source-providers.ts @@ -6,7 +6,7 @@ * logger identity fields. */ -import { buildNamedRegistry } from "./build-named-registry.js"; +import { buildNamedRegistry, type BuiltNameTracking } from "./build-named-registry.js"; import type { SourceProviderDescriptor, @@ -17,12 +17,12 @@ import type { Logger } from "../../../shared/logger.js"; import type { EngineConfig } from "../../engine-config.js"; /** Constructs source providers from configured source provider entries. */ -export async function buildSourceProviders( +export function buildSourceProviders( rawProviders: EngineConfig["sourceProviders"], tools: HostTools, logger: Logger, descriptors: Readonly<Record<string, SourceProviderDescriptor>> -): Promise<SourceProviderRegistry> { +): SourceProviderRegistry & BuiltNameTracking { return buildNamedRegistry({ rawEntries: rawProviders, descriptors, diff --git a/src/main.ts b/src/main.ts index f63e163..13a3ef1 100644 --- a/src/main.ts +++ b/src/main.ts @@ -24,8 +24,14 @@ async function main(): Promise<void> { const envConfig = loadEnvConfig(); const logger = createLogger(envConfig); startupLogger = logger; - logger.info({ component: "server" }, "LLM Context Loader starting..."); + const loaded = await composeApp({ envConfig, env: process.env, logger, httpFetch: globalThis.fetch }); + if (process.argv.includes("--check")) { + logger.info({ component: "check" }, "Configuration valid. Exiting."); + process.exit(0); + } + + logger.info({ component: "server" }, "LLM Context Loader starting..."); const app = await buildHttpApp( loaded.adapters.http.map(({ adapter }) => adapter), logger diff --git a/src/shared/media-types.ts b/src/shared/media-types.ts new file mode 100644 index 0000000..b397a54 --- /dev/null +++ b/src/shared/media-types.ts @@ -0,0 +1,29 @@ +/** + * Common media types, named for use sites. + * The vocabulary is open: any `string` is a valid media type. + */ +export const mediaTypes = { + html: "text/html", + markdown: "text/markdown", + plainText: "text/plain", + json: "application/json", + xml: "application/xml", + xhtml: "application/xhtml+xml", + pdf: "application/pdf", + png: "image/png", + jpeg: "image/jpeg", + octetStream: "application/octet-stream" +} as const; + +/** Returns whether a media type can be decoded and carried as UTF-8 text. */ +export function isTextLike(mediaType: string): boolean { + const normalized = mediaType.trim().toLowerCase(); + return ( + normalized.startsWith("text/") || + normalized === mediaTypes.json || + normalized === mediaTypes.xml || + normalized === mediaTypes.xhtml || + normalized.endsWith("+json") || + normalized.endsWith("+xml") + ); +} diff --git a/tests/app/compose.test.ts b/tests/app/compose.test.ts index c2407fc..f6797b2 100644 --- a/tests/app/compose.test.ts +++ b/tests/app/compose.test.ts @@ -31,7 +31,7 @@ describe("composeApp", () => { expect(loaded.registries.llmProviders.require("default-llm").name).toBe("default-llm"); expect( loaded.registries.pipelines.find(pipeline => pipeline.name === "default")?.steps.map(entry => entry.name) - ).toEqual(["firecrawl", "clean", "truncate"]); + ).toEqual(["fetch", "clean", "truncate"]); expect(loaded.adapters.http.map(adapter => adapter.name)).toEqual(["open-webui"]); expect(logs.some(log => log.level === "warn" && log.message === "Adapter bearer auth is disabled.")).toBe(true); }); @@ -45,7 +45,7 @@ describe("composeApp", () => { logger: createTestLogger(), httpFetch: fetch }) - ).rejects.toThrow("required env FIRECRAWL_BASE_URL not set"); + ).rejects.toThrow("Invalid config for source provider 'default-firecrawl'"); const missingTemplate = await writeConfigFixture(defaultYaml(), false); await expect( @@ -74,7 +74,7 @@ describe("composeApp", () => { ).rejects.toThrow("Duplicate step name"); const unknownField = await writeConfigFixture( - defaultYaml().replace(" maxAge: 0", " maxAge: 0\n mystery: true") + defaultYaml().replace(" stripBase64Images: true", " stripBase64Images: true\n mystery: true") ); await expect( composeApp({ @@ -120,7 +120,7 @@ describe("composeApp", () => { const fixture = await writeConfigFixture( defaultYaml() .replace("onlyMainContent: true", "onlyMainContent: ${FIRECRAWL_ONLY_MAIN:-}") - .replace("maxAge: 0", "maxAge: ${FIRECRAWL_MAX_AGE:-}") + .replace("stripBase64Images: true", "stripBase64Images: ${FIRECRAWL_STRIP_IMAGES:-}") .replace("includeSkipped: true", "includeSkipped: ${DEBUG_XML_INCLUDE_SKIPPED:-}") .replace( " config:\n provider: default-llm", @@ -135,7 +135,7 @@ describe("composeApp", () => { env: { FIRECRAWL_BASE_URL: "https://firecrawl.example", FIRECRAWL_ONLY_MAIN: "false", - FIRECRAWL_MAX_AGE: "", + FIRECRAWL_STRIP_IMAGES: "", LLM_BASE_URL: "https://llm.example/v1", LLM_MODEL: "model", DEBUG_XML_INCLUDE_SKIPPED: "false", @@ -162,7 +162,7 @@ describe("composeApp", () => { .require("default-firecrawl") .provider.load("https://example.com", { signal: new AbortController().signal }); - expect(firecrawlRequestBody).toMatchObject({ onlyMainContent: false, maxAge: 0 }); + expect(firecrawlRequestBody).toMatchObject({ onlyMainContent: false, removeBase64Images: true }); }); it("rejects unknown registry references and invalid diagnostic names", async () => { @@ -248,9 +248,10 @@ function defaultYaml(): string { config: baseUrl: \${FIRECRAWL_BASE_URL} apiKey: \${FIRECRAWL_API_KEY:-} + output: markdown onlyMainContent: true - formats: [markdown] - maxAge: 0 + stripBase64Images: true + parsePdf: true llmProviders: default-llm: type: openai-chat @@ -273,7 +274,7 @@ pipelines: llm: \${LLM_CONCURRENCY:-1} steps: - type: load-source - name: firecrawl + name: fetch concurrencyGroup: source config: provider: default-firecrawl diff --git a/tests/app/runtime-builders.test.ts b/tests/app/runtime-builders.test.ts index aaf1dfd..2e922ee 100644 --- a/tests/app/runtime-builders.test.ts +++ b/tests/app/runtime-builders.test.ts @@ -26,7 +26,7 @@ describe("descriptor builders", () => { external: { type: "opaque_source_provider", config: { provider: "reserved-word", nested: { value: true } } } }; - const providers = await buildSourceProviders(rawProviders, createTestHostTools(), createTestLogger(), { + const providers = buildSourceProviders(rawProviders, createTestHostTools(), createTestLogger(), { opaque_source_provider: { type: "opaque_source_provider", parseConfig: raw => { @@ -37,8 +37,10 @@ describe("descriptor builders", () => { } satisfies SourceProviderDescriptor<unknown> }); + // Lazy: require triggers parse+create + const resolved = providers.require("external"); expect(parsedConfigs).toEqual([{ provider: "reserved-word", nested: { value: true } }]); - expect(providers.require("external").name).toBe("external"); + expect(resolved.name).toBe("external"); }); it("passes LLM provider config through LLM provider descriptors unchanged", async () => { @@ -47,7 +49,7 @@ describe("descriptor builders", () => { external: { type: "opaque_llm_provider", config: { model: "reserved-word", nested: { value: true } } } }; - const providers = await buildLlmProviders(rawProviders, createTestHostTools(), createTestLogger(), { + const providers = buildLlmProviders(rawProviders, createTestHostTools(), createTestLogger(), { opaque_llm_provider: { type: "opaque_llm_provider", parseConfig: raw => { @@ -58,8 +60,10 @@ describe("descriptor builders", () => { } satisfies LlmProviderDescriptor<unknown> }); + // Lazy: require triggers parse+create + const resolved = providers.require("external"); expect(parsedConfigs).toEqual([{ model: "reserved-word", nested: { value: true } }]); - expect(providers.require("external").name).toBe("external"); + expect(resolved.name).toBe("external"); }); it("passes step config through step descriptors unchanged", async () => { @@ -67,6 +71,7 @@ describe("descriptor builders", () => { let createdStep: PipelineStep | undefined; const rawPipelines: EngineConfig["pipelines"] = { default: { + enabled: true, outputRenderer: "test", limiters: { shared: 1 }, steps: [ @@ -166,27 +171,30 @@ describe("descriptor builders", () => { external: { type: "opaque_source_provider", config: {} } }; - await expect( - buildSourceProviders(rawProviders, createTestHostTools(), createTestLogger(), { - opaque_source_provider: { - type: "opaque_source_provider", - parseConfig: raw => raw, - create: () => { - throw cause; - } - } satisfies SourceProviderDescriptor<unknown> - }) - ).rejects.toMatchObject({ - code: "source_provider_create_failed", - details: { cause, name: "external", type: "opaque_source_provider" }, - cause + const providers = buildSourceProviders(rawProviders, createTestHostTools(), createTestLogger(), { + opaque_source_provider: { + type: "opaque_source_provider", + parseConfig: raw => raw, + create: () => { + throw cause; + } + } satisfies SourceProviderDescriptor<unknown> }); + + expect(() => providers.require("external")).toThrow( + expect.objectContaining({ + code: "source_provider_create_failed", + details: { cause, name: "external", type: "opaque_source_provider" }, + cause + }) + ); }); it("wraps pipeline step create failures as ConfigurationError", async () => { const cause = new Error("step factory exploded"); const rawPipelines: EngineConfig["pipelines"] = { default: { + enabled: true, outputRenderer: "test", limiters: {}, steps: [{ type: "opaque_step", name: "opaque", timeoutSeconds: 7, config: {} }] @@ -240,22 +248,22 @@ describe("descriptor builders", () => { external: { type: "opaque_source_provider", config: {} } }; - await expect( - buildSourceProviders(rawProviders, createTestHostTools(), createTestLogger(), { - opaque_source_provider: { - type: "opaque_source_provider", - parseConfig: raw => raw, - create: () => { - throw classified; - } - } satisfies SourceProviderDescriptor<unknown> - }) - ).rejects.toBe(classified); + const providers = buildSourceProviders(rawProviders, createTestHostTools(), createTestLogger(), { + opaque_source_provider: { + type: "opaque_source_provider", + parseConfig: raw => raw, + create: () => { + throw classified; + } + } satisfies SourceProviderDescriptor<unknown> + }); + + expect(() => providers.require("external")).toThrow(classified); }); }); function sourceProvider(): SourceProvider { - return { load: async () => ({ content: "source" }) }; + return { load: async () => ({ kind: "text", content: "source", mediaType: "text/markdown" }) }; } function resourceLoader() { diff --git a/tests/builtins/output-renderers/debug-xml/footer-serializer.test.ts b/tests/builtins/output-renderers/debug-xml/footer-serializer.test.ts index 0b91c9e..884db1b 100644 --- a/tests/builtins/output-renderers/debug-xml/footer-serializer.test.ts +++ b/tests/builtins/output-renderers/debug-xml/footer-serializer.test.ts @@ -11,8 +11,8 @@ describe("serializeFooter", () => { expect(footer).toContain('url="https://example.com/?a=1&b=2"'); expect(footer).toContain('ratio="1.000"'); - expect(footer).toContain('<firecrawl status="ok" duration_ms="2" output_chars="12" title="A & B"/>'); - expect(footer).toContain('<truncate status="skipped" reason="under_target" duration_ms="1" input_chars="12"/>'); + expect(footer).toContain('<firecrawl status="ok" duration_ms="2" output_length="12" title="A & B"/>'); + expect(footer).toContain('<truncate status="skipped" reason="under_target" duration_ms="1" input_length="12"/>'); }); it("can omit skipped step elements", () => { @@ -77,8 +77,8 @@ function makeReport(): PipelineReport { url: "https://example.com/?a=1&b=2", startedAt: 1, durationMs: 3, - initialChars: 12, - finalChars: 12, + initialLength: 12, + finalLength: 12, ratio: 1, returned: "firecrawl", result: "ok", @@ -91,7 +91,7 @@ function makeReport(): PipelineReport { status: "ok", startedAt: 1, durationMs: 2, - outputChars: 12, + outputLength: 12, diagnostics: { attributes: { title: "A & B" } } }, { @@ -101,7 +101,7 @@ function makeReport(): PipelineReport { reason: "under_target", startedAt: 3, durationMs: 1, - inputChars: 12 + inputLength: 12 } ] }; diff --git a/tests/builtins/output-renderers/output-renderers-conformance.test.ts b/tests/builtins/output-renderers/output-renderers-conformance.test.ts index 735377f..8e4341b 100644 --- a/tests/builtins/output-renderers/output-renderers-conformance.test.ts +++ b/tests/builtins/output-renderers/output-renderers-conformance.test.ts @@ -19,7 +19,9 @@ describe("built-in output renderers", () => { { logger: createTestLogger() } ); - const output = renderer.render(makeRenderInput({ body: { content: "hello" } })); + const output = renderer.render( + makeRenderInput({ body: { kind: "text", content: "hello", mediaType: "text/markdown" } }) + ); expect(output.markdown).toContain("hello\n\n<loader_info"); expect(output.markdown).toContain('result="ok"'); @@ -40,8 +42,8 @@ describe("built-in output renderers", () => { ...input.report, result: "failed", returned: "none", - initialChars: 0, - finalChars: 0, + initialLength: 0, + finalLength: 0, error: "source: timeout" } }); @@ -56,7 +58,7 @@ describe("built-in output renderers", () => { const output = renderer.render({ ...input, body: undefined, - report: { ...input.report, result: "failed", returned: "none", initialChars: 0, finalChars: 0 } + report: { ...input.report, result: "failed", returned: "none", initialLength: 0, finalLength: 0 } }); expect(output.markdown).toBe("Pipeline failed."); diff --git a/tests/builtins/pipeline-steps/capture-urls/capture-urls-step.test.ts b/tests/builtins/pipeline-steps/capture-urls/capture-urls-step.test.ts index 62d0036..ccc95ed 100644 --- a/tests/builtins/pipeline-steps/capture-urls/capture-urls-step.test.ts +++ b/tests/builtins/pipeline-steps/capture-urls/capture-urls-step.test.ts @@ -6,6 +6,14 @@ import { createTestLogger } from "../../../helpers/logger.js"; import { makeStepContext } from "../utils.js"; describe("CaptureUrlsStep", () => { + it("skips on binary body", async () => { + const step = new CaptureUrlsStep({ artifact: "trusted-urls" }, { logger: createTestLogger() }); + + const result = await step.run(makeStepContext({ body: { bytes: new Uint8Array([1, 2, 3]) } })); + + expect(result).toMatchObject({ status: "skipped", reason: "unsupported_media_type" }); + }); + it("skips when no body is present", async () => { const step = new CaptureUrlsStep({ artifact: "trusted-urls" }, { logger: createTestLogger() }); diff --git a/tests/builtins/pipeline-steps/llm-pass/llm-pass-step.test.ts b/tests/builtins/pipeline-steps/llm-pass/llm-pass-step.test.ts index d257896..aef91ef 100644 --- a/tests/builtins/pipeline-steps/llm-pass/llm-pass-step.test.ts +++ b/tests/builtins/pipeline-steps/llm-pass/llm-pass-step.test.ts @@ -9,6 +9,14 @@ import { createTestLogger } from "../../../helpers/logger.js"; import { makeStepContext } from "../utils.js"; describe("LlmPassStep", () => { + it("skips on binary body", async () => { + const step = makeStep(llm("ok")); + + const result = await step.run(makeStepContext({ body: { bytes: new Uint8Array([1, 2, 3]) } })); + + expect(result).toMatchObject({ status: "skipped", reason: "unsupported_media_type" }); + }); + it("skips when no body, too short, or too long", async () => { const step = makeStep(llm("ok")); @@ -36,7 +44,12 @@ describe("LlmPassStep", () => { const result = await step.run(makeStepContext({ body: { content: "source text", title: "Title" } })); expect(result.status).toBe("ok"); - expect(result.effects?.body).toEqual({ content: "cleaned", title: "Title" }); + expect(result.effects?.body).toEqual({ + kind: "text", + content: "cleaned", + mediaType: "text/markdown", + title: "Title" + }); expect(JSON.stringify(calls[0])).toContain("source text"); }); diff --git a/tests/builtins/pipeline-steps/load-source/load-source-step.test.ts b/tests/builtins/pipeline-steps/load-source/load-source-step.test.ts index 5b7c0dd..adb7e68 100644 --- a/tests/builtins/pipeline-steps/load-source/load-source-step.test.ts +++ b/tests/builtins/pipeline-steps/load-source/load-source-step.test.ts @@ -10,7 +10,10 @@ import { makeStepContext } from "../utils.js"; describe("LoadSourceStep", () => { it("skips when a body already exists", async () => { - const step = new LoadSourceStep({ sourceProvider: provider({ content: "new" }), logger: createTestLogger() }); + const step = new LoadSourceStep({ + sourceProvider: provider({ kind: "text", content: "new", mediaType: "text/markdown" }), + logger: createTestLogger() + }); await expect(step.run(makeStepContext({ body: { content: "old" } }))).resolves.toMatchObject({ status: "skipped", @@ -20,14 +23,93 @@ describe("LoadSourceStep", () => { it("writes loaded content and title", async () => { const step = new LoadSourceStep({ - sourceProvider: provider({ content: " markdown ", title: "Title" }), + sourceProvider: provider({ kind: "text", content: " markdown ", mediaType: "text/markdown", title: "Title" }), logger: createTestLogger() }); const result = await step.run(makeStepContext()); expect(result).toMatchObject({ status: "ok", diagnostics: { attributes: { title: "Title" } } }); - expect(result.effects?.body).toEqual({ content: "markdown", title: "Title" }); + expect(result.effects?.body).toEqual({ + kind: "text", + content: "markdown", + mediaType: "text/markdown", + title: "Title" + }); + }); + + it("degrades and still writes the body when the provider truncated content", async () => { + const step = new LoadSourceStep({ + sourceProvider: provider({ + kind: "text", + content: " markdown ", + mediaType: "text/markdown", + title: "Title", + truncated: true + }), + logger: createTestLogger() + }); + + const result = await step.run(makeStepContext()); + + expect(result).toMatchObject({ + status: "degraded", + reason: "truncated", + diagnostics: { attributes: { title: "Title" } } + }); + expect(result.effects?.body).toEqual({ + kind: "text", + content: "markdown", + mediaType: "text/markdown", + title: "Title" + }); + }); + + it("stays ok when the provider reports truncated false", async () => { + const step = new LoadSourceStep({ + sourceProvider: provider({ kind: "text", content: "markdown", mediaType: "text/markdown", truncated: false }), + logger: createTestLogger() + }); + + const result = await step.run(makeStepContext()); + + expect(result.status).toBe("ok"); + expect(result.reason).toBeUndefined(); + }); + + it("writes loaded binary content", async () => { + const bytes = new Uint8Array([37, 80, 68, 70]); + const step = new LoadSourceStep({ + sourceProvider: provider({ kind: "binary", bytes, mediaType: "application/pdf" }), + logger: createTestLogger() + }); + + const result = await step.run(makeStepContext()); + + expect(result).toMatchObject({ status: "ok" }); + expect(result.effects?.body).toEqual({ kind: "binary", bytes, mediaType: "application/pdf" }); + }); + + it("degrades and still writes binary content when the provider truncated bytes", async () => { + const bytes = new Uint8Array([1, 2, 3]); + const step = new LoadSourceStep({ + sourceProvider: provider({ kind: "binary", bytes, mediaType: "application/octet-stream", truncated: true }), + logger: createTestLogger() + }); + + const result = await step.run(makeStepContext()); + + expect(result).toMatchObject({ status: "degraded", reason: "truncated" }); + expect(result.effects?.body).toEqual({ kind: "binary", bytes, mediaType: "application/octet-stream" }); + }); + + it("fails empty binary content", async () => { + const step = new LoadSourceStep({ + sourceProvider: provider({ kind: "binary", bytes: new Uint8Array(), mediaType: "application/pdf" }), + logger: createTestLogger() + }); + + await expect(step.run(makeStepContext())).resolves.toMatchObject({ status: "failed", reason: "empty" }); }); it("maps provider errors to load-source reason tokens", async () => { diff --git a/tests/builtins/pipeline-steps/truncate/truncate-step.test.ts b/tests/builtins/pipeline-steps/truncate/truncate-step.test.ts index c07c59a..e6e4859 100644 --- a/tests/builtins/pipeline-steps/truncate/truncate-step.test.ts +++ b/tests/builtins/pipeline-steps/truncate/truncate-step.test.ts @@ -6,6 +6,14 @@ import { createTestLogger } from "../../../helpers/logger.js"; import { makeStepContext } from "../utils.js"; describe("TruncateStep", () => { + it("skips on binary body", async () => { + const step = new TruncateStep({ targetChars: 100 }, { logger: createTestLogger() }); + + const result = await step.run(makeStepContext({ body: { bytes: new Uint8Array([1, 2, 3]) } })); + + expect(result).toMatchObject({ status: "skipped", reason: "unsupported_media_type" }); + }); + it("skips without a body or when under target", async () => { const step = new TruncateStep({ targetChars: 10 }, { logger: createTestLogger() }); @@ -24,8 +32,14 @@ describe("TruncateStep", () => { ); expect(result.status).toBe("ok"); - expect(result.effects?.body?.content).toBe("one t... [TRUNCATED]"); - expect(result.effects?.body?.content.length).toBe(20); + expect(result.effects?.body).toEqual({ + kind: "text", + content: "one t... [TRUNCATED]", + mediaType: "text/markdown", + title: "Title" + }); + expect(result.effects?.body?.kind).toBe("text"); + if (result.effects?.body?.kind === "text") expect(result.effects.body.content.length).toBe(20); expect(result.effects?.body?.title).toBe("Title"); }); @@ -34,7 +48,13 @@ describe("TruncateStep", () => { const result = await step.run(makeStepContext({ body: { content: "long content" } })); - expect(result.effects?.body?.content).toBe("... [TRU"); - expect(result.effects?.body?.content.length).toBe(8); + expect(result.effects?.body).toEqual({ + kind: "text", + content: "... [TRU", + mediaType: "text/markdown", + title: undefined + }); + expect(result.effects?.body?.kind).toBe("text"); + if (result.effects?.body?.kind === "text") expect(result.effects.body.content.length).toBe(8); }); }); diff --git a/tests/builtins/pipeline-steps/utils.ts b/tests/builtins/pipeline-steps/utils.ts index b57ec8c..96e3dce 100644 --- a/tests/builtins/pipeline-steps/utils.ts +++ b/tests/builtins/pipeline-steps/utils.ts @@ -2,11 +2,28 @@ import type { BodyContent, PipelineContext, ScalarValue } from "../../../src/contracts/pipeline/context.js"; import { BodyStore } from "../../../src/core/pipeline/body.js"; import { ReadonlyArtifactBag, ReadonlySignalBag } from "../../../src/core/pipeline/context.js"; +import { textBody, binaryBody } from "../../helpers/body.js"; + +/** Body fragment for test ergonomics. Use `content` for a text body, `bytes` for a binary body. */ +type BodyInput = + | { readonly content: string; readonly mediaType?: string; readonly title?: string } + | { readonly bytes: Uint8Array; readonly mediaType?: string; readonly title?: string }; + +function normalize(body: BodyInput, defaultMediaType: string): BodyContent { + if ("bytes" in body) { + return binaryBody({ + bytes: body.bytes, + mediaType: body.mediaType ?? "application/octet-stream", + title: body.title + }); + } + return textBody({ content: body.content, mediaType: body.mediaType ?? defaultMediaType, title: body.title }); +} export function makeStepContext( args: { - readonly body?: BodyContent; - readonly bodyVersions?: ReadonlyArray<{ readonly stepName: string } & BodyContent>; + readonly body?: BodyInput; + readonly bodyVersions?: ReadonlyArray<{ readonly stepName: string } & BodyInput>; readonly signal?: AbortSignal; readonly signals?: ReadonlyMap<string, ScalarValue>; readonly artifacts?: ReadonlyMap<string, unknown>; @@ -15,9 +32,10 @@ export function makeStepContext( ): PipelineContext { const body = new BodyStore(); if (args.bodyVersions) { - for (const version of args.bodyVersions) body.append({ ...version }); + for (const version of args.bodyVersions) + body.append({ stepName: version.stepName, ...normalize(version, "text/markdown") }); } else if (args.body) { - body.append({ stepName: "source", ...args.body }); + body.append({ stepName: "source", ...normalize(args.body, "text/markdown") }); } return { input: { url: args.url ?? "https://example.com/" }, diff --git a/tests/builtins/pipeline-steps/verify-urls/verify-urls-step.test.ts b/tests/builtins/pipeline-steps/verify-urls/verify-urls-step.test.ts index 1bdaa0a..6964f1d 100644 --- a/tests/builtins/pipeline-steps/verify-urls/verify-urls-step.test.ts +++ b/tests/builtins/pipeline-steps/verify-urls/verify-urls-step.test.ts @@ -13,6 +13,14 @@ function inventory(urls: ReadonlyArray<string>): Map<string, unknown> { } describe("VerifyUrlsStep", () => { + it("skips on binary body", async () => { + const step = new VerifyUrlsStep({ artifact: ARTIFACT, onHallucination: "report" }, { logger: createTestLogger() }); + + const result = await step.run(makeStepContext({ body: { bytes: new Uint8Array([1, 2, 3]) } })); + + expect(result).toMatchObject({ status: "skipped", reason: "unsupported_media_type" }); + }); + it("skips when no body is present", async () => { const step = new VerifyUrlsStep({ artifact: ARTIFACT, onHallucination: "report" }, { logger: createTestLogger() }); @@ -85,7 +93,9 @@ describe("VerifyUrlsStep", () => { expect(result.status).toBe("degraded"); expect(result.reason).toBe("hallucinated_urls"); expect(result.effects?.body).toEqual({ + kind: "text", content: "Original body with [good](https://example.com/good)", + mediaType: "text/markdown", title: "Title" }); expect(result.effects?.artifacts).toBeUndefined(); diff --git a/tests/builtins/source-providers/docling/docling-provider-config.test.ts b/tests/builtins/source-providers/docling/docling-provider-config.test.ts new file mode 100644 index 0000000..3fd05ce --- /dev/null +++ b/tests/builtins/source-providers/docling/docling-provider-config.test.ts @@ -0,0 +1,43 @@ +/** Verifies Docling provider configuration parsing. */ +import { describe, expect, it } from "vitest"; + +import { parseDoclingConfig } from "../../../../src/builtins/source-providers/docling/docling-provider-config.js"; + +describe("parseDoclingConfig", () => { + it("applies defaults and coerces env-substituted scalar strings", () => { + expect(parseDoclingConfig({ baseUrl: "http://docling.example" })).toMatchObject({ + apiKey: "", + output: "markdown", + doOcr: true, + tableMode: "accurate" + }); + expect( + parseDoclingConfig({ baseUrl: "http://docling.example", output: "html", doOcr: " false ", tableMode: "fast" }) + ).toMatchObject({ + output: "html", + doOcr: false, + tableMode: "fast" + }); + expect(parseDoclingConfig({ baseUrl: "http://docling.example", doOcr: "", apiKey: "", output: "" })).toMatchObject({ + output: "markdown", + doOcr: true, + apiKey: "" + }); + }); + + it("rejects invalid boolean strings", () => { + expect(() => parseDoclingConfig({ baseUrl: "http://docling.example", doOcr: "yes" })).toThrow(); + }); + + it("rejects invalid output values", () => { + expect(() => parseDoclingConfig({ baseUrl: "http://docling.example", output: "rawHtml" })).toThrow(); + }); + + it("rejects invalid tableMode values", () => { + expect(() => parseDoclingConfig({ baseUrl: "http://docling.example", tableMode: "exact" })).toThrow(); + }); + + it("rejects unknown fields", () => { + expect(() => parseDoclingConfig({ baseUrl: "http://docling.example", mystery: true })).toThrow(); + }); +}); diff --git a/tests/builtins/source-providers/docling/docling-provider.test.ts b/tests/builtins/source-providers/docling/docling-provider.test.ts new file mode 100644 index 0000000..126b04f --- /dev/null +++ b/tests/builtins/source-providers/docling/docling-provider.test.ts @@ -0,0 +1,244 @@ +/** Verifies Docling source-provider request, response, and failure behavior. */ +import { describe, expect, it } from "vitest"; + +import { UpstreamError } from "../../../../src/shared/errors.js"; +import { parseDoclingConfig } from "../../../../src/builtins/source-providers/docling/docling-provider-config.js"; +import { DoclingProvider } from "../../../../src/builtins/source-providers/docling/docling-provider.js"; +import { createTestLogger } from "../../../helpers/logger.js"; +import { jsonResponse } from "../../../helpers/responses.js"; + +describe("DoclingProvider", () => { + it("sends the convert request and returns markdown content", async () => { + let requestBody: Record<string, unknown> | undefined; + const fetchFn: typeof fetch = async (input, init) => { + requestBody = JSON.parse(String(init?.body)); + expect(String(input)).toBe("http://docling.example/v1/convert/source"); + expect(init?.method).toBe("POST"); + return jsonResponse({ + document: { md_content: "# hello", json_content: { name: "Hello" } }, + status: "success", + processing_time: 0.5 + }); + }; + const provider = new DoclingProvider( + parseDoclingConfig({ baseUrl: "http://docling.example", doOcr: false, tableMode: "fast" }), + { httpFetch: fetchFn, logger: createTestLogger() } + ); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(requestBody).toEqual({ + sources: [{ kind: "http", url: "https://example.com" }], + options: { + to_formats: ["md", "json"], + image_export_mode: "placeholder", + do_ocr: false, + table_mode: "fast" + } + }); + expect(document).toEqual({ kind: "text", content: "# hello", mediaType: "text/markdown", title: "Hello" }); + }); + + it("returns html content when output is html", async () => { + let requestBody: Record<string, unknown> | undefined; + const fetchFn: typeof fetch = async (_input, init) => { + requestBody = JSON.parse(String(init?.body)); + return jsonResponse({ + document: { html_content: "<p>Hello</p>", json_content: { name: "Hello" } }, + status: "success", + processing_time: 0.5 + }); + }; + const provider = new DoclingProvider(parseDoclingConfig({ baseUrl: "http://docling.example", output: "html" }), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(requestBody).toMatchObject({ + options: { to_formats: ["html", "json"] } + }); + expect(document).toEqual({ kind: "text", content: "<p>Hello</p>", mediaType: "text/html", title: "Hello" }); + }); + + it("accepts partial_success status and missing title", async () => { + const provider = new DoclingProvider(parseDoclingConfig({ baseUrl: "http://docling.example" }), { + httpFetch: async () => + jsonResponse({ + document: { md_content: "# ok", json_content: null }, + status: "partial_success", + processing_time: 1.0 + }), + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document).toEqual({ kind: "text", content: "# ok", mediaType: "text/markdown" }); + }); + + it("omits title when json_content.name is missing or empty", async () => { + const noName = new DoclingProvider(parseDoclingConfig({ baseUrl: "http://docling.example" }), { + httpFetch: async () => + jsonResponse({ + document: { md_content: "content", json_content: {} }, + status: "success", + processing_time: 0.5 + }), + logger: createTestLogger() + }); + const emptyName = new DoclingProvider(parseDoclingConfig({ baseUrl: "http://docling.example" }), { + httpFetch: async () => + jsonResponse({ + document: { md_content: "content", json_content: { name: "" } }, + status: "success", + processing_time: 0.5 + }), + logger: createTestLogger() + }); + + await expect(noName.load("https://example.com", { signal: new AbortController().signal })).resolves.toEqual({ + kind: "text", + content: "content", + mediaType: "text/markdown" + }); + await expect(emptyName.load("https://example.com", { signal: new AbortController().signal })).resolves.toEqual({ + kind: "text", + content: "content", + mediaType: "text/markdown" + }); + }); + + it("sends X-Api-Key header when an API key is configured", async () => { + let headers: Record<string, string> | undefined; + const fetchFn: typeof fetch = async (_input, init) => { + headers = init?.headers as Record<string, string>; + return jsonResponse({ + document: { md_content: "ok" }, + status: "success", + processing_time: 0.5 + }); + }; + const provider = new DoclingProvider(parseDoclingConfig({ baseUrl: "http://docling.example", apiKey: "dl-key" }), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(headers?.["x-api-key"]).toBe("dl-key"); + }); + + it("preserves sanitized upstream codes from failed status", async () => { + const fetchFn: typeof fetch = async () => + jsonResponse( + { + document: { md_content: null }, + status: "failure", + errors: [{ error_message: "model timed out" }], + processing_time: 5.0 + }, + { status: 200 } + ); + const provider = new DoclingProvider(parseDoclingConfig({ baseUrl: "http://docling.example" }), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + await expect(provider.load("https://example.com", { signal: new AbortController().signal })).rejects.toMatchObject({ + upstreamCode: "failure", + upstreamStatus: 200 + }); + }); + + it("rejects empty content as an upstream empty response for any output", async () => { + const mdProvider = new DoclingProvider( + parseDoclingConfig({ baseUrl: "http://docling.example", output: "markdown" }), + { + httpFetch: async () => + jsonResponse({ + document: { md_content: " " }, + status: "success", + processing_time: 0.5 + }), + logger: createTestLogger() + } + ); + const htmlProvider = new DoclingProvider( + parseDoclingConfig({ baseUrl: "http://docling.example", output: "html" }), + { + httpFetch: async () => + jsonResponse({ + document: { html_content: " " }, + status: "success", + processing_time: 0.5 + }), + logger: createTestLogger() + } + ); + + await expect( + mdProvider.load("https://example.com", { signal: new AbortController().signal }) + ).rejects.toMatchObject({ + upstreamCode: "empty" + }); + await expect( + htmlProvider.load("https://example.com", { signal: new AbortController().signal }) + ).rejects.toMatchObject({ + upstreamCode: "empty" + }); + }); + + it("wraps non-JSON and network failures as UpstreamError", async () => { + const nonJson = new DoclingProvider(parseDoclingConfig({ baseUrl: "http://docling.example" }), { + httpFetch: async () => new Response("<html></html>", { status: 200 }), + logger: createTestLogger() + }); + await expect(nonJson.load("https://example.com", { signal: new AbortController().signal })).rejects.toMatchObject({ + upstreamCode: "parse_error" + }); + + const network = new DoclingProvider(parseDoclingConfig({ baseUrl: "http://docling.example" }), { + httpFetch: async () => { + throw new Error("network broke"); + }, + logger: createTestLogger() + }); + await expect(network.load("https://example.com", { signal: new AbortController().signal })).rejects.toBeInstanceOf( + UpstreamError + ); + await expect(network.load("https://example.com", { signal: new AbortController().signal })).rejects.toMatchObject({ + upstreamCode: "network" + }); + }); + + it("forwards AbortSignal and lets abort errors propagate", async () => { + const controller = new AbortController(); + const abort = new DOMException("aborted", "AbortError"); + let signal: AbortSignal | null | undefined; + const provider = new DoclingProvider(parseDoclingConfig({ baseUrl: "http://docling.example" }), { + httpFetch: async (_input, init) => { + signal = init?.signal; + throw abort; + }, + logger: createTestLogger() + }); + + await expect(provider.load("https://example.com", { signal: controller.signal })).rejects.toBe(abort); + + expect(signal).toBe(controller.signal); + }); + + it("lets abort errors from JSON parsing propagate", async () => { + const abort = new DOMException("aborted", "AbortError"); + const response = jsonResponse({ document: { md_content: "ok" }, status: "success", processing_time: 0.5 }); + Object.defineProperty(response, "json", { value: async () => Promise.reject(abort) }); + const provider = new DoclingProvider(parseDoclingConfig({ baseUrl: "http://docling.example" }), { + httpFetch: async () => response, + logger: createTestLogger() + }); + + await expect(provider.load("https://example.com", { signal: new AbortController().signal })).rejects.toBe(abort); + }); +}); diff --git a/tests/builtins/source-providers/firecrawl/firecrawl-provider-config.test.ts b/tests/builtins/source-providers/firecrawl/firecrawl-provider-config.test.ts index c809e39..6529b1d 100644 --- a/tests/builtins/source-providers/firecrawl/firecrawl-provider-config.test.ts +++ b/tests/builtins/source-providers/firecrawl/firecrawl-provider-config.test.ts @@ -6,20 +6,42 @@ import { parseFirecrawlConfig } from "../../../../src/builtins/source-providers/ describe("parseFirecrawlConfig", () => { it("applies defaults and coerces env-substituted scalar strings", () => { expect(parseFirecrawlConfig({ baseUrl: "https://firecrawl.example" })).toMatchObject({ + output: "markdown", onlyMainContent: true, - maxAge: 0 + stripBase64Images: true, + parsePdf: true }); expect( - parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", onlyMainContent: " false ", maxAge: "30" }) + parseFirecrawlConfig({ + baseUrl: "https://firecrawl.example", + output: "html", + onlyMainContent: " false ", + stripBase64Images: " false ", + parsePdf: " false " + }) ).toMatchObject({ + output: "html", onlyMainContent: false, - maxAge: 30 + stripBase64Images: false, + parsePdf: false }); expect( - parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", onlyMainContent: "", maxAge: "" }) + parseFirecrawlConfig({ + baseUrl: "https://firecrawl.example", + onlyMainContent: "", + stripBase64Images: "", + parsePdf: "" + }) ).toMatchObject({ onlyMainContent: true, - maxAge: 0 + stripBase64Images: true, + parsePdf: true + }); + }); + + it("accepts rawHtml output", () => { + expect(parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "rawHtml" })).toMatchObject({ + output: "rawHtml" }); }); @@ -27,7 +49,11 @@ describe("parseFirecrawlConfig", () => { expect(() => parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", onlyMainContent: "yes" })).toThrow(); }); - it("rejects invalid numeric limits", () => { - expect(() => parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", maxAge: "abc" })).toThrow(); + it("rejects invalid output values", () => { + expect(() => parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "markup" })).toThrow(); + }); + + it("rejects unknown fields", () => { + expect(() => parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", maxAge: 0 })).toThrow(); }); }); diff --git a/tests/builtins/source-providers/firecrawl/firecrawl-provider.test.ts b/tests/builtins/source-providers/firecrawl/firecrawl-provider.test.ts index 587efcb..10734e2 100644 --- a/tests/builtins/source-providers/firecrawl/firecrawl-provider.test.ts +++ b/tests/builtins/source-providers/firecrawl/firecrawl-provider.test.ts @@ -20,9 +20,10 @@ describe("FirecrawlProvider", () => { parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", apiKey: "", + output: "markdown", onlyMainContent: false, - formats: ["markdown"], - maxAge: 0 + stripBase64Images: true, + parsePdf: true }), { httpFetch: fetchFn, logger: createTestLogger() } ); @@ -33,20 +34,98 @@ describe("FirecrawlProvider", () => { url: "https://example.com", formats: ["markdown"], onlyMainContent: false, - maxAge: 0 + removeBase64Images: true, + parsers: ["pdf"] }); - expect(document).toEqual({ content: "# hello", title: "Hello" }); + expect(document).toEqual({ kind: "text", content: "# hello", mediaType: "text/markdown", title: "Hello" }); }); - it("accepts top-level markdown response fields", async () => { - const provider = new FirecrawlProvider(parseFirecrawlConfig({ baseUrl: "https://firecrawl.example" }), { - httpFetch: async () => jsonResponse({ success: true, markdown: "# top", title: "Top" }), - logger: createTestLogger() - }); + it("returns html content when output is html", async () => { + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "html" }), + { + httpFetch: async () => + jsonResponse({ success: true, data: { html: "<p>Hello</p>", metadata: { title: "Hello" } } }), + logger: createTestLogger() + } + ); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document).toEqual({ kind: "text", content: "<p>Hello</p>", mediaType: "text/html", title: "Hello" }); + }); + + it("returns rawHtml content when output is rawHtml", async () => { + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "rawHtml" }), + { + httpFetch: async () => jsonResponse({ success: true, data: { rawHtml: "<html><body>Raw</body></html>" } }), + logger: createTestLogger() + } + ); const document = await provider.load("https://example.com", { signal: new AbortController().signal }); - expect(document).toEqual({ content: "# top", title: "Top" }); + expect(document).toEqual({ kind: "text", content: "<html><body>Raw</body></html>", mediaType: "text/html" }); + }); + + it("omits parsers when parsePdf is false", async () => { + let requestBody: Record<string, unknown> | undefined; + const fetchFn: typeof fetch = async (_input, init) => { + requestBody = JSON.parse(String(init?.body)); + return jsonResponse({ success: true, data: { markdown: "ok" } }); + }; + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", parsePdf: false }), + { httpFetch: fetchFn, logger: createTestLogger() } + ); + + await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(requestBody?.parsers).toEqual([]); + }); + + it("accepts top-level response fields for any output format", async () => { + const mdProvider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "markdown" }), + { + httpFetch: async () => jsonResponse({ success: true, markdown: "# top", title: "MdTop" }), + logger: createTestLogger() + } + ); + const htmlProvider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "html" }), + { + httpFetch: async () => jsonResponse({ success: true, html: "<p>top</p>", title: "HtmlTop" }), + logger: createTestLogger() + } + ); + const rawProvider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "rawHtml" }), + { + httpFetch: async () => jsonResponse({ success: true, rawHtml: "<html>raw</html>", title: "RawTop" }), + logger: createTestLogger() + } + ); + + await expect(mdProvider.load("https://example.com", { signal: new AbortController().signal })).resolves.toEqual({ + kind: "text", + content: "# top", + mediaType: "text/markdown", + title: "MdTop" + }); + await expect(htmlProvider.load("https://example.com", { signal: new AbortController().signal })).resolves.toEqual({ + kind: "text", + content: "<p>top</p>", + mediaType: "text/html", + title: "HtmlTop" + }); + await expect(rawProvider.load("https://example.com", { signal: new AbortController().signal })).resolves.toEqual({ + kind: "text", + content: "<html>raw</html>", + mediaType: "text/html", + title: "RawTop" + }); }); it("sends bearer authorization when an API key is configured", async () => { @@ -82,15 +161,44 @@ describe("FirecrawlProvider", () => { }); }); - it("rejects empty markdown as an upstream empty response", async () => { - const provider = new FirecrawlProvider(parseFirecrawlConfig({ baseUrl: "https://firecrawl.example" }), { - httpFetch: async () => jsonResponse({ success: true, data: { markdown: " " } }), - logger: createTestLogger() - }); + it("rejects empty content as an upstream empty response for any output", async () => { + const provider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "markdown" }), + { + httpFetch: async () => jsonResponse({ success: true, data: { markdown: " " } }), + logger: createTestLogger() + } + ); await expect(provider.load("https://example.com", { signal: new AbortController().signal })).rejects.toMatchObject({ upstreamCode: "empty" }); + + const htmlProvider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "html" }), + { + httpFetch: async () => jsonResponse({ success: true, data: { html: " " } }), + logger: createTestLogger() + } + ); + await expect( + htmlProvider.load("https://example.com", { signal: new AbortController().signal }) + ).rejects.toMatchObject({ + upstreamCode: "empty" + }); + + const rawProvider = new FirecrawlProvider( + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "rawHtml" }), + { + httpFetch: async () => jsonResponse({ success: true, data: { rawHtml: " " } }), + logger: createTestLogger() + } + ); + await expect( + rawProvider.load("https://example.com", { signal: new AbortController().signal }) + ).rejects.toMatchObject({ + upstreamCode: "empty" + }); }); it("wraps non-JSON and network failures as UpstreamError", async () => { diff --git a/tests/builtins/source-providers/http/http-provider-config.test.ts b/tests/builtins/source-providers/http/http-provider-config.test.ts new file mode 100644 index 0000000..fd4b04a --- /dev/null +++ b/tests/builtins/source-providers/http/http-provider-config.test.ts @@ -0,0 +1,38 @@ +/** Verifies HTTP provider configuration parsing. */ +import { describe, expect, it } from "vitest"; + +import { parseHttpConfig } from "../../../../src/builtins/source-providers/http/http-provider-config.js"; + +describe("parseHttpConfig", () => { + it("applies defaults and coerces env-substituted scalar strings", () => { + expect(parseHttpConfig({})).toMatchObject({ + maxBytes: 5_000_000, + titleFromHtml: true + }); + expect(parseHttpConfig({ maxBytes: "100", titleFromHtml: " false " })).toMatchObject({ + maxBytes: 100, + titleFromHtml: false + }); + expect(parseHttpConfig({ maxBytes: "", titleFromHtml: "" })).toMatchObject({ + maxBytes: 5_000_000, + titleFromHtml: true + }); + }); + + it("rejects invalid boolean strings", () => { + expect(() => parseHttpConfig({ titleFromHtml: "yes" })).toThrow(); + }); + + it("rejects non-positive maxBytes", () => { + expect(() => parseHttpConfig({ maxBytes: -1 })).toThrow(); + expect(() => parseHttpConfig({ maxBytes: 0 })).toThrow(); + }); + + it("rejects invalid numeric maxBytes", () => { + expect(() => parseHttpConfig({ maxBytes: "abc" })).toThrow(); + }); + + it("rejects unknown fields", () => { + expect(() => parseHttpConfig({ mystery: true })).toThrow(); + }); +}); diff --git a/tests/builtins/source-providers/http/http-provider.test.ts b/tests/builtins/source-providers/http/http-provider.test.ts new file mode 100644 index 0000000..5b714e3 --- /dev/null +++ b/tests/builtins/source-providers/http/http-provider.test.ts @@ -0,0 +1,401 @@ +/** Verifies HTTP source-provider request, response, and failure behavior. */ +import { describe, expect, it } from "vitest"; + +import { parseHttpConfig } from "../../../../src/builtins/source-providers/http/http-provider-config.js"; +import { HttpProvider } from "../../../../src/builtins/source-providers/http/http-provider.js"; +import { createTestLogger } from "../../../helpers/logger.js"; + +describe("HttpProvider", () => { + it("fetches a URL and returns the raw body with title from <title>", async () => { + const fetchFn: typeof fetch = async (input, init) => { + expect(String(input)).toBe("https://example.com"); + expect((init?.headers as Record<string, string>)?.["user-agent"]).toBeTruthy(); + return new Response("<html><head><title> My Page Hello"); + }; + const provider = new HttpProvider(parseHttpConfig({}), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document).toEqual({ + kind: "text", + content: expect.stringContaining("Hello"), + mediaType: "text/plain", + title: "My Page", + truncated: false + }); + }); + + it("omits title when titleFromHtml is false", async () => { + const fetchFn: typeof fetch = async () => new Response("Test"); + const provider = new HttpProvider(parseHttpConfig({ titleFromHtml: false }), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document).toEqual({ + kind: "text", + content: expect.stringContaining("Test"), + mediaType: "text/plain", + truncated: false + }); + }); + + it("sets the configured user agent", async () => { + let userAgent: string | undefined; + const fetchFn: typeof fetch = async (_input, init) => { + userAgent = (init?.headers as Record)?.["user-agent"]; + return new Response("content"); + }; + const provider = new HttpProvider(parseHttpConfig({ userAgent: "test-bot/1.0" }), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(userAgent).toBe("test-bot/1.0"); + }); + + it("truncates oversized response bodies", async () => { + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode("a".repeat(500))); + controller.enqueue(encoder.encode("b".repeat(500))); + controller.close(); + } + }); + const fetchFn: typeof fetch = async () => new Response(stream); + const provider = new HttpProvider(parseHttpConfig({ maxBytes: 100 }), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.kind).toBe("text"); + if (document.kind === "text") { + expect(document.content.length).toBe(100); + expect(document.content).toBe("a".repeat(100)); + } + expect(document.truncated).toBe(true); + }); + + it("truncates at a chunk boundary when the cap lands mid-chunk", async () => { + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode("a".repeat(500))); + controller.enqueue(encoder.encode("b".repeat(500))); + controller.close(); + } + }); + const fetchFn: typeof fetch = async () => new Response(stream); + const provider = new HttpProvider(parseHttpConfig({ maxBytes: 600 }), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.kind).toBe("text"); + if (document.kind === "text") { + expect(document.content.length).toBe(600); + expect(document.content).toBe("a".repeat(500) + "b".repeat(100)); + } + expect(document.truncated).toBe(true); + }); + + it("reports truncated when a streamed body exactly fills the cap before more data", async () => { + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode("a".repeat(100))); + controller.enqueue(encoder.encode("b")); + controller.close(); + } + }); + const fetchFn: typeof fetch = async () => new Response(stream); + const provider = new HttpProvider(parseHttpConfig({ maxBytes: 100 }), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.kind).toBe("text"); + if (document.kind === "text") expect(document.content).toBe("a".repeat(100)); + expect(document.truncated).toBe(true); + }); + + it("reports not truncated when a streamed body fits within the cap", async () => { + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode("a".repeat(50))); + controller.close(); + } + }); + const fetchFn: typeof fetch = async () => new Response(stream); + const provider = new HttpProvider(parseHttpConfig({ maxBytes: 100 }), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.kind).toBe("text"); + if (document.kind === "text") expect(document.content).toBe("a".repeat(50)); + expect(document.truncated).toBe(false); + }); + + it("wraps stream read failures as UpstreamError", async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([104, 105])); // "hi" + controller.error(new TypeError("stream corrupted")); + } + }); + const fetchFn: typeof fetch = async () => new Response(stream); + const provider = new HttpProvider(parseHttpConfig({ maxBytes: 100 }), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + await expect(provider.load("https://example.com", { signal: new AbortController().signal })).rejects.toMatchObject({ + upstreamCode: "network" + }); + }); + + it("rejects non-ok HTTP status", async () => { + const fetchFn: typeof fetch = async () => new Response("Not Found", { status: 404 }); + const provider = new HttpProvider(parseHttpConfig({}), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + await expect(provider.load("https://example.com", { signal: new AbortController().signal })).rejects.toMatchObject({ + upstreamCode: "http_404", + upstreamStatus: 404 + }); + }); + + it("returns application/pdf as a binary body", async () => { + const pdfBytes = new TextEncoder().encode("%PDF-1.7..."); + const fetchFn: typeof fetch = async () => + new Response(pdfBytes, { + status: 200, + headers: { "content-type": "application/pdf" } + }); + const provider = new HttpProvider(parseHttpConfig({}), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document).toMatchObject({ + kind: "binary", + mediaType: "application/pdf", + truncated: false + }); + }); + + it("returns binary for application/pdf and reads the body", async () => { + const fetchFn: typeof fetch = async () => + new Response(new Uint8Array([37, 80, 68, 70]), { + status: 200, + headers: { "content-type": "application/pdf" } + }); + const provider = new HttpProvider(parseHttpConfig({}), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.kind).toBe("binary"); + if (document.kind === "binary") expect(document.bytes.byteLength).toBeGreaterThan(0); + }); + + it("returns application/octet-stream body as binary", async () => { + const raw = new Uint8Array([104, 105, 0, 101]); + const fetchFn: typeof fetch = async () => + new Response(raw, { + status: 200, + headers: { "content-type": "application/octet-stream" } + }); + const provider = new HttpProvider(parseHttpConfig({}), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.kind).toBe("binary"); + if (document.kind === "binary") { + expect(document.mediaType).toBe("application/octet-stream"); + expect(document.bytes).toEqual(raw); + } + }); + + it("reclassifies a missing header with NUL bytes as binary", async () => { + const raw = new Uint8Array([78, 0, 97, 0, 109, 0, 101]); + const fetchFn: typeof fetch = async () => new Response(raw); + const provider = new HttpProvider(parseHttpConfig({}), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.kind).toBe("binary"); + if (document.kind === "binary") expect(document.bytes).toEqual(raw); + }); + + it("accepts text/html content type", async () => { + const fetchFn: typeof fetch = async () => + new Response("Hello", { + status: 200, + headers: { "content-type": "text/html; charset=utf-8" } + }); + const provider = new HttpProvider(parseHttpConfig({}), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.kind).toBe("text"); + if (document.kind === "text") expect(document.content).toContain("Hello"); + expect(document.mediaType).toBe("text/html"); + }); + + it("returns text/markdown for text/markdown content type", async () => { + const fetchFn: typeof fetch = async () => + new Response("# Markdown body", { + status: 200, + headers: { "content-type": "text/markdown" } + }); + const provider = new HttpProvider(parseHttpConfig({}), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.mediaType).toBe("text/markdown"); + }); + + it("preserves application/json content type", async () => { + const fetchFn: typeof fetch = async () => + new Response('{"key": "value"}', { + status: 200, + headers: { "content-type": "application/json" } + }); + const provider = new HttpProvider(parseHttpConfig({}), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.kind).toBe("text"); + if (document.kind === "text") expect(document.content).toBe('{"key": "value"}'); + expect(document.mediaType).toBe("application/json"); + }); + + it("preserves other textual content types (no collapse to text/plain)", async () => { + const fetchFn: typeof fetch = async () => + new Response("css", { + status: 200, + headers: { "content-type": "text/css" } + }); + const provider = new HttpProvider(parseHttpConfig({}), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.mediaType).toBe("text/css"); + }); + + it("preserves structured +xml content types", async () => { + const fetchFn: typeof fetch = async () => + new Response("", { + status: 200, + headers: { "content-type": "application/rss+xml; charset=utf-8" } + }); + const provider = new HttpProvider(parseHttpConfig({}), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.mediaType).toBe("application/rss+xml"); + }); + + it("accepts clean text with no content-type header", async () => { + const fetchFn: typeof fetch = async () => new Response("just text, no NUL bytes"); + const provider = new HttpProvider(parseHttpConfig({}), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(document.kind).toBe("text"); + if (document.kind === "text") expect(document.content).toBe("just text, no NUL bytes"); + }); + + it("rejects empty body as upstream empty response", async () => { + const fetchFn: typeof fetch = async () => new Response(" "); + const provider = new HttpProvider(parseHttpConfig({}), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + await expect(provider.load("https://example.com", { signal: new AbortController().signal })).rejects.toMatchObject({ + upstreamCode: "empty", + upstreamStatus: 200 + }); + }); + + it("wraps network failures as UpstreamError", async () => { + const fetchFn: typeof fetch = async () => { + throw new Error("connection refused"); + }; + const provider = new HttpProvider(parseHttpConfig({}), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + await expect(provider.load("https://example.com", { signal: new AbortController().signal })).rejects.toMatchObject({ + upstreamCode: "network" + }); + }); + + it("forwards AbortSignal and lets abort errors propagate", async () => { + const controller = new AbortController(); + const abort = new DOMException("aborted", "AbortError"); + let signalFromInit: AbortSignal | undefined; + const fetchFn: typeof fetch = async (_input, init) => { + signalFromInit = init?.signal ?? undefined; + throw abort; + }; + const provider = new HttpProvider(parseHttpConfig({}), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + await expect(provider.load("https://example.com", { signal: controller.signal })).rejects.toBe(abort); + expect(signalFromInit).toBe(controller.signal); + }); +}); diff --git a/tests/bundles/default-engine-descriptors.test.ts b/tests/bundles/default-engine-descriptors.test.ts index 5f87490..8614c2f 100644 --- a/tests/bundles/default-engine-descriptors.test.ts +++ b/tests/bundles/default-engine-descriptors.test.ts @@ -80,7 +80,7 @@ describe("default engine descriptor bundle", () => { }); function sourceProvider(): SourceProvider { - return { load: async () => ({ content: "source" }) }; + return { load: async () => ({ kind: "text", content: "source", mediaType: "text/markdown" }) }; } function llmProvider(): LlmProvider { diff --git a/tests/config/yaml/yaml-app-config.test.ts b/tests/config/yaml/yaml-app-config.test.ts index fa8df2f..a2caa06 100644 --- a/tests/config/yaml/yaml-app-config.test.ts +++ b/tests/config/yaml/yaml-app-config.test.ts @@ -22,12 +22,10 @@ describe("yamlToAppConfig", () => { const appConfig = yamlToAppConfig(raw); - expect(appConfig.engineConfig).toEqual({ - sourceProviders: raw.sourceProviders, - llmProviders: raw.llmProviders, - outputRenderers: raw.outputRenderers, - pipelines: raw.pipelines - }); + expect(appConfig.engineConfig.sourceProviders).toEqual(raw.sourceProviders); + expect(appConfig.engineConfig.llmProviders).toEqual(raw.llmProviders); + expect(appConfig.engineConfig.outputRenderers).toEqual(raw.outputRenderers); + expect(appConfig.engineConfig.pipelines.default?.enabled).toBe(true); expect(appConfig.adapters.http).toEqual(raw.httpAdapters); expect(appConfig.metadata?.schemaVersion).toBe(1); }); @@ -75,17 +73,24 @@ httpAdapters: }); describe("substituteEnv", () => { - it("supports required values, fallbacks, empty fallbacks, and literal escapes", () => { + it("supports required values, fallbacks, empty fallbacks, literal escapes, and missing-no-default", () => { expect( substituteEnv( { required: "${REQUIRED}", fallback: "${MISSING:-fallback}", empty: "${EMPTY:-fallback}", - literal: "$${REQUIRED}" + literal: "$${REQUIRED}", + missing: "${MISSING_NO_DEFAULT}" }, { REQUIRED: "value", EMPTY: "" } ) - ).toEqual({ required: "value", fallback: "fallback", empty: "fallback", literal: "${REQUIRED}" }); + ).toEqual({ + required: "value", + fallback: "fallback", + empty: "fallback", + literal: "${REQUIRED}", + missing: "" + }); }); }); diff --git a/tests/contracts/output-renderer-conformance.ts b/tests/contracts/output-renderer-conformance.ts index 125ab25..fae79fc 100644 --- a/tests/contracts/output-renderer-conformance.ts +++ b/tests/contracts/output-renderer-conformance.ts @@ -2,19 +2,20 @@ import { expect } from "vitest"; import type { OutputRenderer, OutputRendererInput } from "../../src/contracts/extensions/output-renderer.js"; +import { textBody, binaryBody } from "../helpers/body.js"; export function makeRenderInput(overrides: Partial = {}): OutputRendererInput { return { pipelineName: "test", - body: { content: "hello" }, + body: textBody({ content: "hello" }), signals: new Map(), artifacts: new Map(), report: { url: "https://example.com/", startedAt: 1, durationMs: 2, - initialChars: 5, - finalChars: 5, + initialLength: 5, + finalLength: 5, returned: "source", result: "ok", steps: [] @@ -29,4 +30,16 @@ export async function assertOutputRendererConformance(renderer: OutputRenderer): const withoutBody = await renderer.render(makeRenderInput({ body: undefined })); expect(typeof withoutBody.markdown).toBe("string"); + + const withBinary = await renderer.render( + makeRenderInput({ + body: binaryBody({ bytes: new Uint8Array([37, 80, 68, 70]), mediaType: "application/pdf" }), + report: { + ...makeRenderInput().report, + result: "failed", + error: "unconverted_binary: application/pdf" + } + }) + ); + expect(typeof withBinary.markdown).toBe("string"); } diff --git a/tests/core/pipeline/body-effects.test.ts b/tests/core/pipeline/body-effects.test.ts index 8e35bbb..9570202 100644 --- a/tests/core/pipeline/body-effects.test.ts +++ b/tests/core/pipeline/body-effects.test.ts @@ -3,19 +3,37 @@ import { describe, expect, it } from "vitest"; import { BodyStore } from "../../../src/core/pipeline/body.js"; import { applyStepEffects } from "../../../src/core/pipeline/effects.js"; +import { binaryBody, textBody } from "../../helpers/body.js"; describe("BodyStore", () => { it("tracks current body and immutable version snapshots", () => { const body = new BodyStore(); - body.append({ stepName: "fetch", content: "source", title: "Title" }); - body.append({ stepName: "clean", content: "clean", title: "Title" }); + body.append({ stepName: "fetch", kind: "text", content: "source", mediaType: "text/markdown", title: "Title" }); + body.append({ stepName: "clean", kind: "text", content: "clean", mediaType: "text/markdown", title: "Title" }); const versions = body.versions(); - expect(body.current()).toEqual({ content: "clean", title: "Title" }); + expect(body.current()).toEqual(textBody({ content: "clean", title: "Title" })); expect(versions.map(version => version.stepName)).toEqual(["fetch", "clean"]); expect(versions).not.toBe(body.versions()); }); + + it("copies binary bytes when writing and reading snapshots", () => { + const body = new BodyStore(); + const bytes = new Uint8Array([1, 2, 3]); + + body.append({ stepName: "fetch", ...binaryBody({ bytes, mediaType: "application/pdf" }) }); + bytes[0] = 9; + const current = body.current(); + const versions = body.versions(); + + expect(current).toEqual(binaryBody({ bytes: new Uint8Array([1, 2, 3]), mediaType: "application/pdf" })); + if (current?.kind === "binary") current.bytes[1] = 8; + const firstVersion = versions[0]; + if (firstVersion?.kind === "binary") firstVersion.bytes[2] = 7; + + expect(body.current()).toEqual(binaryBody({ bytes: new Uint8Array([1, 2, 3]), mediaType: "application/pdf" })); + }); }); describe("applyStepEffects", () => { @@ -27,7 +45,7 @@ describe("applyStepEffects", () => { { status: "ok", effects: { - body: { content: "source" }, + body: textBody({ content: "source" }), signals: { "feature.enabled": true }, artifacts: { "feature.payload": { value: 1 } } } @@ -35,8 +53,8 @@ describe("applyStepEffects", () => { state ); - expect(summary).toEqual({ outputChars: 6, wroteBody: true }); - expect(state.body.current()).toEqual({ content: "source", title: undefined }); + expect(summary).toEqual({ outputLength: 6, wroteBody: true }); + expect(state.body.current()).toEqual(textBody({ content: "source" })); expect(state.signals.get("feature.enabled")).toBe(true); expect(state.artifacts.get("feature.payload")).toEqual({ value: 1 }); @@ -52,7 +70,13 @@ describe("applyStepEffects", () => { it("applies body, signal, and artifact effects for degraded results", () => { const state = { body: new BodyStore(), signals: new Map(), artifacts: new Map() }; - state.body.append({ stepName: "fetch", content: "source", title: "Title" }); + state.body.append({ + stepName: "fetch", + kind: "text", + content: "source", + mediaType: "text/markdown", + title: "Title" + }); state.signals.set("feature.enabled", true); state.artifacts.set("feature.payload", { value: 1 }); @@ -62,7 +86,7 @@ describe("applyStepEffects", () => { status: "degraded", reason: "hallucinated_urls", effects: { - body: { content: "source", title: "Title" }, + body: textBody({ content: "source", title: "Title" }), signals: { "feature.enabled": null }, artifacts: { "feature.payload": null } } @@ -70,7 +94,7 @@ describe("applyStepEffects", () => { state ); - expect(summary).toEqual({ outputChars: 6, wroteBody: true }); + expect(summary).toEqual({ outputLength: 6, wroteBody: true }); expect(state.body.versions().map(version => version.stepName)).toEqual(["fetch", "rollback"]); expect(state.signals.has("feature.enabled")).toBe(false); expect(state.artifacts.has("feature.payload")).toBe(false); @@ -78,7 +102,13 @@ describe("applyStepEffects", () => { it("ignores effects on failed results", () => { const state = { body: new BodyStore(), signals: new Map(), artifacts: new Map() }; - state.body.append({ stepName: "fetch", content: "source", title: "Title" }); + state.body.append({ + stepName: "fetch", + kind: "text", + content: "source", + mediaType: "text/markdown", + title: "Title" + }); state.signals.set("feature.enabled", true); state.artifacts.set("feature.payload", { value: 1 }); @@ -88,7 +118,7 @@ describe("applyStepEffects", () => { status: "failed", reason: "hallucinated_urls", effects: { - body: { content: "replaced", title: "Title" }, + body: textBody({ content: "replaced", title: "Title" }), signals: { "feature.enabled": null }, artifacts: { "feature.payload": null } } @@ -111,7 +141,7 @@ describe("applyStepEffects", () => { status: "skipped", reason: "no_body", effects: { - body: { content: "ignored" }, + body: textBody({ content: "ignored" }), signals: { "feature.enabled": true }, artifacts: { "feature.payload": { value: 1 } } } diff --git a/tests/core/pipeline/orchestrator.test.ts b/tests/core/pipeline/orchestrator.test.ts index a185126..b6cc0c7 100644 --- a/tests/core/pipeline/orchestrator.test.ts +++ b/tests/core/pipeline/orchestrator.test.ts @@ -1,12 +1,13 @@ /** Verifies pipeline orchestration, effects, timeouts, reporting, and limiter grouping. */ import { afterEach, describe, expect, it, vi } from "vitest"; -import type { PipelineContext } from "../../../src/contracts/pipeline/context.js"; +import { isTextBody, type PipelineContext } from "../../../src/contracts/pipeline/context.js"; import type { CompiledPipelineStep } from "../../../src/core/pipeline/compiled.js"; import type { PipelineStep, StepResult } from "../../../src/contracts/pipeline/step.js"; import type { ConcurrencyLimiter } from "../../../src/shared/limiters.js"; import { PipelineOrchestrator } from "../../../src/core/pipeline/orchestrator.js"; import { createConcurrencyLimiter } from "../../../src/shared/limiters.js"; +import { binaryBody, textBody } from "../../helpers/body.js"; import { createTestLogger } from "../../helpers/logger.js"; import { makePipeline } from "../../helpers/pipeline.js"; @@ -38,7 +39,14 @@ class BodyEchoStep implements PipelineStep { async run(ctx: PipelineContext): Promise { const body = ctx.body.current(); - return { status: "ok", effects: { body: { content: `${body?.content ?? ""}!`, title: body?.title } } }; + const bodyContent = body && isTextBody(body) ? body.content : ""; + const mediaType = body?.mediaType ?? "text/markdown"; + return { + status: "ok", + effects: { + body: { kind: "text", content: `${bodyContent}!`, mediaType, title: body?.title } + } + }; } } @@ -98,7 +106,13 @@ describe("PipelineOrchestrator", () => { it("ignores effects on skipped results and reports degraded fallback", async () => { const pipeline = makePipeline({ steps: [ - withMeta(new FakeStep("fetch", { status: "ok", effects: { body: { content: "source", title: "Title" } } }), 5), + withMeta( + new FakeStep("fetch", { + status: "ok", + effects: { body: textBody({ content: "source", title: "Title" }) } + }), + 5 + ), withMeta(new FakeStep("clean", { status: "failed", reason: "network" }), 5), withMeta(new BodyEchoStep(), 5) ] @@ -106,24 +120,30 @@ describe("PipelineOrchestrator", () => { const result = await makeOrchestrator().run(pipeline, { url: "https://example.com/" }); - expect(result.body).toEqual({ content: "source!", title: "Title" }); + expect(result.body).toEqual(textBody({ content: "source!", title: "Title" })); expect(result.report.result).toBe("degraded"); - expect(result.report.initialChars).toBe(6); - expect(result.report.finalChars).toBe(7); + expect(result.report.initialLength).toBe(6); + expect(result.report.finalLength).toBe(7); expect(result.report.returned).toBe("echo"); expect(result.report.steps.map(step => step.status)).toEqual(["ok", "failed", "ok"]); - expect(result.report.steps[1]?.outputChars).toBeUndefined(); + expect(result.report.steps[1]?.outputLength).toBeUndefined(); }); it("applies body effects on degraded results and rolls the run up as degraded", async () => { const pipeline = makePipeline({ steps: [ - withMeta(new FakeStep("fetch", { status: "ok", effects: { body: { content: "source", title: "Title" } } }), 5), + withMeta( + new FakeStep("fetch", { + status: "ok", + effects: { body: textBody({ content: "source", title: "Title" }) } + }), + 5 + ), withMeta( new FakeStep("verify", { status: "degraded", reason: "hallucinated_urls", - effects: { body: { content: "source", title: "Title" } } + effects: { body: textBody({ content: "source", title: "Title" }) } }), 5 ) @@ -132,7 +152,7 @@ describe("PipelineOrchestrator", () => { const result = await makeOrchestrator().run(pipeline, { url: "https://example.com/" }); - expect(result.body).toEqual({ content: "source", title: "Title" }); + expect(result.body).toEqual(textBody({ content: "source", title: "Title" })); expect(result.report.result).toBe("degraded"); expect(result.report.returned).toBe("verify"); expect(result.report.bodyChangedBy).toBe("verify"); @@ -141,19 +161,51 @@ describe("PipelineOrchestrator", () => { name: "verify", status: "degraded", reason: "hallucinated_urls", - outputChars: 6 + outputLength: 6 + }); + }); + + it("attributes the final body to the step that produced the final media type", async () => { + const pipeline = makePipeline({ + steps: [ + withMeta( + new FakeStep("fetch", { + status: "ok", + effects: { body: textBody({ content: "same", mediaType: "text/html", title: "Title" }) } + }), + 5 + ), + withMeta( + new FakeStep("convert", { + status: "ok", + effects: { body: textBody({ content: "same", mediaType: "text/markdown", title: "Title" }) } + }), + 5 + ) + ] }); + + const result = await makeOrchestrator().run(pipeline, { url: "https://example.com/" }); + + expect(result.report.bodyProducedBy).toBe("convert"); + expect(result.report.bodyChangedBy).toBe("convert"); }); it("ignores effects on failed results and rolls the run up as degraded", async () => { const pipeline = makePipeline({ steps: [ - withMeta(new FakeStep("fetch", { status: "ok", effects: { body: { content: "source", title: "Title" } } }), 5), + withMeta( + new FakeStep("fetch", { + status: "ok", + effects: { body: textBody({ content: "source", title: "Title" }) } + }), + 5 + ), withMeta( new FakeStep("verify", { status: "failed", reason: "hallucinated_urls", - effects: { body: { content: "rewritten", title: "Title" } } + effects: { body: textBody({ content: "rewritten", title: "Title" }) } }), 5 ) @@ -162,7 +214,7 @@ describe("PipelineOrchestrator", () => { const result = await makeOrchestrator().run(pipeline, { url: "https://example.com/" }); - expect(result.body).toEqual({ content: "source", title: "Title" }); + expect(result.body).toEqual(textBody({ content: "source", title: "Title" })); expect(result.report.result).toBe("degraded"); expect(result.report.bodyChangedBy).toBe("fetch"); expect(result.report.steps[1]).toMatchObject({ @@ -170,7 +222,7 @@ describe("PipelineOrchestrator", () => { status: "failed", reason: "hallucinated_urls" }); - expect(result.report.steps[1]?.outputChars).toBeUndefined(); + expect(result.report.steps[1]?.outputLength).toBeUndefined(); }); it("copies child diagnostics from step results into reports", async () => { @@ -180,7 +232,7 @@ describe("PipelineOrchestrator", () => { new FakeStep("fetch", { status: "ok", diagnostics: { children: [{ name: "attempt", attributes: { count: 1 } }] }, - effects: { body: { content: "source" } } + effects: { body: textBody({ content: "source" }) } }), 5 ) @@ -307,6 +359,30 @@ describe("PipelineOrchestrator", () => { expect(result.report.result).toBe("failed"); expect(result.report.steps[0]).toMatchObject({ name: "explode", status: "failed", reason: "thrown" }); }); + + it("reports terminal binary body as failed with unconverted_binary", async () => { + const pipeline = makePipeline({ + steps: [ + withMeta( + new FakeStep("fetch", { + status: "ok", + effects: { + body: binaryBody({ bytes: new Uint8Array([37, 80, 68, 70]), mediaType: "application/pdf" }) + } + }), + 5 + ) + ] + }); + + const result = await makeOrchestrator().run(pipeline, { url: "https://example.com/" }); + + expect(result.report.result).toBe("failed"); + expect(result.report.error).toBe("unconverted_binary: application/pdf"); + expect(result.report.initialLength).toBe(4); + expect(result.report.finalLength).toBe(4); + expect(result.report.returned).toBe("fetch"); + }); }); function makeOrchestrator(): PipelineOrchestrator { diff --git a/tests/core/pipeline/render-failure.test.ts b/tests/core/pipeline/render-failure.test.ts index 19cb0c5..e0f1bd4 100644 --- a/tests/core/pipeline/render-failure.test.ts +++ b/tests/core/pipeline/render-failure.test.ts @@ -52,8 +52,8 @@ describe("PipelineRunner.renderFailure", () => { url: "https://example.com/", result: "failed", returned: "none", - initialChars: 0, - finalChars: 0, + initialLength: 0, + finalLength: 0, steps: [] }); }); diff --git a/tests/core/pipeline/signals-artifacts.test.ts b/tests/core/pipeline/signals-artifacts.test.ts index c207aaf..8c2eb4a 100644 --- a/tests/core/pipeline/signals-artifacts.test.ts +++ b/tests/core/pipeline/signals-artifacts.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import type { PipelineContext } from "../../../src/contracts/pipeline/context.js"; +import { isTextBody } from "../../../src/contracts/pipeline/context.js"; import type { OutputRenderer, OutputRendererInput, @@ -18,7 +19,7 @@ class EmittingStep implements PipelineStep { return { status: "ok", effects: { - body: { content: "hello" }, + body: { kind: "text", content: "hello", mediaType: "text/markdown" }, signals: { tone: "friendly", score: 7 }, artifacts: { metrics: { tokens: 42 } } } @@ -33,7 +34,8 @@ class RecordingRenderer implements OutputRenderer { render(input: OutputRendererInput): OutputRendererResult { this.received = input; - return { markdown: input.body?.content ?? "" }; + const content = input.body && isTextBody(input.body) ? input.body.content : ""; + return { markdown: content }; } } diff --git a/tests/engine/activation-lazy.test.ts b/tests/engine/activation-lazy.test.ts new file mode 100644 index 0000000..b393a98 --- /dev/null +++ b/tests/engine/activation-lazy.test.ts @@ -0,0 +1,123 @@ +/** Verifies lazy provider/renderer registries (D1). */ +import { describe, expect, it } from "vitest"; + +import { createEngine } from "../../src/engine/create-engine.js"; +import { createTestHostTools } from "../helpers/host-tools.js"; +import { createTestLogger } from "../helpers/logger.js"; +import { markdownRendererDescriptor } from "./renderer-descriptor.js"; + +import { sourceProviderRegistryKey } from "../../src/contracts/extensions/source-provider.js"; +import type { SourceProviderDescriptor } from "../../src/contracts/extensions/source-provider.js"; +import type { PipelineStepDescriptor } from "../../src/contracts/pipeline/step.js"; + +describe("lazy provider/registry activation", () => { + it("unused provider with missing required config does not fail startup", async () => { + await expect( + createEngine({ + config: { + sourceProviders: { + "bad-provider": { type: "validated-source", config: { requiredField: "" } } + }, + llmProviders: {}, + outputRenderers: { markdown: { type: "markdown", config: {} } }, + pipelines: { + default: { + enabled: true, + outputRenderer: "markdown", + limiters: {}, + steps: [{ type: "static-body", name: "write", timeoutSeconds: 5, config: { content: "ok" } }] + } + } + }, + descriptors: { + sourceProviders: { + "validated-source": { + type: "validated-source", + parseConfig: raw => raw, + create: ({ config }) => { + if (!config || !(config as { requiredField: string }).requiredField) + throw new Error("validated-source requiredField is missing"); + return { load: async () => ({ kind: "text" as const, content: "", mediaType: "text/markdown" }) }; + } + } satisfies SourceProviderDescriptor + }, + llmProviders: {}, + outputRenderers: { markdown: markdownRendererDescriptor }, + pipelineSteps: { + "static-body": staticBodyStepDescriptor + } + }, + tools: createTestHostTools(), + logger: createTestLogger() + }) + ).resolves.toBeDefined(); + }); + + it("reachable provider with missing required config fails with descriptor error", async () => { + await expect( + createEngine({ + config: { + sourceProviders: { + "bad-provider": { type: "validated-source", config: { requiredField: "" } } + }, + llmProviders: {}, + outputRenderers: { markdown: { type: "markdown", config: {} } }, + pipelines: { + default: { + enabled: true, + outputRenderer: "markdown", + limiters: {}, + steps: [{ type: "uses-provider", name: "fetch", timeoutSeconds: 5, config: { provider: "bad-provider" } }] + } + } + }, + descriptors: { + sourceProviders: { + "validated-source": { + type: "validated-source", + parseConfig: raw => raw, + create: ({ config }) => { + if (!config || !(config as { requiredField: string }).requiredField) + throw new Error("validated-source requiredField is missing"); + return { load: async () => ({ kind: "text" as const, content: "", mediaType: "text/markdown" }) }; + } + } satisfies SourceProviderDescriptor + }, + llmProviders: {}, + outputRenderers: { markdown: markdownRendererDescriptor }, + pipelineSteps: { + "uses-provider": { + type: "uses-provider", + parseConfig: raw => raw as { provider: string }, + create: async ({ config, services }) => { + // Step that resolves a source provider at create time. + const cfg = config as { provider: string }; + services.require(sourceProviderRegistryKey).require(cfg.provider); + return { run: async () => ({ status: "ok" as const }) }; + } + } satisfies PipelineStepDescriptor, + "static-body": staticBodyStepDescriptor + } + }, + tools: createTestHostTools(), + logger: createTestLogger() + }) + ).rejects.toThrow("Failed to create source provider 'bad-provider'"); + }); +}); + +const staticBodyStepDescriptor = { + type: "static-body", + parseConfig: (raw: unknown) => raw, + create: ({ config }: { config: unknown }) => { + const cfg = config as { content: string }; + return { + run: async () => ({ + status: "ok" as const, + effects: { + body: { kind: "text" as const, content: cfg.content, mediaType: "text/markdown" as const } + } + }) + }; + } +} satisfies PipelineStepDescriptor; diff --git a/tests/engine/activation-pipeline.test.ts b/tests/engine/activation-pipeline.test.ts new file mode 100644 index 0000000..84d7848 --- /dev/null +++ b/tests/engine/activation-pipeline.test.ts @@ -0,0 +1,115 @@ +/** Verifies pipeline activation, disabled-but-referenced error, active-set filtering, and D7 log. */ +import { describe, expect, it } from "vitest"; + +import { yamlToAppConfig } from "../../src/config/yaml/yaml-app-config.js"; +import { rawYamlConfigSchema } from "../../src/config/yaml/yaml-config.js"; +import { createEngine } from "../../src/engine/create-engine.js"; +import { createTestHostTools } from "../helpers/host-tools.js"; +import { createTestLogger, type CapturedLog } from "../helpers/logger.js"; +import { markdownRendererDescriptor } from "./renderer-descriptor.js"; + +import type { SourceProviderDescriptor } from "../../src/contracts/extensions/source-provider.js"; +import type { PipelineStepDescriptor } from "../../src/contracts/pipeline/step.js"; + +describe("pipeline activation", () => { + it("disabled-but-referenced pipeline throws clear error in yamlToAppConfig", () => { + const raw = rawYamlConfigSchema.parse({ + schemaVersion: 1, + outputRenderers: { markdown: { type: "markdown", config: {} } }, + pipelines: { parked: { enabled: false, outputRenderer: "markdown", limiters: {}, steps: [] } }, + httpAdapters: { owui: { type: "open-webui", pipeline: "parked", config: { path: "/" } } } + }); + + expect(() => yamlToAppConfig(raw)).toThrow( + "Pipeline 'parked' is referenced by an HTTP adapter but disabled (enabled: false)" + ); + }); + + it("disabled pipeline does not appear in pipelineInfos or handles", async () => { + const logs: CapturedLog[] = []; + const engine = await createEngine({ + config: { + sourceProviders: {}, + llmProviders: {}, + outputRenderers: { markdown: { type: "markdown", config: {} } }, + pipelines: { + active: { + enabled: true, + outputRenderer: "markdown", + limiters: {}, + steps: [{ type: "static-body", name: "write", timeoutSeconds: 5, config: { content: "hello" } }] + }, + parked: { + enabled: false, + outputRenderer: "markdown", + limiters: {}, + steps: [{ type: "static-body", name: "write", timeoutSeconds: 5, config: { content: "parked" } }] + } + } + }, + descriptors: { + sourceProviders: {}, + llmProviders: {}, + outputRenderers: { markdown: markdownRendererDescriptor }, + pipelineSteps: { "static-body": staticBodyStepDescriptor } + }, + tools: createTestHostTools(), + logger: createTestLogger(logs) + }); + + const pipelines = engine.listPipelines(); + expect(pipelines).toHaveLength(1); + expect(pipelines[0]!.name).toBe("active"); + expect(engine.getPipeline("active")).toBeDefined(); + expect(engine.getPipeline("parked")).toBeUndefined(); + }); + + it("defined-but-unreferenced providers appear in D7 skipped log", async () => { + const logs: CapturedLog[] = []; + const sourceProv: SourceProviderDescriptor = { + type: "never-built", + parseConfig: () => ({}), + create: () => ({ load: async () => ({ kind: "text" as const, content: "", mediaType: "text/markdown" }) }) + }; + + await createEngine({ + config: { + sourceProviders: { "never-used": { type: "never-built", config: {} } }, + llmProviders: {}, + outputRenderers: { markdown: { type: "markdown", config: {} } }, + pipelines: {} + }, + descriptors: { + sourceProviders: { "never-built": sourceProv }, + llmProviders: {}, + outputRenderers: { markdown: markdownRendererDescriptor }, + pipelineSteps: {} + }, + tools: createTestHostTools(), + logger: createTestLogger(logs) + }); + + const skippedLogs = logs.filter( + log => log.level === "info" && (log.message ?? "").includes("defined but not referenced") + ); + expect(skippedLogs.length).toBeGreaterThan(0); + expect(skippedLogs.some(log => (log.message ?? "").includes("source provider"))).toBe(true); + expect((skippedLogs[0]!.value as { skipped: string[] }).skipped).toContain("never-used"); + }); +}); + +const staticBodyStepDescriptor = { + type: "static-body", + parseConfig: (raw: unknown) => raw, + create: ({ config }: { config: unknown }) => { + const cfg = config as { content: string }; + return { + run: async () => ({ + status: "ok" as const, + effects: { + body: { kind: "text" as const, content: cfg.content, mediaType: "text/markdown" as const } + } + }) + }; + } +} satisfies PipelineStepDescriptor; diff --git a/tests/engine/create-engine.test.ts b/tests/engine/create-engine.test.ts index 000890e..807a995 100644 --- a/tests/engine/create-engine.test.ts +++ b/tests/engine/create-engine.test.ts @@ -6,6 +6,7 @@ import type { PipelineStepDescriptor } from "../../src/contracts/pipeline/step.j import { createEngine } from "../../src/engine/create-engine.js"; import { createTestHostTools } from "../helpers/host-tools.js"; import { createTestLogger } from "../helpers/logger.js"; +import { markdownRendererDescriptor } from "./renderer-descriptor.js"; describe("createEngine", () => { it("constructs, lists, and runs pipelines without YAML or HTTP", async () => { @@ -16,6 +17,7 @@ describe("createEngine", () => { outputRenderers: { markdown: { type: "markdown", config: {} } }, pipelines: { default: { + enabled: true, outputRenderer: "markdown", limiters: {}, steps: [{ type: "static-body", name: "write_body", timeoutSeconds: 5, config: { content: "hello engine" } }] @@ -64,16 +66,13 @@ describe("createEngine", () => { }); }); -const markdownRendererDescriptor = { - type: "markdown", - parseConfig: () => ({}), - create: () => ({ render: input => ({ markdown: input.body?.content ?? "" }) }) -} satisfies OutputRendererDescriptor; - const staticBodyStepDescriptor = { type: "static-body", parseConfig: raw => raw as { content: string }, create: ({ config }) => ({ - run: async () => ({ status: "ok" as const, effects: { body: { content: config.content } } }) + run: async () => ({ + status: "ok" as const, + effects: { body: { kind: "text", content: config.content, mediaType: "text/markdown" } } + }) }) } satisfies PipelineStepDescriptor<{ content: string }>; diff --git a/tests/engine/renderer-descriptor.ts b/tests/engine/renderer-descriptor.ts new file mode 100644 index 0000000..117b986 --- /dev/null +++ b/tests/engine/renderer-descriptor.ts @@ -0,0 +1,16 @@ +/** + * Shared renderer descriptor for create-engine tests. + */ +import { isTextBody } from "../../src/contracts/pipeline/context.js"; +import type { OutputRendererDescriptor, OutputRendererInput } from "../../src/contracts/extensions/output-renderer.js"; + +export const markdownRendererDescriptor = { + type: "markdown", + parseConfig: () => ({}), + create: () => ({ + render: (input: OutputRendererInput) => { + const content = input.body && isTextBody(input.body) ? input.body.content : ""; + return { markdown: content }; + } + }) +} satisfies OutputRendererDescriptor; diff --git a/tests/helpers/body.ts b/tests/helpers/body.ts new file mode 100644 index 0000000..c014698 --- /dev/null +++ b/tests/helpers/body.ts @@ -0,0 +1,29 @@ +/** + * Canonical body factories for tests. + * + * Used for both construction and assertion so build and assert agree. + * A future body reshape edits this single factory file. + */ + +import type { BinaryBody, TextBody } from "../../src/contracts/pipeline/context.js"; + +export function textBody(args: { + readonly content: string; + readonly mediaType?: string; + readonly title?: string; +}): TextBody { + return { kind: "text", content: args.content, mediaType: args.mediaType ?? "text/markdown", title: args.title }; +} + +export function binaryBody(args: { + readonly bytes: Uint8Array; + readonly mediaType?: string; + readonly title?: string; +}): BinaryBody { + return { + kind: "binary", + bytes: args.bytes, + mediaType: args.mediaType ?? "application/octet-stream", + title: args.title + }; +} diff --git a/tests/helpers/pipeline.ts b/tests/helpers/pipeline.ts index 92f07d3..9c7d9ee 100644 --- a/tests/helpers/pipeline.ts +++ b/tests/helpers/pipeline.ts @@ -11,12 +11,16 @@ import { PassthroughRenderer } from "../../src/builtins/output-renderers/passthr import { PipelineOrchestrator } from "../../src/core/pipeline/orchestrator.js"; import { PipelineRunner } from "../../src/core/pipeline/runner.js"; import { createTestLogger } from "./logger.js"; +import { textBody, binaryBody } from "./body.js"; export class StaticBodyStep implements PipelineStep { constructor(private readonly content: string) {} async run(_ctx: PipelineContext): Promise { - return { status: "ok", effects: { body: { content: this.content, title: "Test title" } } }; + return { + status: "ok", + effects: { body: textBody({ content: this.content, title: "Test title" }) } + }; } } @@ -91,21 +95,21 @@ export function makeStaticPipelineHandle( export function makePipelineResult(content = "hello"): PipelineRunResult { return { - body: { content, title: "Test title" }, + body: textBody({ content, title: "Test title" }), signals: new Map(), artifacts: new Map(), report: { url: "https://example.com/", startedAt: 1, durationMs: 2, - initialChars: content.length, - finalChars: content.length, + initialLength: content.length, + finalLength: content.length, ratio: 1, returned: "source", result: "ok", bodyProducedBy: "source", bodyChangedBy: "source", - steps: [{ name: "source", type: "test", status: "ok", startedAt: 1, durationMs: 2, outputChars: content.length }] + steps: [{ name: "source", type: "test", status: "ok", startedAt: 1, durationMs: 2, outputLength: content.length }] } }; } @@ -119,8 +123,8 @@ export function makeFailedPipelineResult(error = "firecrawl: scrape_retry_limit" url: "https://example.com/", startedAt: 1, durationMs: 2, - initialChars: 0, - finalChars: 0, + initialLength: 0, + finalLength: 0, returned: "none", result: "failed", error, @@ -137,3 +141,38 @@ export function makeFailedPipelineResult(error = "firecrawl: scrape_retry_limit" } }; } + +export function makeTerminalBinaryPipelineResult( + bytes: Uint8Array, + mediaType = "application/pdf", + errorMsg = `unconverted_binary: ${mediaType}` +): PipelineRunResult { + return { + body: binaryBody({ bytes, mediaType }), + signals: new Map(), + artifacts: new Map(), + report: { + url: "https://example.com/", + startedAt: 1, + durationMs: 2, + initialLength: bytes.byteLength, + finalLength: bytes.byteLength, + ratio: 1, + returned: "source", + result: "failed", + bodyProducedBy: "source", + bodyChangedBy: "source", + error: errorMsg, + steps: [ + { + name: "source", + type: "load-source", + status: "ok", + startedAt: 1, + durationMs: 2, + outputLength: bytes.byteLength + } + ] + } + }; +} diff --git a/tests/shared/media-types.test.ts b/tests/shared/media-types.test.ts new file mode 100644 index 0000000..432a715 --- /dev/null +++ b/tests/shared/media-types.test.ts @@ -0,0 +1,50 @@ +/** Verifies media-type predicates and constants. */ +import { describe, expect, it } from "vitest"; + +import { isTextLike, mediaTypes } from "../../src/shared/media-types.js"; + +describe("mediaTypes", () => { + it("exports expected text media type constants", () => { + expect(mediaTypes.html).toBe("text/html"); + expect(mediaTypes.markdown).toBe("text/markdown"); + expect(mediaTypes.plainText).toBe("text/plain"); + expect(mediaTypes.json).toBe("application/json"); + expect(mediaTypes.xml).toBe("application/xml"); + expect(mediaTypes.xhtml).toBe("application/xhtml+xml"); + }); +}); + +describe("isTextLike", () => { + it("returns true for text/* types", () => { + expect(isTextLike("text/html")).toBe(true); + expect(isTextLike(" Text/HTML ")).toBe(true); + expect(isTextLike("text/markdown")).toBe(true); + expect(isTextLike("text/plain")).toBe(true); + expect(isTextLike("text/css")).toBe(true); + expect(isTextLike("text/csv")).toBe(true); + }); + + it("returns true for known textual application/* types", () => { + expect(isTextLike("application/json")).toBe(true); + expect(isTextLike("application/xml")).toBe(true); + expect(isTextLike("application/xhtml+xml")).toBe(true); + }); + + it("returns true for +json and +xml structured suffix types", () => { + expect(isTextLike("application/rss+xml")).toBe(true); + expect(isTextLike("application/atom+xml")).toBe(true); + expect(isTextLike("application/vnd.api+json")).toBe(true); + expect(isTextLike("image/svg+xml")).toBe(true); + }); + + it("returns false for known binary types", () => { + expect(isTextLike("application/pdf")).toBe(false); + expect(isTextLike("application/octet-stream")).toBe(false); + expect(isTextLike("image/png")).toBe(false); + expect(isTextLike("image/jpeg")).toBe(false); + }); + + it("returns false for empty string", () => { + expect(isTextLike("")).toBe(false); + }); +});