diff --git a/.env.example b/.env.example index ae8d489..666dfd9 100644 --- a/.env.example +++ b/.env.example @@ -22,8 +22,8 @@ LOG_LEVEL=info LOG_PRETTY=auto # --- Pipeline ------------------------------------------------------------ -# Active pipeline selected by HTTP adapters. Default: truncate (load-source + truncate) -DEFAULT_PIPELINE=clean-combined +# Active pipeline selected by HTTP adapters. Default: full (full-featured; external deps disabled by default). +DEFAULT_PIPELINE=full # Maximum concurrent source-loading groups. Default: 1; raise only if providers can handle it. SOURCE_CONCURRENCY=5 @@ -44,28 +44,37 @@ DEBUG_XML_INCLUDE_SKIPPED=false OUTPUT_TARGET_CHARS=35000 # --- Source provider ----------------------------------------------------- -# Optional source-provider override. Leave empty to use the selected pipeline's fallback. -SOURCE_PROVIDER= +# Enable Firecrawl source loading. Disabled by default. Default: false +FIRECRAWL_ENABLED=false -# Firecrawl base URL. Required only when a referenced provider uses it (e.g. firecrawl-markdown, firecrawl-html). +# Firecrawl base URL. Required only when FIRECRAWL_ENABLED=true. FIRECRAWL_BASE_URL=https://firecrawl.example # Optional Firecrawl bearer token. Default: empty (no token) FIRECRAWL_API_KEY= -# Firecrawl PDF parsing. Set to false to return raw PDF bytes (binary body) instead of parsed text. -FIRECRAWL_PARSE_PDF= +# Opaque Firecrawl scrape options, keeping real API parameter names. +FIRECRAWL_OPTIONS= -# Docling Serve base URL. Required only when a referenced provider uses it (e.g. docling-ocr). +# Enable Docling OCR source loading for binary documents. Disabled by default. Default: false +DOCLING_ENABLED=false + +# Docling Serve base URL. Required only when DOCLING_ENABLED=true. DOCLING_BASE_URL=https://docling.example # Optional Docling API key (X-Api-Key header). Default: empty (no token) DOCLING_API_KEY= +# Opaque JSON object of Docling convert options, merged into the convert body. +DOCLING_OPTIONS= + # --- Content transformers ------------------------------------------------ # Post-conversion link and whitespace cleanup for mdream-convert. Ignored when minimal=true (mdream-aggressive). Default: true MDREAM_CLEAN=true +# Enable Readability HTML-to-article extraction. Enabled by default (local; no external service needed). Default: true +READABILITY_ENABLED=true + # Minimum content length for Readability's isProbablyReaderable gate. Default: 140 READABILITY_MIN_CONTENT_CHARS=140 @@ -76,7 +85,13 @@ READABILITY_MIN_SCORE=20 READABILITY_MAX_ELEMENTS=0 # --- LLM provider -------------------------------------------------------- -# OpenAI-compatible Chat Completions /v1 base URL. Required only when a referenced provider uses it (e.g. llm-default). +# Enable the clean LLM pass. Disabled by default. Default: false +LLM_CLEAN_ENABLED=false + +# Enable the summarize LLM pass. Disabled by default. Default: false +LLM_SUMMARIZE_ENABLED=false + +# OpenAI-compatible Chat Completions /v1 base URL. Required only when LLM_CLEAN_ENABLED or LLM_SUMMARIZE_ENABLED. LLM_BASE_URL=https://openai-compatible.example/v1 # Optional LLM bearer token. Default: empty (no token) @@ -95,17 +110,23 @@ LLM_CHARS_PER_TOKEN=3.5 LLM_SAFETY_MARGIN_TOKENS=128 # --- Step thresholds and timeouts --------------------------------------- -# Timeout in seconds for the source-loading step. Default: 20 -LOAD_SOURCE_TIMEOUT_SECONDS=20 +# Timeout in seconds for the native HTTP source-loading step. Default: 20 +FETCH_TIMEOUT_SECONDS=20 + +# Timeout in seconds for the Firecrawl source-loading step. Default: 20 +FIRECRAWL_TIMEOUT_SECONDS=20 + +# Timeout in seconds for the Docling OCR source-loading step. Default: 60; OCR is slower. +DOCLING_TIMEOUT_SECONDS=60 # Timeout in seconds for the clean LLM pass. Default: 60; raise for slower models. -CLEAN_TIMEOUT_SECONDS=90 +LLM_CLEAN_TIMEOUT_SECONDS=90 # Minimum body length before the clean LLM pass runs. Default: 1000; lower values run cleanup more often. -CLEAN_MIN_INPUT_CHARS=500 +LLM_CLEAN_MIN_INPUT_CHARS=500 # Timeout in seconds for the summarize LLM pass. Default: 60 -SUMMARIZE_TIMEOUT_SECONDS=60 +LLM_SUMMARIZE_TIMEOUT_SECONDS=60 # --- Inbound auth -------------------------------------------------------- # Open WebUI adapter bearer token. Default: empty (leaves POST / open) diff --git a/README.md b/README.md index e434261..8699867 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ LLM Context Loader is a small HTTP service that turns URLs into markdown for LLM ## Current Shape -- **Pipelines:** `truncate` (default, no LLM), `clean-deterministic` (Firecrawl HTML + Readability + mdream, no LLM), `clean-llm` (Firecrawl + LLM), and `clean-combined` (Firecrawl HTML + Readability + mdream + LLM summarize), selected via `DEFAULT_PIPELINE`. +- **Pipelines:** `full` (default; full-featured with toggleable Docling, Firecrawl, Readability, and LLM passes) and `smoke` (zero-dependency HTTP fetch + aggressive markdown + truncation), selected via `DEFAULT_PIPELINE`. - **Source providers:** native HTTP fetch, Firecrawl, Docling. - **Content transformers:** `readability` (article HTML extraction), `mdream` (HTML to markdown). - **LLM providers:** OpenAI-compatible `/chat/completions`. @@ -64,7 +64,7 @@ docker compose -f compose.deploy.yaml up -d The deploy compose file requires `LLMC_IMAGE_TAG`. The example env file uses `latest`, but repeatable deployments should pin it to an immutable release tag. -The out-of-box `truncate` pipeline needs no provider config. Switch to `clean-llm` with `DEFAULT_PIPELINE=clean-llm` and the appropriate provider vars. The Compose files pass all variables through without failing early; the service validates only the active pipeline's providers at startup. If Firecrawl or the LLM server runs on the host, use a LAN address or `host.docker.internal` instead of `localhost`. +The `smoke` pipeline needs no provider config — it uses native HTTP fetch only. The Compose files pass all variables through without failing early; the service validates only the active pipeline's providers at startup. If Firecrawl or the LLM server runs on the host, use a LAN address or `host.docker.internal` instead of `localhost`. ## API diff --git a/compose.deploy.yaml b/compose.deploy.yaml index 81fbae5..e6e0629 100644 --- a/compose.deploy.yaml +++ b/compose.deploy.yaml @@ -11,7 +11,7 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l LOG_PRETTY: "${LOG_PRETTY:-auto}" # Active pipeline selected by HTTP adapters. - DEFAULT_PIPELINE: "${DEFAULT_PIPELINE:-truncate}" + DEFAULT_PIPELINE: "${DEFAULT_PIPELINE:-full}" # Pipeline concurrency and renderer selection. SOURCE_CONCURRENCY: "${SOURCE_CONCURRENCY:-1}" @@ -21,13 +21,20 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l DEBUG_XML_INCLUDE_SKIPPED: "${DEBUG_XML_INCLUDE_SKIPPED:-false}" OUTPUT_TARGET_CHARS: "${OUTPUT_TARGET_CHARS:-25000}" + # Step toggles (disabled by default). + DOCLING_ENABLED: "${DOCLING_ENABLED:-false}" + FIRECRAWL_ENABLED: "${FIRECRAWL_ENABLED:-false}" + READABILITY_ENABLED: "${READABILITY_ENABLED:-true}" + LLM_CLEAN_ENABLED: "${LLM_CLEAN_ENABLED:-false}" + LLM_SUMMARIZE_ENABLED: "${LLM_SUMMARIZE_ENABLED:-false}" + # Source provider. - SOURCE_PROVIDER: "${SOURCE_PROVIDER:-}" FIRECRAWL_BASE_URL: "${FIRECRAWL_BASE_URL:-}" FIRECRAWL_API_KEY: "${FIRECRAWL_API_KEY:-}" - FIRECRAWL_PARSE_PDF: "${FIRECRAWL_PARSE_PDF:-}" + FIRECRAWL_OPTIONS: "${FIRECRAWL_OPTIONS:-}" DOCLING_BASE_URL: "${DOCLING_BASE_URL:-}" DOCLING_API_KEY: "${DOCLING_API_KEY:-}" + DOCLING_OPTIONS: "${DOCLING_OPTIONS:-}" # Content transformers. MDREAM_CLEAN: "${MDREAM_CLEAN:-true}" @@ -44,10 +51,12 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l LLM_SAFETY_MARGIN_TOKENS: "${LLM_SAFETY_MARGIN_TOKENS:-128}" # Step thresholds and timeouts. - LOAD_SOURCE_TIMEOUT_SECONDS: "${LOAD_SOURCE_TIMEOUT_SECONDS:-20}" - CLEAN_TIMEOUT_SECONDS: "${CLEAN_TIMEOUT_SECONDS:-60}" - CLEAN_MIN_INPUT_CHARS: "${CLEAN_MIN_INPUT_CHARS:-1000}" - SUMMARIZE_TIMEOUT_SECONDS: "${SUMMARIZE_TIMEOUT_SECONDS:-60}" + FETCH_TIMEOUT_SECONDS: "${FETCH_TIMEOUT_SECONDS:-20}" + FIRECRAWL_TIMEOUT_SECONDS: "${FIRECRAWL_TIMEOUT_SECONDS:-20}" + DOCLING_TIMEOUT_SECONDS: "${DOCLING_TIMEOUT_SECONDS:-60}" + LLM_CLEAN_TIMEOUT_SECONDS: "${LLM_CLEAN_TIMEOUT_SECONDS:-60}" + LLM_CLEAN_MIN_INPUT_CHARS: "${LLM_CLEAN_MIN_INPUT_CHARS:-1000}" + LLM_SUMMARIZE_TIMEOUT_SECONDS: "${LLM_SUMMARIZE_TIMEOUT_SECONDS:-60}" # Per-adapter inbound bearer tokens. Empty means open route. OWUI_AUTH_TOKEN: "${OWUI_AUTH_TOKEN:-}" diff --git a/compose.yaml b/compose.yaml index 2530eeb..22b60e5 100644 --- a/compose.yaml +++ b/compose.yaml @@ -11,7 +11,7 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l LOG_PRETTY: "${LOG_PRETTY:-auto}" # Active pipeline selected by HTTP adapters. - DEFAULT_PIPELINE: "${DEFAULT_PIPELINE:-truncate}" + DEFAULT_PIPELINE: "${DEFAULT_PIPELINE:-full}" # Pipeline concurrency and renderer selection. SOURCE_CONCURRENCY: "${SOURCE_CONCURRENCY:-1}" @@ -21,13 +21,20 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l DEBUG_XML_INCLUDE_SKIPPED: "${DEBUG_XML_INCLUDE_SKIPPED:-false}" OUTPUT_TARGET_CHARS: "${OUTPUT_TARGET_CHARS:-25000}" + # Step toggles (disabled by default). + DOCLING_ENABLED: "${DOCLING_ENABLED:-false}" + FIRECRAWL_ENABLED: "${FIRECRAWL_ENABLED:-false}" + READABILITY_ENABLED: "${READABILITY_ENABLED:-true}" + LLM_CLEAN_ENABLED: "${LLM_CLEAN_ENABLED:-false}" + LLM_SUMMARIZE_ENABLED: "${LLM_SUMMARIZE_ENABLED:-false}" + # Source provider. - SOURCE_PROVIDER: "${SOURCE_PROVIDER:-}" FIRECRAWL_BASE_URL: "${FIRECRAWL_BASE_URL:-}" FIRECRAWL_API_KEY: "${FIRECRAWL_API_KEY:-}" - FIRECRAWL_PARSE_PDF: "${FIRECRAWL_PARSE_PDF:-}" + FIRECRAWL_OPTIONS: "${FIRECRAWL_OPTIONS:-}" DOCLING_BASE_URL: "${DOCLING_BASE_URL:-}" DOCLING_API_KEY: "${DOCLING_API_KEY:-}" + DOCLING_OPTIONS: "${DOCLING_OPTIONS:-}" # Content transformers. MDREAM_CLEAN: "${MDREAM_CLEAN:-true}" @@ -44,10 +51,12 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l LLM_SAFETY_MARGIN_TOKENS: "${LLM_SAFETY_MARGIN_TOKENS:-128}" # Step thresholds and timeouts. - LOAD_SOURCE_TIMEOUT_SECONDS: "${LOAD_SOURCE_TIMEOUT_SECONDS:-20}" - CLEAN_TIMEOUT_SECONDS: "${CLEAN_TIMEOUT_SECONDS:-60}" - CLEAN_MIN_INPUT_CHARS: "${CLEAN_MIN_INPUT_CHARS:-1000}" - SUMMARIZE_TIMEOUT_SECONDS: "${SUMMARIZE_TIMEOUT_SECONDS:-60}" + FETCH_TIMEOUT_SECONDS: "${FETCH_TIMEOUT_SECONDS:-20}" + FIRECRAWL_TIMEOUT_SECONDS: "${FIRECRAWL_TIMEOUT_SECONDS:-20}" + DOCLING_TIMEOUT_SECONDS: "${DOCLING_TIMEOUT_SECONDS:-60}" + LLM_CLEAN_TIMEOUT_SECONDS: "${LLM_CLEAN_TIMEOUT_SECONDS:-60}" + LLM_CLEAN_MIN_INPUT_CHARS: "${LLM_CLEAN_MIN_INPUT_CHARS:-1000}" + LLM_SUMMARIZE_TIMEOUT_SECONDS: "${LLM_SUMMARIZE_TIMEOUT_SECONDS:-60}" # Per-adapter inbound bearer tokens. Empty means open route. OWUI_AUTH_TOKEN: "${OWUI_AUTH_TOKEN:-}" diff --git a/config/llm-context-loader.yaml b/config/llm-context-loader.yaml index ae10795..c6e7586 100644 --- a/config/llm-context-loader.yaml +++ b/config/llm-context-loader.yaml @@ -4,13 +4,13 @@ schemaVersion: 1 httpAdapters: open-webui: type: open-webui - pipeline: ${DEFAULT_PIPELINE:-truncate} + pipeline: ${DEFAULT_PIPELINE:-full} config: auth: bearerToken: ${OWUI_AUTH_TOKEN:-} jina: type: jina - pipeline: ${DEFAULT_PIPELINE:-truncate} + pipeline: ${DEFAULT_PIPELINE:-full} config: auth: bearerToken: ${JINA_AUTH_TOKEN:-} @@ -40,24 +40,21 @@ sourceProviders: baseUrl: ${FIRECRAWL_BASE_URL:-} apiKey: ${FIRECRAWL_API_KEY:-} output: rawHtml - parsePdf: ${FIRECRAWL_PARSE_PDF:-true} + options: ${FIRECRAWL_OPTIONS:-} firecrawl-markdown: type: firecrawl config: baseUrl: ${FIRECRAWL_BASE_URL:-} apiKey: ${FIRECRAWL_API_KEY:-} output: markdown - onlyMainContent: true - stripBase64Images: true - parsePdf: ${FIRECRAWL_PARSE_PDF:-true} - docling-ocr: + options: ${FIRECRAWL_OPTIONS:-} + docling-default: type: docling config: baseUrl: ${DOCLING_BASE_URL:-} apiKey: ${DOCLING_API_KEY:-} output: markdown - doOcr: true - tableMode: accurate + options: ${DOCLING_OPTIONS:-} # Content transformers rewrite the body between representations (e.g. HTML to markdown). contentTransformers: @@ -89,39 +86,51 @@ llmProviders: safetyMarginTokens: ${LLM_SAFETY_MARGIN_TOKENS:-128} extraBody: {} -# Shipped pipelines. DEFAULT_PIPELINE selects which is active (truncate by default). +# Shipped pipelines. DEFAULT_PIPELINE selects which is active (full by default). pipelines: - truncate: + full: outputRenderer: ${DEFAULT_OUTPUT_RENDERER:-debug-xml} limiters: source: ${SOURCE_CONCURRENCY:-1} + process: ${PROCESS_CONCURRENCY:-5} + llm: ${LLM_CONCURRENCY:-1} steps: + - type: classify-url + name: classify + config: + rules: + - signal: binary_doc + extensionIn: [pdf, docx, doc, pptx, ppt, xlsx, xls] + - signal: binary_doc + pattern: "/pdf/\\d+(\\.\\d+)?($|\\?|#)" + - signal: code_host + anyHost: [github.com, gitlab.com, bitbucket.org] - type: load-source - name: fetch + name: fetch_docling concurrencyGroup: source - timeoutSeconds: ${LOAD_SOURCE_TIMEOUT_SECONDS:-20} + enabled: ${DOCLING_ENABLED:-false} + runIf: binary_doc + timeoutSeconds: ${DOCLING_TIMEOUT_SECONDS:-60} config: - provider: ${SOURCE_PROVIDER:-http-default} - - type: truncate - name: truncate + provider: docling-default + - type: load-source + name: fetch_firecrawl + concurrencyGroup: source + enabled: ${FIRECRAWL_ENABLED:-false} + timeoutSeconds: ${FIRECRAWL_TIMEOUT_SECONDS:-20} config: - targetChars: ${OUTPUT_TARGET_CHARS:-25000} - - clean-deterministic: - outputRenderer: ${DEFAULT_OUTPUT_RENDERER:-debug-xml} - limiters: - source: ${SOURCE_CONCURRENCY:-1} - process: ${PROCESS_CONCURRENCY:-5} - steps: + provider: firecrawl-html - type: load-source - name: fetch + name: fetch_http concurrencyGroup: source - timeoutSeconds: ${LOAD_SOURCE_TIMEOUT_SECONDS:-20} + timeoutSeconds: ${FETCH_TIMEOUT_SECONDS:-20} config: - provider: ${SOURCE_PROVIDER:-firecrawl-html} + provider: http-default - type: transform name: clean concurrencyGroup: process + enabled: ${READABILITY_ENABLED:-true} + skipIf: code_host config: transformer: readability-default target: text/html @@ -131,35 +140,19 @@ pipelines: config: transformer: mdream-convert target: text/markdown - - type: truncate - name: truncate - config: - targetChars: ${OUTPUT_TARGET_CHARS:-25000} - - clean-llm: - outputRenderer: ${DEFAULT_OUTPUT_RENDERER:-debug-xml} - limiters: - source: ${SOURCE_CONCURRENCY:-1} - llm: ${LLM_CONCURRENCY:-1} - steps: - - type: load-source - name: fetch - concurrencyGroup: source - timeoutSeconds: ${LOAD_SOURCE_TIMEOUT_SECONDS:-20} - config: - provider: ${SOURCE_PROVIDER:-firecrawl-markdown} - type: capture-urls name: capture_source_urls - concurrencyGroup: source + concurrencyGroup: process config: artifact: trusted-urls - type: llm-pass - name: clean + name: clean_llm concurrencyGroup: llm - timeoutSeconds: ${CLEAN_TIMEOUT_SECONDS:-60} + enabled: ${LLM_CLEAN_ENABLED:-false} + timeoutSeconds: ${LLM_CLEAN_TIMEOUT_SECONDS:-60} config: provider: llm-default - minInputChars: ${CLEAN_MIN_INPUT_CHARS:-1000} + minInputChars: ${LLM_CLEAN_MIN_INPUT_CHARS:-1000} outputReserveRatio: 1.0 templates: system: ../templates/clean.system.md @@ -167,6 +160,7 @@ pipelines: - type: verify-urls name: verify_after_clean concurrencyGroup: llm + enabled: ${LLM_CLEAN_ENABLED:-false} config: artifact: trusted-urls onHallucination: rollback @@ -174,7 +168,8 @@ pipelines: - type: llm-pass name: summarize concurrencyGroup: llm - timeoutSeconds: ${SUMMARIZE_TIMEOUT_SECONDS:-60} + enabled: ${LLM_SUMMARIZE_ENABLED:-false} + timeoutSeconds: ${LLM_SUMMARIZE_TIMEOUT_SECONDS:-60} config: provider: llm-default minInputChars: ${OUTPUT_TARGET_CHARS:-25000} @@ -187,6 +182,7 @@ pipelines: - type: verify-urls name: verify_after_summarize concurrencyGroup: llm + enabled: ${LLM_SUMMARIZE_ENABLED:-false} config: artifact: trusted-urls onHallucination: report @@ -196,56 +192,24 @@ pipelines: config: targetChars: ${OUTPUT_TARGET_CHARS:-25000} - clean-combined: + smoke: outputRenderer: ${DEFAULT_OUTPUT_RENDERER:-debug-xml} limiters: source: ${SOURCE_CONCURRENCY:-1} process: ${PROCESS_CONCURRENCY:-5} - llm: ${LLM_CONCURRENCY:-1} steps: - type: load-source name: fetch concurrencyGroup: source - timeoutSeconds: ${LOAD_SOURCE_TIMEOUT_SECONDS:-20} - config: - provider: ${SOURCE_PROVIDER:-firecrawl-html} - - type: transform - name: clean - concurrencyGroup: process + timeoutSeconds: ${FETCH_TIMEOUT_SECONDS:-20} config: - transformer: readability-default - target: text/html + provider: http-default - type: transform name: convert concurrencyGroup: process config: - transformer: mdream-convert + transformer: mdream-aggressive target: text/markdown - - type: capture-urls - name: capture_source_urls - concurrencyGroup: process - config: - artifact: trusted-urls - - type: llm-pass - name: summarize - concurrencyGroup: llm - timeoutSeconds: ${SUMMARIZE_TIMEOUT_SECONDS:-60} - config: - provider: llm-default - minInputChars: ${OUTPUT_TARGET_CHARS:-25000} - outputReserveChars: ${OUTPUT_TARGET_CHARS:-25000} - templates: - system: ../templates/summarize.system.md - user: ../templates/summarize.user.md - vars: - targetChars: ${OUTPUT_TARGET_CHARS:-25000} - - type: verify-urls - name: verify_after_summarize - concurrencyGroup: llm - config: - artifact: trusted-urls - onHallucination: report - maxReportedUrls: 50 - type: truncate name: truncate config: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 71d8a60..677ac54 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -70,7 +70,7 @@ The current built-ins are: - Source providers: `http`, `firecrawl`, `docling`. - Content transformers: `readability`, `mdream`. - LLM providers: `openai-chat`. -- Pipeline steps: `load-source`, `llm-pass`, `transform`, `truncate`, `capture-urls`, `verify-urls`. +- Pipeline steps: `classify-url`, `load-source`, `llm-pass`, `transform`, `truncate`, `capture-urls`, `verify-urls`. - Output renderers: `debug-xml`, `passthrough`. ## Dependency Boundaries @@ -119,7 +119,7 @@ Key pieces: - `StepResult` carries `status: "ok" | "skipped" | "degraded" | "failed"`, optional `reason`, `effects`, and `diagnostics`. Diagnostics are observability-only: they surface in the persisted `StepReport` and in renderers, but are never visible to later steps. - `StepOutcome` (`src/contracts/pipeline/report.ts`) is the compact, semantic view later steps see via `PipelineContext.outcomes`; inter-step coordination uses `signals` and `artifacts`, not diagnostics. `StepReport` extends it with timing and a mirrored diagnostics payload. - `applyStepEffects` applies effects on `ok` or `degraded` status (in body, signal, artifact order); `skipped` and `failed` results never apply effects. A `degraded` step may still carry effects, for example a quality gate rolling the body back to its previous version. -- `PipelineOrchestrator` runs one compiled pipeline for one URL, applies per-step timeouts, acquires concurrency-group limiters, records reports, and returns detached signal/artifact snapshots. +- `PipelineOrchestrator` runs one compiled pipeline for one URL, evaluates `runIf`/`skipIf` gates before each step (recording `skipped` with reason without running the step when gated), applies per-step timeouts, acquires concurrency-group limiters, records reports, and returns detached signal/artifact snapshots. - `PipelineRunner` (`src/core/pipeline/runner.ts`) wraps the orchestrator and applies the pipeline's configured renderer, including a synthetic failure report for adapter-level per-URL failures. - `OutputRenderer` (`src/contracts/extensions/output-renderer.ts`) is the runtime rendering port. diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index f76c52e..e431417 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -53,27 +53,23 @@ fail startup. The placeholders below are grouped by the part of the pipeline they configure. -### Source provider selection - -| Variable | Default / behavior | Purpose | -| ----------------- | -------------------------- | ---------------------------------------------------- | -| `SOURCE_PROVIDER` | selected pipeline fallback | Optional source-provider override for `load-source`. | - -Leave `SOURCE_PROVIDER` empty to use the selected pipeline's fallback: `http-default` for `truncate`, `firecrawl-html` for `clean-deterministic` and `clean-combined`, and `firecrawl-markdown` for `clean-llm`. Set it only when intentionally overriding that choice. - ### Source provider (Firecrawl) -| Variable | Default / behavior | Purpose | -| -------------------- | ------------------ | ------------------------------------------------------------------------------------------------------------------ | -| `FIRECRAWL_BASE_URL` | empty | Firecrawl base URL. Required when any Firecrawl-based provider is active (`firecrawl-markdown`, `firecrawl-html`). | -| `FIRECRAWL_API_KEY` | empty | Optional Firecrawl bearer token. | +| Variable | Default / behavior | Purpose | +| -------------------- | ------------------ | ---------------------------------------------------------------------------------------------------------------------------- | +| `FIRECRAWL_ENABLED` | `false` | Enable Firecrawl source loading (step `fetch_firecrawl`). When set to `true`, `FIRECRAWL_BASE_URL` must also be set. | +| `FIRECRAWL_BASE_URL` | empty | Firecrawl base URL. Required when an enabled step references a Firecrawl provider. | +| `FIRECRAWL_API_KEY` | empty | Optional Firecrawl bearer token. | +| `FIRECRAWL_OPTIONS` | empty | JSON object of Firecrawl scrape options. Opaque passthrough merged into the request body. See CUSTOMIZATION.md for examples. | ### Source provider (Docling) -| Variable | Default / behavior | Purpose | -| ------------------ | ------------------ | ------------------------------------------------------------------------------------------ | -| `DOCLING_BASE_URL` | empty | Docling Serve base URL. Required only when a pipeline referencing `docling-ocr` is active. | -| `DOCLING_API_KEY` | empty | Optional Docling API key. | +| Variable | Default / behavior | Purpose | +| ------------------ | ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------- | +| `DOCLING_ENABLED` | `false` | Enable Docling source loading (step `fetch_docling`, gated by `runIf: binary_doc`). When set to `true`, `DOCLING_BASE_URL` must also be set. | +| `DOCLING_BASE_URL` | empty | Docling Serve base URL. Required when an enabled step references `docling-default`. | +| `DOCLING_API_KEY` | empty | Optional Docling API key. | +| `DOCLING_OPTIONS` | empty | JSON object of Docling convert options. Opaque passthrough merged into the convert body. See CUSTOMIZATION.md for examples. | ### Content transformer (mdream) @@ -91,24 +87,29 @@ Leave `SOURCE_PROVIDER` empty to use the selected pipeline's fallback: `http-def ### LLM provider (OpenAI-compatible) -| Variable | Default / behavior | Purpose | -| -------------------------- | ------------------ | --------------------------------------------------------------------------------------------------------------------------- | -| `LLM_BASE_URL` | empty | OpenAI-compatible API base URL, usually ending in `/v1`. Required only when a pipeline referencing `llm-default` is active. | -| `LLM_API_KEY` | empty | Optional LLM bearer token. | -| `LLM_MODEL` | empty | Model identifier sent to chat completions. Required only when a pipeline referencing `llm-default` is active. | -| `LLM_CONTEXT_TOKENS` | empty in YAML | Optional model context window in tokens. Empty disables the context-fit gate. | -| `LLM_CHARS_PER_TOKEN` | `3.5` | Conservative chars-per-token estimator used for the context-fit gate. | -| `LLM_SAFETY_MARGIN_TOKENS` | `128` | Extra tokens reserved for chat-template framing and estimator drift. | +| Variable | Default / behavior | Purpose | +| -------------------------- | ------------------ | --------------------------------------------------------------------------------------------------------------------------------------- | +| `LLM_CLEAN_ENABLED` | `false` | Enable the clean LLM pass (steps `clean_llm` + `verify_after_clean`). When `true`, `LLM_BASE_URL` and `LLM_MODEL` are required. | +| `LLM_SUMMARIZE_ENABLED` | `false` | Enable the summarize LLM pass (steps `summarize` + `verify_after_summarize`). When `true`, `LLM_BASE_URL` and `LLM_MODEL` are required. | +| `LLM_BASE_URL` | empty | OpenAI-compatible API base URL, usually ending in `/v1`. Required when an enabled step references `llm-default`. | +| `LLM_API_KEY` | empty | Optional LLM bearer token. | +| `LLM_MODEL` | empty | Model identifier sent to chat completions. Required when an enabled step references `llm-default`. | +| `LLM_CONTEXT_TOKENS` | empty in YAML | Optional model context window in tokens. Empty disables the context-fit gate. | +| `LLM_CHARS_PER_TOKEN` | `3.5` | Conservative chars-per-token estimator used for the context-fit gate. | +| `LLM_SAFETY_MARGIN_TOKENS` | `128` | Extra tokens reserved for chat-template framing and estimator drift. | ### Pipeline output and step thresholds -| Variable | Default / behavior | Purpose | -| ----------------------------- | ------------------ | ----------------------------------------------------------- | -| `OUTPUT_TARGET_CHARS` | `25000` | Desired maximum characters returned by the active pipeline. | -| `LOAD_SOURCE_TIMEOUT_SECONDS` | `20` | Per-call timeout for the source-loading step. | -| `CLEAN_MIN_INPUT_CHARS` | `1000` | Minimum body characters before the clean LLM pass runs. | -| `CLEAN_TIMEOUT_SECONDS` | `60` | Per-call timeout for the clean LLM pass. | -| `SUMMARIZE_TIMEOUT_SECONDS` | `60` | Per-call timeout for the summarize LLM pass. | +| Variable | Default / behavior | Purpose | +| ------------------------------- | ------------------ | ---------------------------------------------------------------------------------------------------------- | +| `OUTPUT_TARGET_CHARS` | `25000` | Desired maximum characters returned by the active pipeline. | +| `READABILITY_ENABLED` | `true` | Enable Readability HTML-to-article extraction (step `clean`). Local operation; no external service needed. | +| `FETCH_TIMEOUT_SECONDS` | `20` | Per-call timeout for the native HTTP source-loading step. | +| `FIRECRAWL_TIMEOUT_SECONDS` | `20` | Per-call timeout for the Firecrawl source-loading step. | +| `DOCLING_TIMEOUT_SECONDS` | `60` | Per-call timeout for the Docling OCR source-loading step. | +| `LLM_CLEAN_MIN_INPUT_CHARS` | `1000` | Minimum body characters before the clean LLM pass runs. | +| `LLM_CLEAN_TIMEOUT_SECONDS` | `60` | Per-call timeout for the clean LLM pass. | +| `LLM_SUMMARIZE_TIMEOUT_SECONDS` | `60` | Per-call timeout for the summarize LLM pass. | ### Concurrency @@ -148,6 +149,8 @@ Substitution applies recursively to YAML string values. Non-string YAML values a Substitution itself is string-only. Numeric and boolean fields recover their types during schema parsing, and blank env values use the field's schema default when that field has one. Blank strings remain meaningful for token fields such as `FIRECRAWL_API_KEY`, `DOCLING_API_KEY`, `LLM_API_KEY`, `OWUI_AUTH_TOKEN`, and `JINA_AUTH_TOKEN`, where empty means no token. +Some opaque record fields (`extraBody`, Docling `options`, Firecrawl `options`, future providers) also accept a whole JSON-string blob from a single env var — see [Opaque Passthrough Fields](CUSTOMIZATION.md#opaque-passthrough-fields) in customization for details. + Docker Compose performs its own interpolation before the container starts. The shipped Compose files pass all provider variables through with empty defaults (`${VAR:-}`) so the container always starts and the Node process validates only the active pipeline's providers at startup. ## Authentication diff --git a/docs/CUSTOMIZATION.md b/docs/CUSTOMIZATION.md index 2e020b8..4a293fb 100644 --- a/docs/CUSTOMIZATION.md +++ b/docs/CUSTOMIZATION.md @@ -35,7 +35,7 @@ HTTP adapters also name the pipeline they expose: ```yaml some-adapter: type: open-webui - pipeline: ${DEFAULT_PIPELINE:-truncate} + pipeline: ${DEFAULT_PIPELINE:-full} config: {} ``` @@ -44,7 +44,7 @@ omitted, a pipeline is active only when at least one HTTP adapter references it. `enabled: false` parks it. Setting `enabled: true` pins it active regardless of adapter references. The shipped pipelines leave `enabled` omitted, so `DEFAULT_PIPELINE` alone drives the active set. Active-set pipelines are compiled and validated at startup; inactive ones are -skipped (their providers are never built). +excluded from compilation. Pipeline steps include common orchestration fields plus a type-specific `config` block: @@ -53,11 +53,58 @@ Pipeline steps include common orchestration fields plus a type-specific `config` name: clean concurrencyGroup: llm timeoutSeconds: 90 + enabled: false + runIf: binary_doc + skipIf: code_host config: {} ``` +`enabled` is an optional compile-time exclusion toggle (default `true`). When `enabled: false`, +the step is completely excluded from compilation — it is never name-validated, never +config-parsed, and never constructed. Its linked provider (if any) is never required, so +missing provider configuration does not cause a startup failure. This differs from `runIf` +and `skipIf`, which are runtime gates: a gated step is still compiled and appears in the +footer as skipped. + +`runIf` and `skipIf` are optional per-step gates (`string` or structured predicate — see +[Conditional step execution](#conditional-step-execution) below). When both are present, +`runIf` is checked first. + Step names must be unique within a pipeline and must be valid diagnostic names: lowercase letters, numbers, and underscores, starting with a lowercase letter. +## Conditional Step Execution + +Pipeline steps support signal-based gating via `runIf` and `skipIf`. When the gate evaluates to false (or true, for `skipIf`), the step is recorded as `skipped` with a reserved reason (`run_if_unmet` or `skip_if_met`) without running. A gate is a predicate evaluated against runtime signals emitted by earlier steps (such as `classify-url`). The simplest form is a bare signal name: + +```yaml +runIf: binary_doc # runs only when signal exists and is truthy +skipIf: code_host # skips when signal exists and is truthy +``` + +Structured combinators let you build compound conditions: + +```yaml +runIf: + all: + - binary_doc + - code_host + +skipIf: + any: + - is_pdf + - is_office_doc + +runIf: + not: binary_doc + +skipIf: + any: + - all: [binary_doc, code_host] + - not: processed +``` + +`all: []` is always true. `any: []` is always false. Truthy values: present and not `false`, `0`, or `""`. + ## HTTP Adapters HTTP adapters register inbound routes and bind them to one pipeline. @@ -139,27 +186,29 @@ config: baseUrl: ${FIRECRAWL_BASE_URL} apiKey: ${FIRECRAWL_API_KEY:-} output: markdown - onlyMainContent: true - stripBase64Images: true - parsePdf: true + options: ${FIRECRAWL_OPTIONS:-} ``` -| Knob | Values | Default | Purpose | -| ------------------- | --------------------------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `output` | `markdown` \| `html` \| `rawHtml` | `markdown` | Which format Firecrawl returns. `html` is cleaned main-content HTML. `rawHtml` is "raw" — full JS-rendered DOM for web pages, viewer `` wrapper for images, and bare extracted text for PDFs (use for Readability testing). | -| `onlyMainContent` | bool | `true` | When enabled Firecrawl extracts the main page content and strips headers, nav, footers. No-op for `rawHtml`. | -| `stripBase64Images` | bool | `true` | Maps to `removeBase64Images` — replaces inline data URIs with short placeholders in markdown output. No-op for `html`/`rawHtml`. | -| `parsePdf` | bool | `true` | Configurable via `FIRECRAWL_PARSE_PDF`. When `true`, Firecrawl parses PDF files to extracted text. When `false`, raw PDF bytes are returned as a binary body (`application/pdf`) — the pipeline fails with `unconverted_binary` until a PDF converter (e.g. Docling) is wired up. | +| Knob | Values | Default | Purpose | +| --------- | --------------------------------- | ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `output` | `markdown` \| `html` \| `rawHtml` | `markdown` | Which format Firecrawl returns. `html` is cleaned main-content HTML. `rawHtml` is "raw" — full JS-rendered DOM for web pages, viewer `` wrapper for images, and bare extracted text for PDFs (use for Readability testing). | +| `options` | object | `{}` | Opaque passthrough merged into the Firecrawl `/v2/scrape` body. See [Opaque Passthrough Fields](#opaque-passthrough-fields). | + +`output` stays typed because the provider depends on it: it selects the requested format, decides which response field to read (`data.markdown`, `data.html`, `data.rawHtml`), and sets the returned media type. The `formats` field in the request body is provider-managed and cannot be overridden through `options`. + +**Opaque `options`.** Every other Firecrawl scrape knob flows through `options` verbatim — `onlyMainContent`, `removeBase64Images`, `waitFor`, `timeout`, `maxAge`, `includeTags`, `excludeTags`, `blockAds`, `proxy`, `location`, `headers`, `mobile`, and all others. Omitted knobs use Firecrawl's server defaults (`onlyMainContent:true`, `removeBase64Images:true`, `parsers:["pdf"]`, etc.), so an empty `options` produces the same behavior as the previous explicit defaults. + +**PDF passthrough (raw bytes).** To return raw PDF bytes instead of parsed text (e.g. for Docling), set `parsers: []` in options: `options: '{"parsers":[]}'`. The provider detects the passthrough by base64-decoding the content and verifying the `%PDF` magic header, so the binary response is truthful regardless of the source's `Content-Type` header. The provider calls `/v2/scrape`. Upstream HTTP, parse, empty, and network failures are converted to degradable upstream errors. Title is read from `data.metadata.title` (format-independent). The media type of returned bodies is derived truthfully: -| `output` | HTML / docx source | image source | PDF source (parsePdf:true) | -| -------- | ------------------ | --------------- | -------------------------- | -| markdown | `text/markdown` | `text/markdown` | `text/markdown` | -| html | `text/html` | `text/html` | `text/html` (wrapped) | -| rawHtml | `text/html` | `text/html` | **`text/plain`** | +| `output` | HTML / docx source | image source | PDF source (parsers default `["pdf"]`) | +| -------- | ------------------ | --------------- | -------------------------------------- | +| markdown | `text/markdown` | `text/markdown` | `text/markdown` | +| html | `text/html` | `text/html` | `text/html` (wrapped) | +| rawHtml | `text/html` | `text/html` | **`text/plain`** | -`rawHtml` is "raw": for web pages and images it returns HTML, but for PDFs it returns bare extracted text. The provider detects this by looking at whether the content starts with an HTML tag — if not, it labels it `text/plain` so downstream transformers (which gate on `isHtmlMediaType`) skip it correctly. When `parsePdf:false`, PDFs are returned as binary (`application/pdf`) from base64-decoded raw bytes supplied by Firecrawl's empty-`parsers` mode. +`rawHtml` is "raw": for web pages and images it returns HTML, but for PDFs it returns bare extracted text. The provider detects this by looking at whether the content starts with an HTML tag — if not, it labels it `text/plain` so downstream transformers (which gate on `isHtmlMediaType`) skip it correctly. ### `docling` @@ -168,17 +217,19 @@ config: baseUrl: ${DOCLING_BASE_URL} apiKey: ${DOCLING_API_KEY:-} output: markdown - doOcr: true - tableMode: accurate + options: ${DOCLING_OPTIONS:-} ``` -| Knob | Values | Default | Purpose | -| ----------- | -------------------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `output` | `markdown` \| `html` | `markdown` | Which format Docling returns. `html` is the Docling HTML serializer output (polished semantic HTML with inline CSS — does NOT provide raw/rough HTML). | -| `doOcr` | bool | `true` | Run OCR on scanned documents and images within PDFs. | -| `tableMode` | `fast` \| `accurate` | `accurate` | Table extraction quality. `accurate` is slower but better for complex layouts. | +| Knob | Values | Default | Purpose | +| --------- | -------------------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `output` | `markdown` \| `html` | `markdown` | Which format Docling returns. `html` is the Docling HTML serializer output (polished semantic HTML with inline CSS — does NOT provide raw/rough HTML). | +| `options` | object | `{}` | Opaque passthrough merged into the Docling convert `options` body. See [Opaque Passthrough Fields](#opaque-passthrough-fields). | -The provider calls `POST /v1/convert/source`. Title is read from `document.json_content.name` (the JSON format is always requested internally regardless of the `output` setting). Upstream HTTP, parse, empty, and network failures are converted to degradable upstream errors. `output: markdown` returns `text/markdown`; `output: html` returns `text/html`. +`output` stays typed because the provider depends on it: it selects the requested formats, decides whether to read `md_content` or `html_content`, and sets the returned media type. The JSON format is always requested internally (for the title from `document.json_content.name`), so `to_formats` is provider-managed and cannot be overridden through `options`. + +**Opaque `options`.** Every other Docling convert knob flows through `options` verbatim. The provider-managed `to_formats` value is applied after `options` and cannot be overridden, because the provider depends on it to read the response correctly. + +The provider calls `POST /v1/convert/source`. Title is read from `document.json_content.name`. Upstream HTTP, parse, empty, and network failures are converted to degradable upstream errors. `output: markdown` returns `text/markdown`; `output: html` returns `text/html`. ## Content Transformers @@ -249,7 +300,7 @@ config: The provider joins `baseUrl` with `/chat/completions`; for OpenAI-compatible servers this usually means configuring a `/v1` base URL. It sends `model` and `messages`, using only `system` and `user` roles. `extraBody` is spread into the request body after the standard fields, so it can provide sampler or server-specific parameters. -Values inside `extraBody` are opaque YAML values. YAML booleans and numbers stay typed, but env placeholders inside `extraBody` substitute as strings. +`extraBody` is an opaque passthrough field; see [Opaque Passthrough Fields](#opaque-passthrough-fields). One provider instance corresponds to one model. To use several models against the same server, declare additional `openai-chat` instances with different names and `model` values. @@ -259,6 +310,26 @@ Context-fit fields are optional and own the char-to-token conversion: - `charsPerToken` — conservative chars-per-token estimator. Lower values reject more aggressively. Default `3.5`. This is a character-based approximation rather than a real tokenizer. - `safetyMarginTokens` — extra tokens reserved for chat-template framing or estimator drift. Default `0` in the provider schema; the bundled YAML defaults it to `128` via `LLM_SAFETY_MARGIN_TOKENS`. Increase if the model still rejects prompts the gate accepts. +## Opaque Passthrough Fields + +Several providers expose an opaque record field that is merged verbatim into the outgoing request body — `extraBody` (openai-chat), `options` (docling), `options` (firecrawl), and future source providers. These records support **three authoring modes**: + +| Mode | Example | Behavior | +| -------------------------- | -------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| Inline YAML object | `options:\n do_ocr: true\n table_mode: accurate` | Typed YAML values (booleans stay booleans, etc.) | +| Single env var (JSON blob) | `options: ${DOCLING_OPTIONS:-}` | The env value is `JSON.parse`'d into an object; blank/unset → `{}` | +| Mixed (not recommended) | `options:\n do_ocr: ${ENV_BOOL:-true}` | Env placeholders substitute as **strings**, not their original type. Prefer one of the two modes above. | + +The single-env-var mode is convenient when you have many keys or inconsistent per-deployment values. Your env file holds the entire object: + +```dotenv +DOCLING_OPTIONS='{"do_ocr":true,"table_mode":"accurate"}' +``` + +The JSON blob is parsed once by the shared `jsonStringAsObjectOrUndefined` preprocessor before the schema validates it as a record. Nested values (`picture_description_api`) are real objects — unlike Open WebUI's approach, there is **no double-encoding**. + +All opaque fields use the same preprocessor, so the three authoring modes and their limitation (env placeholders inside inline YAML objects substitute as strings) apply consistently everywhere. + ## Pipelines A pipeline chooses one output renderer, declares limiter groups, and lists steps in order. @@ -275,17 +346,40 @@ pipelines: `outputRenderer` references a named output renderer instance. `limiters` declares concurrency group names and their maximum concurrency. A step can opt into a group with `concurrencyGroup`. -Adjacent steps that share the same `concurrencyGroup` share one limiter acquisition. In the `clean-llm` pipeline, `llm-pass(clean)`, `verify_after_clean`, `llm-pass(summarize)`, and `verify_after_summarize` all use the `llm` group, so a URL holds one LLM workflow slot across both passes and their URL checks. Inserting a step with a different (or no) group between them splits that acquisition. +Adjacent steps that share the same `concurrencyGroup` share one limiter acquisition. In the `full` pipeline, `llm-pass(clean_llm)`, `verify_after_clean`, `llm-pass(summarize)`, and `verify_after_summarize` all use the `llm` group, so a URL holds one LLM workflow slot across both passes and their URL checks. Inserting a step with a different (or no) group between them splits that acquisition. `timeoutSeconds` applies to both limiter waiting and step execution. For one adjacent concurrency block, limiter waiting uses the longest `timeoutSeconds` value in that block. ## Steps +### `classify-url` + +```yaml +config: + rules: + - signal: binary_doc + extensionIn: [pdf, docx, doc, pptx, ppt, xlsx, xls] + - signal: code_host + anyHost: [github.com, gitlab.com, bitbucket.org] + - signal: is_html_article + pattern: "/articles/\\d+" +``` + +Inspects the input URL against a list of rules and emits boolean signals (`true`) for each match. Placed as the first step in a pipeline, `classify-url` lets downstream steps react through `runIf`/`skipIf`. Each rule must have exactly one matcher: + +| Matcher | Matches when | +| ------------- | ----------------------------------------------------------------------------- | +| `pattern` | URL matches the regex pattern | +| `anyHost` | URL hostname equals one of the listed hosts (or is a subdomain, www‑stripped) | +| `extensionIn` | URL pathname ends with one of the listed extensions (case‑insensitive) | + +Multiple rules may share the same `signal` name for natural OR semantics. The `matched` diagnostic attribute lists all matched signal names (space‑separated) or `"none"`. + ### `load-source` ```yaml config: - provider: ${SOURCE_PROVIDER:-http-default} + provider: http-default ``` Loads the initial body from a named source provider. If a body already exists, the step skips with `body_present`. @@ -398,36 +492,37 @@ Shipped defaults: `verify_after_clean` uses `rollback` with `maxReportedUrls: 0` ## Shipped Pipelines -The config ships four pipelines, selected via `DEFAULT_PIPELINE` (defaults to `truncate`). - -### `truncate` (default) - -```text -load-source(http-default) -> truncate -``` - -Fetches the URL directly and truncates to budget. No external services required. +The config ships two pipelines, selected via `DEFAULT_PIPELINE` (defaults to `full`). -### `clean-deterministic` +### `full` (default) ```text -load-source(firecrawl-html) -> transform(clean via readability) -> transform(convert via mdream) -> truncate +classify-url -> load-source(docling-default)* -> load-source(firecrawl-html)* -> load-source(http-default) -> transform(clean via readability)* -> transform(convert via mdream) -> capture-urls -> llm-pass(clean_llm)* -> verify-urls(rollback)* -> llm-pass(summarize)* -> verify-urls(report)* -> truncate + * toggleable via env var (see below) ``` -Loads Firecrawl raw HTML, extracts article HTML with `readability` when suitable, converts HTML to markdown with `mdream`, then truncates. No LLM passes. Pipeline fallback: `firecrawl-html`. Select with `DEFAULT_PIPELINE=clean-deterministic`. +The full-featured pipeline. Uses `classify-url` for binary-doc and code-host detection, tries +Docling OCR for binary documents, tries Firecrawl, then falls back to native HTTP fetch, +extracts article content with Readability, converts to markdown, captures URLs, and runs +optional LLM passes (clean and/or summarize) with URL-hallucination verification. -### `clean-llm` +Selected steps can be toggled via environment variables: -```text -load-source(firecrawl-markdown) -> capture-urls -> llm-pass(clean) -> verify-urls(rollback) -> llm-pass(summarize) -> verify-urls(report) -> truncate -``` +| Step(s) | Env toggle | Default | External dependency required when enabled | +| --------------------- | ----------------------- | ------- | ------------------------------------------------------ | +| `fetch_docling` | `DOCLING_ENABLED` | false | `DOCLING_BASE_URL` (+ optional `DOCLING_API_KEY`) | +| `fetch_firecrawl` | `FIRECRAWL_ENABLED` | false | `FIRECRAWL_BASE_URL` (+ optional `FIRECRAWL_API_KEY`) | +| `clean` (readability) | `READABILITY_ENABLED` | true | _(local; no external service needed)_ | +| `clean_llm` + verify | `LLM_CLEAN_ENABLED` | false | `LLM_BASE_URL`, `LLM_MODEL` (+ optional `LLM_API_KEY`) | +| `summarize` + verify | `LLM_SUMMARIZE_ENABLED` | false | `LLM_BASE_URL`, `LLM_MODEL` (+ optional `LLM_API_KEY`) | -The original reference pipeline. Loads markdown from a Firecrawl instance, runs clean and summarize LLM passes with URL-hallucination gates, then truncates. Pipeline fallback: `firecrawl-markdown`. Select with `DEFAULT_PIPELINE=clean-llm`. +Select with `DEFAULT_PIPELINE=full`. -### `clean-combined` +### `smoke` ```text -load-source(firecrawl-html) -> transform(clean via readability) -> transform(convert via mdream) -> capture-urls -> llm-pass(summarize) -> verify-urls(report) -> truncate +load-source(http-default) -> transform(convert via mdream-aggressive) -> truncate ``` -Combines the deterministic HTML-to-markdown path with an LLM summarize pass. Pipeline fallback: `firecrawl-html`. Select with `DEFAULT_PIPELINE=clean-combined`. +Minimal test pipeline with zero external dependencies. Native HTTP fetch, aggressive markdown +conversion, and truncation. Select with `DEFAULT_PIPELINE=smoke`. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 8a877ff..63d7cd8 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -13,8 +13,7 @@ Implemented today: - `POST /` for Open WebUI's external web-loader contract. - `GET /r/` and `GET /r?url=` for limited Jina Reader-style URL-to-markdown compatibility. -- Firecrawl source provider using `/v2/scrape` with markdown, html, and rawHtml output. -- Docling source provider (experimental, PDFs and Office docs → markdown via Docling Serve API). +- Firecrawl and Docling source providers. - Deterministic content transformers: Readability (article extraction) and mdream (HTML → markdown). - Optional LLM clean stage, optional LLM summarize stage, final truncation, and XML diagnostic footer. - YAML-driven configuration for pipelines, providers, content transformers, output renderers, and HTTP adapters. @@ -22,27 +21,17 @@ Implemented today: Current provider implementations are intentionally few. The interfaces exist so replacements can be added without rewriting the use case. -## Direction - -Deterministic extraction (readability + mdream) handles most pages in milliseconds. The next step is to make the pipeline self-aware: route through deterministic or LLM paths based on URL heuristics and per-step signals, reserving LLM passes for pages that need them. - ## Coming soon -1. **Conditional step execution based on signals.** Engine-level `skipIfSignal` / `runIfSignal` configuration on pipeline steps. When a signal name is set and the condition is met, the step is skipped entirely without running. -2. **URL heuristic step to classify input.** A pipeline step that matches the URL against patterns (path, domain, expected content type) and sets runtime signals for downstream steps. Use the feature to avoid Readibility step on GitHub domain etc. -3. **Route binary content to Docling.** Use the Docling source provider based on URL heuristics as an alternative to Firecrawl for PDFs and other binary content. -4. **Keep summaries honest.** Extend the existing URL quality gate from all-or-nothing rejection into a deterministic repair pass, and implement fenced-code-block verification/repair. - -A couple of polish items ride along: - -- **Diagnostics footer cleanup.** Consolidate the `` payload for consistency and readability. -- **Better defaults and prompts.** Test and tune the shipped defaults and prompts. +1. **Playwright source provider** to remove the hard dependency on a running Firecrawl instance for HTML pages. +2. **LLM pass auto-repair.** Repair URLs and code blocks, instead of mere detection. +3. **Diagnostics footer cleanup.** Consolidate the `` payload for consistency and readability. +4. **Better defaults and prompts.** Test and tune what ships. ## Short term -Near-term additions once the pivot above settles. +Near-term additions once the work above settles. -- **Playwright source provider** to remove the hard dependency on a running Firecrawl instance for HTML pages. - **Docling content transformer** to convert PDFs, Office documents, and other document-type files to markdown. - **Crawl4AI source provider** as a potentially better alternative to Firecrawl. diff --git a/docs/agents/extension-authoring.md b/docs/agents/extension-authoring.md index e0b6bfe..df3b54d 100644 --- a/docs/agents/extension-authoring.md +++ b/docs/agents/extension-authoring.md @@ -31,7 +31,7 @@ const schema = z.object({ }); ``` -Use `emptyStringAsUndefined` around numeric fields with defaults or `.optional()` so blank env values do not silently become `0`. Use `booleanStringAsBooleanOrUndefined` instead of `z.coerce.boolean()`; JavaScript truthiness would parse `"false"` as `true`. Leave string secrets and tokens as strings when blank is meaningful. Leave opaque payloads such as provider-specific `extraBody` as YAML values; do not infer nested scalar types unless the field has a real schema. +Use `emptyStringAsUndefined` around numeric fields with defaults or `.optional()` so blank env values do not silently become `0`. Use `booleanStringAsBooleanOrUndefined` instead of `z.coerce.boolean()`; JavaScript truthiness would parse `"false"` as `true`. Leave string secrets and tokens as strings when blank is meaningful. Use `jsonStringAsObjectOrUndefined` for opaque record fields such as provider-specific `extraBody` or `options`, so they support inline YAML objects and single env-var JSON blobs without inferring nested scalar types. ## Construction Dependencies diff --git a/docs/agents/smoke-testing.md b/docs/agents/smoke-testing.md index 240bd51..aa79b6c 100644 --- a/docs/agents/smoke-testing.md +++ b/docs/agents/smoke-testing.md @@ -30,21 +30,25 @@ $env:SEARX_BASE = 'http://:' ./scripts/smoke-jina.ps1 -Urls @('https://example.com/article', 'https://example.com/file.pdf') ``` -## Choosing a Source Provider +## Choosing a Pipeline and Toggle Set -`SOURCE_PROVIDER` selects one of the source providers declared in the YAML config. Read the config for the current names, then run the set against each backend you care about. The provider categories behave differently, which is the usual "is this broken or expected?" question: +The shipped pipelines are `full` (default) and `smoke`; `DEFAULT_PIPELINE` selects the one +exposed by the HTTP adapters. External-dependent steps are toggled off by default via env vars +— set them to `true` to enable, and provide their required service URLs. The provider +categories behave differently: -- **Native fetch** — requests the URL directly, no external service. Content it cannot turn into text (PDFs, images) returns a failed result with a binary-rejection error. -- **External scraper** — hands fetching and cleanup to an external service that handles many content types, including PDFs. Reads its base URL from the environment. -- **Document converter** — converts uploaded documents (PDF, Office, images) to markdown. It does not scrape generic HTML pages; those come back as an empty-source failure. Pair it with document URLs. - -Providers that depend on an external service read the hostname from the environment — never hard-code hosts in the doc or scripts. +- **Native HTTP fetch** — requests the URL directly, no external service. Always enabled in + `full` (last-resort fallback after Docling and Firecrawl) and in `smoke`. +- **Firecrawl** — external scraper. Toggle with `FIRECRAWL_ENABLED=true` + `FIRECRAWL_BASE_URL`. +- **Docling OCR** — document-to-markdown converter. Toggle with `DOCLING_ENABLED=true` + + `DOCLING_BASE_URL`. Gated by `runIf: binary_doc`. +- **LLM passes** — clean and/or summarize. Toggle with `LLM_CLEAN_ENABLED=true` / + `LLM_SUMMARIZE_ENABLED=true` + `LLM_BASE_URL` + `LLM_MODEL`. ## Running Locally (Node) ```powershell -# SOURCE_PROVIDER is read once at startup — set it BEFORE starting the server. -$env:SOURCE_PROVIDER = '' +# Step toggles and provider URLs are read once at startup — set them BEFORE starting. npm run dev # In a second terminal: @@ -60,13 +64,13 @@ Get-Process -Name node -ErrorAction SilentlyContinue | Stop-Process -Force -ErrorAction SilentlyContinue ``` -Changing `$env:SOURCE_PROVIDER` while the server runs has no effect — restart it. +Changing env vars while the server runs has no effect — restart it. ## Running in Docker ```powershell # Compose reads variables from the .env FILE. Terminal $env: vars are NOT passed in. -# Edit SOURCE_PROVIDER in .env first, then: +# Edit toggles and provider URLs in .env first, then: docker compose build # only when source code changed docker compose up -d ./scripts/smoke-jina.ps1 -UrlFile scripts/out/urls-smoke.txt -HealthTimeoutSec 30 -Concurrency 2 @@ -109,7 +113,7 @@ A `result="failed"` with `final_length=0` does not imply a pipeline bug by itsel ## Common Traps - [ ] `SEARX_BASE` exported before `gather-urls.ps1`? It falls back to localhost otherwise. -- [ ] `SOURCE_PROVIDER` set **before** `npm run dev`? It is read once at startup. +- [ ] Step toggles (`FIRECRAWL_ENABLED`, `DOCLING_ENABLED`, `LLM_CLEAN_ENABLED`, `LLM_SUMMARIZE_ENABLED`) set **before** `npm run dev`? They are read once at startup. - [ ] `DEFAULT_PIPELINE` set **before** `npm run dev`? It is also read once at startup. - [ ] For Docker, edited `.env` rather than a terminal `$env:` var? Compose only reads the file. - [ ] Previous server stopped before switching providers? Otherwise the port stays taken. @@ -126,6 +130,6 @@ A `result="failed"` with `final_length=0` does not imply a pipeline bug by itsel ## Cleanup -- Restore `.env` to its original `SOURCE_PROVIDER`. +- Restore `.env` to its original toggle settings. - `docker compose down` if you started a container. - Stop any leftover dev server with the kill command above. diff --git a/src/builtins/llm-providers/openai-chat/openai-chat-provider-config.ts b/src/builtins/llm-providers/openai-chat/openai-chat-provider-config.ts index 4d2a7c5..3f0c518 100644 --- a/src/builtins/llm-providers/openai-chat/openai-chat-provider-config.ts +++ b/src/builtins/llm-providers/openai-chat/openai-chat-provider-config.ts @@ -8,14 +8,14 @@ import { z } from "zod"; -import { emptyStringAsUndefined } from "../../../shared/config-coercion.js"; +import { emptyStringAsUndefined, jsonStringAsObjectOrUndefined } from "../../../shared/config-coercion.js"; const openAiChatConfigSchema = z .object({ baseUrl: z.string().min(1, "openai-chat baseUrl is required").url("openai-chat baseUrl must be a valid URL"), apiKey: z.string().default(""), model: z.string().min(1, "openai-chat model is required"), - extraBody: z.record(z.unknown()).default({}), + extraBody: z.preprocess(jsonStringAsObjectOrUndefined, z.record(z.unknown()).default({})), contextTokens: z.preprocess(emptyStringAsUndefined, z.coerce.number().int().positive().optional()), charsPerToken: z.preprocess(emptyStringAsUndefined, z.coerce.number().positive().default(3.5)), safetyMarginTokens: z.preprocess(emptyStringAsUndefined, z.coerce.number().int().nonnegative().default(0)) diff --git a/src/builtins/pipeline-steps/classify-url/classify-url-step-config.ts b/src/builtins/pipeline-steps/classify-url/classify-url-step-config.ts new file mode 100644 index 0000000..e3fc25c --- /dev/null +++ b/src/builtins/pipeline-steps/classify-url/classify-url-step-config.ts @@ -0,0 +1,119 @@ +/** + * Parses YAML configuration for the built-in classify-url pipeline step. + * + * Rules define URL classification: each rule has one signal name and exactly + * one matcher (pattern regex, anyHost domain list, or extensionIn suffix + * list). Precompiled matchers are embedded in the options so the step never + * re-parses at runtime. + */ + +import { z } from "zod"; + +import { assertDiagnosticName } from "../../../shared/diagnostic-names.js"; +import { ConfigurationError } from "../../../shared/errors.js"; + +/** A compiled URL classifier. */ +export interface UrlClassifyRule { + readonly signal: string; + readonly match: (url: string) => boolean; +} + +/** Describes constructor configuration for a classify-url step instance. */ +export interface ClassifyUrlStepOptions { + readonly rules: ReadonlyArray; +} + +const ruleSchema = z + .object({ + signal: z.string().min(1), + pattern: z.string().optional(), + anyHost: z.array(z.string().min(1)).optional(), + extensionIn: z.array(z.string().min(1)).optional() + }) + .strict() + .refine( + rule => Number(Boolean(rule.pattern)) + Number(Boolean(rule.anyHost)) + Number(Boolean(rule.extensionIn)) === 1, + { message: "Exactly one of pattern, anyHost, or extensionIn is required per rule" } + ); + +const configSchema = z + .object({ + rules: z.array(ruleSchema).default([]) + }) + .strict(); + +/** Configuration shape validated by the Zod schema. */ +type ParsedConfig = z.infer; + +/** Compiles one rule into a UrlClassifyRule with a precompiled match function. */ +function compileRule(raw: ParsedConfig["rules"][number]): UrlClassifyRule { + validateSignalName(raw.signal); + + if (raw.pattern !== undefined) { + let regex: RegExp; + try { + regex = new RegExp(raw.pattern); + } catch (cause) { + throw new ConfigurationError( + `Invalid regex in classify-url pattern: ${raw.pattern}`, + "invalid_classify_pattern", + { cause } + ); + } + return { signal: raw.signal, match: url => regex.test(url) }; + } + + if (raw.anyHost !== undefined) { + const hosts = raw.anyHost.map(h => + h + .toLowerCase() + .replace(/^www\./, "") + .replace(/^\./, "") + ); + return { + signal: raw.signal, + match: url => { + try { + const host = new URL(url).hostname.toLowerCase().replace(/^www\./, ""); + return hosts.some(h => host === h || host.endsWith("." + h)); + } catch { + return false; + } + } + }; + } + + if (raw.extensionIn !== undefined) { + const extensions = raw.extensionIn.map(e => "." + e.toLowerCase().replace(/^\./, "")); + return { + signal: raw.signal, + match: url => { + try { + const pathname = new URL(url).pathname; + const extIndex = pathname.lastIndexOf("."); + return extIndex >= 0 && extensions.includes(pathname.slice(extIndex).toLowerCase()); + } catch { + return false; + } + } + }; + } + + throw new ConfigurationError("Empty classify-url rule", "invalid_classify_rule"); +} + +/** Translates diagnostic-name failures into operator-facing classifier config errors. */ +function validateSignalName(signal: string): void { + try { + assertDiagnosticName(signal); + } catch (cause) { + throw new ConfigurationError(`Invalid classify-url signal: ${signal}`, "invalid_classify_signal", { cause }); + } +} + +/** Parses classify-url step configuration from YAML. */ +export function parseClassifyUrlStepConfig(raw: unknown): ClassifyUrlStepOptions { + const parsed = configSchema.parse(raw); + const rules = parsed.rules.map(compileRule); + return Object.freeze({ rules }); +} diff --git a/src/builtins/pipeline-steps/classify-url/classify-url-step-descriptor.ts b/src/builtins/pipeline-steps/classify-url/classify-url-step-descriptor.ts new file mode 100644 index 0000000..07748e2 --- /dev/null +++ b/src/builtins/pipeline-steps/classify-url/classify-url-step-descriptor.ts @@ -0,0 +1,19 @@ +/** + * Exposes the classify-url pipeline step descriptor to engine bundles. + * + * The runtime step classifies the input URL against configured rules and emits + * boolean signals that downstream runIf/skipIf gates consume. + */ + +import { parseClassifyUrlStepConfig } from "./classify-url-step-config.js"; +import { ClassifyUrlStep } from "./classify-url-step.js"; + +import type { PipelineStepDescriptor } from "../../../contracts/pipeline/step.js"; +import type { ClassifyUrlStepOptions } from "./classify-url-step-config.js"; + +/** Defines the built-in classify-url step type. */ +export const classifyUrlStepDescriptor = { + type: "classify-url", + parseConfig: parseClassifyUrlStepConfig, + create: args => new ClassifyUrlStep(args.config, { logger: args.deps.logger }) +} satisfies PipelineStepDescriptor; diff --git a/src/builtins/pipeline-steps/classify-url/classify-url-step.ts b/src/builtins/pipeline-steps/classify-url/classify-url-step.ts new file mode 100644 index 0000000..a5d5bd3 --- /dev/null +++ b/src/builtins/pipeline-steps/classify-url/classify-url-step.ts @@ -0,0 +1,36 @@ +/** + * Classifies the input URL against configured rules and emits matching signals. + * + * Runs at most once per pipeline. Every matching rule emits one boolean signal + * (true). Multiple rules may share the same signal name (natural OR). + */ + +import type { PipelineContext } from "../../../contracts/pipeline/context.js"; +import type { PipelineStep, StepResult } from "../../../contracts/pipeline/step.js"; +import type { Logger } from "../../../shared/logger.js"; +import type { ClassifyUrlStepOptions } from "./classify-url-step-config.js"; + +/** Classifies the pipeline input URL and emits matching signals. */ +export class ClassifyUrlStep implements PipelineStep { + /** Creates a classify-url step instance. */ + constructor( + private readonly config: ClassifyUrlStepOptions, + private readonly deps: { readonly logger: Logger } + ) {} + + /** Evaluates all rules against the input URL and returns matching signals. */ + async run(ctx: PipelineContext): Promise { + const matched: Record = {}; + + for (const rule of this.config.rules) { + if (rule.match(ctx.input.url)) matched[rule.signal] = true; + } + + const keys = Object.keys(matched); + return { + status: "ok", + effects: keys.length > 0 ? { signals: Object.fromEntries(keys.map(k => [k, true])) } : undefined, + diagnostics: { attributes: { matched: keys.length > 0 ? keys.join(" ") : "none" } } + }; + } +} diff --git a/src/builtins/source-providers/docling/docling-provider-config.ts b/src/builtins/source-providers/docling/docling-provider-config.ts index 2bc9e31..7500c5a 100644 --- a/src/builtins/source-providers/docling/docling-provider-config.ts +++ b/src/builtins/source-providers/docling/docling-provider-config.ts @@ -1,21 +1,22 @@ /** * Parses YAML configuration for the built-in Docling source provider. * - * Environment substitution runs before this schema, so boolean fields use - * shared preprocessors to treat blank placeholders as omitted values. + * Environment substitution runs before this schema, so blank fields use shared + * preprocessors. `output` stays typed because it drives the requested formats, + * the response field read, and the returned media type. Remaining Docling + * convert knobs flow through the shared opaque-record preprocessor. */ import { z } from "zod"; -import { booleanStringAsBooleanOrUndefined, emptyStringAsUndefined } from "../../../shared/config-coercion.js"; +import { emptyStringAsUndefined, jsonStringAsObjectOrUndefined } from "../../../shared/config-coercion.js"; const doclingConfigSchema = z .object({ baseUrl: z.string().min(1, "docling baseUrl is required").url("docling baseUrl must be a valid URL"), apiKey: z.string().default(""), output: z.preprocess(emptyStringAsUndefined, z.enum(["markdown", "html"]).default("markdown")), - doOcr: z.preprocess(booleanStringAsBooleanOrUndefined, z.boolean().default(true)), - tableMode: z.enum(["fast", "accurate"]).default("accurate") + options: z.preprocess(jsonStringAsObjectOrUndefined, z.record(z.unknown()).default({})) }) .strict(); diff --git a/src/builtins/source-providers/docling/docling-provider.ts b/src/builtins/source-providers/docling/docling-provider.ts index 5f44717..24a0319 100644 --- a/src/builtins/source-providers/docling/docling-provider.ts +++ b/src/builtins/source-providers/docling/docling-provider.ts @@ -116,10 +116,8 @@ export class DoclingProvider implements SourceProvider { return { sources: [{ kind: "http", url }], options: { - to_formats: this.config.output === "html" ? ["html", "json"] : ["md", "json"], - image_export_mode: "placeholder", - do_ocr: this.config.doOcr, - table_mode: this.config.tableMode + ...this.config.options, + to_formats: this.config.output === "html" ? ["html", "json"] : ["md", "json"] } }; } diff --git a/src/builtins/source-providers/firecrawl/firecrawl-provider-config.ts b/src/builtins/source-providers/firecrawl/firecrawl-provider-config.ts index 768cfbe..31c4d17 100644 --- a/src/builtins/source-providers/firecrawl/firecrawl-provider-config.ts +++ b/src/builtins/source-providers/firecrawl/firecrawl-provider-config.ts @@ -1,22 +1,22 @@ /** * Parses YAML configuration for the built-in Firecrawl source provider. * - * Environment substitution runs before this schema, so numeric and boolean - * fields use shared preprocessors to treat blank placeholders as omitted values. + * Environment substitution runs before this schema, so blank fields use shared + * preprocessors. `output` stays typed because it drives the requested formats, + * the response field read, and the returned media type. Remaining Firecrawl + * scrape knobs flow through the shared opaque-record preprocessor. */ import { z } from "zod"; -import { booleanStringAsBooleanOrUndefined } from "../../../shared/config-coercion.js"; +import { emptyStringAsUndefined, jsonStringAsObjectOrUndefined } from "../../../shared/config-coercion.js"; const firecrawlConfigSchema = z .object({ baseUrl: z.string().min(1, "firecrawl baseUrl is required").url("firecrawl baseUrl must be a valid URL"), apiKey: z.string().default(""), - output: z.enum(["markdown", "html", "rawHtml"]).default("markdown"), - onlyMainContent: z.preprocess(booleanStringAsBooleanOrUndefined, z.boolean().default(true)), - stripBase64Images: z.preprocess(booleanStringAsBooleanOrUndefined, z.boolean().default(true)), - parsePdf: z.preprocess(booleanStringAsBooleanOrUndefined, z.boolean().default(true)) + output: z.preprocess(emptyStringAsUndefined, z.enum(["markdown", "html", "rawHtml"]).default("markdown")), + options: z.preprocess(jsonStringAsObjectOrUndefined, z.record(z.unknown()).default({})) }) .strict(); diff --git a/src/builtins/source-providers/firecrawl/firecrawl-provider.ts b/src/builtins/source-providers/firecrawl/firecrawl-provider.ts index 1b1febd..1903ebf 100644 --- a/src/builtins/source-providers/firecrawl/firecrawl-provider.ts +++ b/src/builtins/source-providers/firecrawl/firecrawl-provider.ts @@ -56,10 +56,8 @@ export class FirecrawlProvider implements SourceProvider { headers: this.headers(), body: JSON.stringify({ url, - formats: [this.config.output], - onlyMainContent: this.config.onlyMainContent, - removeBase64Images: this.config.stripBase64Images, - parsers: this.config.parsePdf ? ["pdf"] : [] + ...this.config.options, + formats: [this.config.output] }) }); @@ -93,25 +91,10 @@ export class FirecrawlProvider implements SourceProvider { upstreamStatus: response.status }); const title = data.title ?? (typeof metadata.title === "string" ? metadata.title : undefined); - const contentType = typeof metadata.contentType === "string" ? metadata.contentType : ""; - if (!this.config.parsePdf) { - const essence = contentType.split(";", 1)[0].trim().toLowerCase(); - if (essence === mediaTypes.pdf) { - const raw = extractBase64Payload(content, this.config.output); - if (!raw) { - throw new UpstreamError("Firecrawl returned empty base64 PDF payload", "empty", { - upstreamStatus: response.status - }); - } - const bytes = Buffer.from(raw, "base64"); - if (bytes.byteLength === 0) { - throw new UpstreamError("Firecrawl returned empty PDF bytes after base64 decode", "empty", { - upstreamStatus: response.status - }); - } - return { kind: "binary", bytes, mediaType: mediaTypes.pdf, title }; - } + const pdfBytes = decodePdfPassthrough(content, this.config.output); + if (pdfBytes) { + return { kind: "binary", bytes: pdfBytes, mediaType: mediaTypes.pdf, title }; } const mediaType = deriveTextMediaType(this.config.output, content); @@ -130,6 +113,29 @@ export class FirecrawlProvider implements SourceProvider { } } +/** + * Attempts to decode a base64 PDF payload from Firecrawl content. + * + * With empty parsers (options:{parsers:[]}), Firecrawl returns the raw file + * bytes as base64. When output=html, the base64 is wrapped in + * `…`. This function unwraps if needed, gates on the + * PDF base64 prefix (`JVBERi0`), decodes, and verifies the `%PDF` magic header. + * Returns undefined when the content is not a PDF passthrough. + */ +function decodePdfPassthrough(content: string, output: string): Buffer | undefined { + const candidate = extractBase64Payload(content, output); + if (!candidate || !candidate.startsWith("JVBERi0")) return undefined; + let bytes: Buffer; + try { + bytes = Buffer.from(candidate, "base64"); + } catch { + return undefined; + } + if (bytes.byteLength < 4) return undefined; + if (bytes[0] !== 0x25 || bytes[1] !== 0x50 || bytes[2] !== 0x44 || bytes[3] !== 0x46) return undefined; + return bytes; +} + /** * Derives a truthful media type for the text returned by Firecrawl. * diff --git a/src/bundles/default-engine-descriptors.ts b/src/bundles/default-engine-descriptors.ts index 2a28b32..d346190 100644 --- a/src/bundles/default-engine-descriptors.ts +++ b/src/bundles/default-engine-descriptors.ts @@ -12,14 +12,15 @@ import { openAiChatProviderDescriptor } from "../builtins/llm-providers/openai-c import { debugXmlRendererDescriptor } from "../builtins/output-renderers/debug-xml/debug-xml-renderer-descriptor.js"; import { passthroughRendererDescriptor } from "../builtins/output-renderers/passthrough/passthrough-renderer-descriptor.js"; import { captureUrlsStepDescriptor } from "../builtins/pipeline-steps/capture-urls/capture-urls-step-descriptor.js"; +import { classifyUrlStepDescriptor } from "../builtins/pipeline-steps/classify-url/classify-url-step-descriptor.js"; import { llmPassStepDescriptor } from "../builtins/pipeline-steps/llm-pass/llm-pass-step-descriptor.js"; import { loadSourceStepDescriptor } from "../builtins/pipeline-steps/load-source/load-source-step-descriptor.js"; import { transformStepDescriptor } from "../builtins/pipeline-steps/transform/transform-step-descriptor.js"; import { truncateStepDescriptor } from "../builtins/pipeline-steps/truncate/truncate-step-descriptor.js"; import { verifyUrlsStepDescriptor } from "../builtins/pipeline-steps/verify-urls/verify-urls-step-descriptor.js"; -import { httpProviderDescriptor } from "../builtins/source-providers/http/http-provider-descriptor.js"; -import { firecrawlProviderDescriptor } from "../builtins/source-providers/firecrawl/firecrawl-provider-descriptor.js"; import { doclingProviderDescriptor } from "../builtins/source-providers/docling/docling-provider-descriptor.js"; +import { firecrawlProviderDescriptor } from "../builtins/source-providers/firecrawl/firecrawl-provider-descriptor.js"; +import { httpProviderDescriptor } from "../builtins/source-providers/http/http-provider-descriptor.js"; import { createDescriptorRecord } from "../shared/descriptors.js"; import type { ContentTransformerDescriptor } from "../contracts/extensions/content-transformer.js"; @@ -55,6 +56,7 @@ export const DEFAULT_ENGINE_DESCRIPTOR_BUNDLE: EngineDescriptorBundle = Object.f pipelineSteps: createDescriptorRecord( [ captureUrlsStepDescriptor, + classifyUrlStepDescriptor, loadSourceStepDescriptor, llmPassStepDescriptor, transformStepDescriptor, diff --git a/src/config/yaml/yaml-config.ts b/src/config/yaml/yaml-config.ts index 15cddfd..c7d8f7b 100644 --- a/src/config/yaml/yaml-config.ts +++ b/src/config/yaml/yaml-config.ts @@ -9,6 +9,25 @@ import { z } from "zod"; import { booleanStringAsBooleanOrUndefined, emptyStringAsUndefined } from "../../shared/config-coercion.js"; +import { isDiagnosticName } from "../../shared/diagnostic-names.js"; + +/** Represents recursive runIf/skipIf predicates without importing engine contracts into YAML parsing. */ +export type RawCondition = + | string + | { readonly all: ReadonlyArray } + | { readonly any: ReadonlyArray } + | { readonly not: RawCondition }; + +const signalNameSchema = z.string().min(1).refine(isDiagnosticName, { message: "Invalid signal name" }); + +const conditionSchema: z.ZodType = z.lazy(() => + z.union([ + signalNameSchema, + z.object({ all: z.array(conditionSchema) }).strict(), + z.object({ any: z.array(conditionSchema) }).strict(), + z.object({ not: conditionSchema }).strict() + ]) +); const providerEntrySchema = z .object({ @@ -23,6 +42,9 @@ const stepSchema = z name: z.string().min(1), concurrencyGroup: z.string().min(1).optional(), timeoutSeconds: z.preprocess(emptyStringAsUndefined, z.coerce.number().int().positive().default(60)), + runIf: conditionSchema.optional(), + skipIf: conditionSchema.optional(), + enabled: z.preprocess(booleanStringAsBooleanOrUndefined, z.boolean().optional()), config: z.unknown().default({}) }) .strict(); diff --git a/src/contracts/pipeline/condition.ts b/src/contracts/pipeline/condition.ts new file mode 100644 index 0000000..b6cc05b --- /dev/null +++ b/src/contracts/pipeline/condition.ts @@ -0,0 +1,14 @@ +/** + * Defines the condition model for engine-driven step gating (runIf/skipIf). + * + * Conditions are a recursive discriminated union evaluated against the + * runtime signal bag. A bare string leaf tests whether that signal + * exists and is truthy. Combinators all/any/not nest arbitrarily. + */ + +/** Describes a signal-based predicate for runIf/skipIf evaluation. */ +export type Condition = + | string + | { readonly all: ReadonlyArray } + | { readonly any: ReadonlyArray } + | { readonly not: Condition }; diff --git a/src/core/pipeline/compiled.ts b/src/core/pipeline/compiled.ts index 95f8f39..35851e6 100644 --- a/src/core/pipeline/compiled.ts +++ b/src/core/pipeline/compiled.ts @@ -7,6 +7,7 @@ */ import type { OutputRenderer } from "../../contracts/extensions/output-renderer.js"; +import type { Condition } from "../../contracts/pipeline/condition.js"; import type { PipelineStep } from "../../contracts/pipeline/step.js"; import type { ConcurrencyLimiter } from "../../shared/limiters.js"; @@ -16,6 +17,8 @@ export interface CompiledPipelineStep { readonly type: string; readonly timeoutSeconds: number; readonly concurrencyGroup?: string; + readonly runIf?: Condition; + readonly skipIf?: Condition; readonly step: PipelineStep; } diff --git a/src/core/pipeline/conditions.ts b/src/core/pipeline/conditions.ts new file mode 100644 index 0000000..4da31c1 --- /dev/null +++ b/src/core/pipeline/conditions.ts @@ -0,0 +1,47 @@ +/** + * Evaluates Condition predicates against a runtime signal lookup. + * + * The signal lookup is intentionally an abstract function so this module + * stays decoupled from the orchestrator-owned signal map. Gate resolution + * follows the rule: step runs iff (runIf absent or true) AND (skipIf absent + * or false). + */ + +import type { Condition } from "../../contracts/pipeline/condition.js"; +import type { ScalarValue } from "../../contracts/pipeline/context.js"; + +/** Looks up a signal by name, returning undefined when absent. */ +export type SignalLookup = (name: string) => ScalarValue | undefined; + +/** Describes the reserved skip reason produced by engine-level step gates. */ +export type StepGateReason = "run_if_unmet" | "skip_if_met"; + +/** Returns true when a scalar value is truthy in the signal domain. */ +export function isTruthySignal(value: ScalarValue | undefined): boolean { + return value !== undefined && value !== false && value !== 0 && value !== ""; +} + +/** Evaluates a Condition against a signal lookup. */ +export function evaluateCondition(condition: Condition, lookup: SignalLookup): boolean { + if (typeof condition === "string") return isTruthySignal(lookup(condition)); + if ("all" in condition) return condition.all.every(c => evaluateCondition(c, lookup)); + if ("any" in condition) return condition.any.some(c => evaluateCondition(c, lookup)); + if ("not" in condition) return !evaluateCondition(condition.not, lookup); + return false; +} + +/** + * Resolves the engine-driven gate for one step. + * + * Returns a reserved skip reason when the step should be gated, or undefined + * when the step should run normally. runIf takes precedence over skipIf when + * both are present. + */ +export function resolveStepGate( + gates: { readonly runIf?: Condition; readonly skipIf?: Condition }, + lookup: SignalLookup +): StepGateReason | undefined { + if (gates.runIf !== undefined && !evaluateCondition(gates.runIf, lookup)) return "run_if_unmet"; + if (gates.skipIf !== undefined && evaluateCondition(gates.skipIf, lookup)) return "skip_if_met"; + return undefined; +} diff --git a/src/core/pipeline/orchestrator.ts b/src/core/pipeline/orchestrator.ts index e95ba0e..0d44729 100644 --- a/src/core/pipeline/orchestrator.ts +++ b/src/core/pipeline/orchestrator.ts @@ -12,6 +12,7 @@ import { randomUUID } from "node:crypto"; import { InternalError, isAbortError } from "../../shared/errors.js"; import { getRequestContext, runWithRequestContext, withChildRequestContext } from "../../shared/request-context.js"; import { BodyStore } from "./body.js"; +import { resolveStepGate } from "./conditions.js"; import { ReadonlyArtifactBag, ReadonlySignalBag } from "./context.js"; import { applyStepEffects } from "./effects.js"; import { finalizeReport } from "./report.js"; @@ -167,9 +168,23 @@ export class PipelineOrchestrator { const { entry, index } = step; const current = state.body.current(); const inputLength = current ? bodyLength(current) : undefined; + const stepStartedAt = this.clock.now(); + + const gateReason = resolveStepGate(entry, name => state.signals.get(name)); + if (gateReason) { + this.recordStepResult( + step, + pipeline, + { status: "skipped", reason: gateReason }, + state, + stepStartedAt, + inputLength + ); + return; + } + const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), entry.timeoutSeconds * 1000); - const stepStartedAt = this.clock.now(); let result: StepResult; this.logger.debug( diff --git a/src/engine/create-engine.ts b/src/engine/create-engine.ts index b89d924..4a465af 100644 --- a/src/engine/create-engine.ts +++ b/src/engine/create-engine.ts @@ -98,12 +98,14 @@ function pipelineInfos(config: EngineConfig): ReadonlyArray { .map(([name, pipeline]) => ({ name, outputRenderer: pipeline.outputRenderer, - steps: pipeline.steps.map(step => ({ - name: step.name, - type: step.type, - timeoutSeconds: step.timeoutSeconds, - concurrencyGroup: step.concurrencyGroup - })) + steps: pipeline.steps + .filter(step => step.enabled !== false) + .map(step => ({ + name: step.name, + type: step.type, + timeoutSeconds: step.timeoutSeconds, + concurrencyGroup: step.concurrencyGroup + })) })); } diff --git a/src/engine/engine-config.ts b/src/engine/engine-config.ts index afacd8b..b8dab6f 100644 --- a/src/engine/engine-config.ts +++ b/src/engine/engine-config.ts @@ -6,6 +6,8 @@ * boundary before calling `createEngine`. */ +import type { Condition } from "../contracts/pipeline/condition.js"; + /** Configures one named provider or output renderer instance. */ export interface EngineComponentConfig { readonly type: string; @@ -18,6 +20,9 @@ export interface EngineStepConfig { readonly name: string; readonly concurrencyGroup?: string; readonly timeoutSeconds: number; + readonly runIf?: Condition; + readonly skipIf?: Condition; + readonly enabled?: boolean; readonly config?: unknown; } diff --git a/src/engine/internal/builders/build-pipelines.ts b/src/engine/internal/builders/build-pipelines.ts index 00d1667..a729a89 100644 --- a/src/engine/internal/builders/build-pipelines.ts +++ b/src/engine/internal/builders/build-pipelines.ts @@ -1,9 +1,10 @@ /** * Compiles configured pipeline declarations into executable pipeline plans. * - * Pipeline compilation validates renderer references, concurrency-group - * references, diagnostic-safe step names, duplicate step names, descriptor - * lookup, config parsing, and step factory failures before any request can run. + * Pipeline compilation validates renderer references and enabled-step + * concurrency groups, diagnostic names, descriptor lookup, config parsing, and + * factory failures before any request can run. Disabled steps are excluded + * before those step-level checks so their providers are never required. */ import { outputRendererRegistryKey } from "../../../contracts/extensions/output-renderer.js"; @@ -29,7 +30,10 @@ export async function buildPipelines( const rendererRegistry = services.require(outputRendererRegistryKey); const pipelines = new Map(); for (const [pipelineName, pipeline] of Object.entries(rawPipelines)) { - if (!pipeline.enabled) continue; + if (!pipeline.enabled) { + logger.info({ pipeline: pipelineName }, "pipeline disabled; excluded from compilation"); + continue; + } const renderer = rendererRegistry.tryGet(pipeline.outputRenderer); if (!renderer) throw new ConfigurationError( @@ -40,8 +44,13 @@ export async function buildPipelines( Object.entries(pipeline.limiters).map(([name, concurrency]) => [name, createConcurrencyLimiter(concurrency)]) ); const seenStepNames = new Set(); + const disabledStepNames: string[] = []; const steps: CompiledPipelineStep[] = []; for (const common of pipeline.steps) { + if (common.enabled === false) { + disabledStepNames.push(common.name); + continue; + } try { assertDiagnosticName(common.name); } catch (cause) { @@ -108,9 +117,14 @@ export async function buildPipelines( type: common.type, timeoutSeconds: common.timeoutSeconds, concurrencyGroup: common.concurrencyGroup, + runIf: common.runIf, + skipIf: common.skipIf, step }); } + if (disabledStepNames.length > 0) + logger.info({ pipeline: pipelineName, disabled: disabledStepNames }, "steps disabled; excluded from compilation"); + if (steps.length === 0) logger.warn({ pipeline: pipelineName }, "pipeline compiled with no enabled steps"); pipelines.set(pipelineName, { name: pipelineName, renderer: renderer.renderer, steps, groups }); } return pipelines; diff --git a/src/shared/config-coercion.ts b/src/shared/config-coercion.ts index 5e665a4..f915ddc 100644 --- a/src/shared/config-coercion.ts +++ b/src/shared/config-coercion.ts @@ -20,3 +20,18 @@ export function booleanStringAsBooleanOrUndefined(value: unknown): unknown { if (normalized === "false") return false; return value; } + +/** Parses a JSON string into an object, for opaque record fields that accept both + * an inline YAML object and a JSON-string blob from an env var. + * Non-strings pass through (inline object). Blank strings become undefined (empty + * env var / default). Invalid JSON is returned raw so the schema reports the error. */ +export function jsonStringAsObjectOrUndefined(value: unknown): unknown { + if (typeof value !== "string") return value; + const trimmed = value.trim(); + if (trimmed === "") return undefined; + try { + return JSON.parse(trimmed) as unknown; + } catch { + return value; + } +} diff --git a/src/shared/diagnostic-names.ts b/src/shared/diagnostic-names.ts index 7e5bfc9..112449d 100644 --- a/src/shared/diagnostic-names.ts +++ b/src/shared/diagnostic-names.ts @@ -10,8 +10,12 @@ import { InternalError } from "./errors.js"; const DIAGNOSTIC_NAME_PATTERN = /^[a-z][a-z0-9_]*$/; +/** Returns whether a value is safe for diagnostic XML names and signal keys. */ +export function isDiagnosticName(name: string): boolean { + return DIAGNOSTIC_NAME_PATTERN.test(name); +} + /** Validates a diagnostic element, attribute, or step name. */ export function assertDiagnosticName(name: string): void { - if (!DIAGNOSTIC_NAME_PATTERN.test(name)) - throw new InternalError(`Invalid diagnostic name: ${name}`, "invalid_diagnostic_name"); + if (!isDiagnosticName(name)) throw new InternalError(`Invalid diagnostic name: ${name}`, "invalid_diagnostic_name"); } diff --git a/tests/app/compose.test.ts b/tests/app/compose.test.ts index 61b33f3..e229f84 100644 --- a/tests/app/compose.test.ts +++ b/tests/app/compose.test.ts @@ -74,7 +74,7 @@ describe("composeApp", () => { ).rejects.toThrow("Duplicate step name"); const unknownField = await writeConfigFixture( - defaultYaml().replace(" stripBase64Images: true", " stripBase64Images: true\n mystery: true") + defaultYaml().replace(" output: markdown", " output: markdown\n mystery: true") ); await expect( composeApp({ @@ -119,12 +119,10 @@ describe("composeApp", () => { it("coerces env-substituted config scalars through schema parsers", async () => { const fixture = await writeConfigFixture( defaultYaml() - .replace("onlyMainContent: true", "onlyMainContent: ${FIRECRAWL_ONLY_MAIN:-}") - .replace("stripBase64Images: true", "stripBase64Images: ${FIRECRAWL_STRIP_IMAGES:-}") .replace("includeSkipped: true", "includeSkipped: ${DEBUG_XML_INCLUDE_SKIPPED:-}") .replace( " config:\n provider: llm-default", - " timeoutSeconds: ${CLEAN_TIMEOUT_SECONDS:-}\n config:\n provider: llm-default" + " timeoutSeconds: ${LLM_CLEAN_TIMEOUT_SECONDS:-}\n config:\n provider: llm-default" ) .replace("minInputChars: 1", "minInputChars: ${MIN_INPUT_CHARS:-}") .replace("maxInputChars: 100", "maxInputChars: ${MAX_INPUT_CHARS:-}") @@ -134,12 +132,11 @@ describe("composeApp", () => { envConfig: loadEnvConfig({ CONFIG_FILE: fixture.configPath }), env: { FIRECRAWL_BASE_URL: "https://firecrawl.example", - FIRECRAWL_ONLY_MAIN: "false", - FIRECRAWL_STRIP_IMAGES: "", + FIRECRAWL_OPTIONS: '{"onlyMainContent":false}', LLM_BASE_URL: "https://llm.example/v1", LLM_MODEL: "model", DEBUG_XML_INCLUDE_SKIPPED: "false", - CLEAN_TIMEOUT_SECONDS: "", + LLM_CLEAN_TIMEOUT_SECONDS: "", MIN_INPUT_CHARS: "", MAX_INPUT_CHARS: "" }, @@ -162,7 +159,7 @@ describe("composeApp", () => { .require("firecrawl-markdown") .provider.load("https://example.com", { signal: new AbortController().signal }); - expect(firecrawlRequestBody).toMatchObject({ onlyMainContent: false, removeBase64Images: true }); + expect(firecrawlRequestBody).toMatchObject({ onlyMainContent: false }); }); it("rejects unknown registry references and invalid diagnostic names", async () => { @@ -249,9 +246,7 @@ function defaultYaml(): string { baseUrl: \${FIRECRAWL_BASE_URL} apiKey: \${FIRECRAWL_API_KEY:-} output: markdown - onlyMainContent: true - stripBase64Images: true - parsePdf: true + options: \${FIRECRAWL_OPTIONS:-} llmProviders: llm-default: type: openai-chat diff --git a/tests/builtins/llm-providers/openai-chat/openai-chat-provider-config.test.ts b/tests/builtins/llm-providers/openai-chat/openai-chat-provider-config.test.ts index 791eb14..64315cd 100644 --- a/tests/builtins/llm-providers/openai-chat/openai-chat-provider-config.test.ts +++ b/tests/builtins/llm-providers/openai-chat/openai-chat-provider-config.test.ts @@ -34,4 +34,41 @@ describe("parseOpenAiChatConfig", () => { parseOpenAiChatConfig({ baseUrl: "https://llm.example/v1", model: "test-model", contextTokens: "0" }) ).toThrow(); }); + + it("passes through inline extraBody object unmodified", () => { + const config = parseOpenAiChatConfig({ + baseUrl: "https://llm.example/v1", + model: "test-model", + extraBody: { temperature: 0.7, top_p: 0.9 } + }); + expect(config.extraBody).toEqual({ temperature: 0.7, top_p: 0.9 }); + }); + + it("parses a JSON-string blob in extraBody from a single env var", () => { + const config = parseOpenAiChatConfig({ + baseUrl: "https://llm.example/v1", + model: "test-model", + extraBody: '{"temperature":0.7,"top_p":0.9}' + }); + expect(config.extraBody).toEqual({ temperature: 0.7, top_p: 0.9 }); + }); + + it("coerces a blank extraBody env var to empty object", () => { + const config = parseOpenAiChatConfig({ + baseUrl: "https://llm.example/v1", + model: "test-model", + extraBody: "" + }); + expect(config.extraBody).toEqual({}); + }); + + it("rejects invalid JSON in extraBody", () => { + expect(() => + parseOpenAiChatConfig({ + baseUrl: "https://llm.example/v1", + model: "test-model", + extraBody: "{bad" + }) + ).toThrow(); + }); }); diff --git a/tests/builtins/pipeline-steps/classify-url/classify-url-step-config.test.ts b/tests/builtins/pipeline-steps/classify-url/classify-url-step-config.test.ts new file mode 100644 index 0000000..47eddef --- /dev/null +++ b/tests/builtins/pipeline-steps/classify-url/classify-url-step-config.test.ts @@ -0,0 +1,102 @@ +/** Verifies the classify-url step configuration and matcher compilation. */ +import { describe, expect, it } from "vitest"; + +import { parseClassifyUrlStepConfig } from "../../../../src/builtins/pipeline-steps/classify-url/classify-url-step-config.js"; +import { ConfigurationError } from "../../../../src/shared/errors.js"; + +describe("parseClassifyUrlStepConfig", () => { + it("accepts empty rules", () => { + const config = parseClassifyUrlStepConfig({ rules: [] }); + expect(config.rules).toEqual([]); + }); + + it("compiles a pattern matcher", () => { + const config = parseClassifyUrlStepConfig({ + rules: [{ signal: "binary_doc", pattern: "\\.pdf($|\\?|#)" }] + }); + expect(config.rules).toHaveLength(1); + expect(config.rules[0]?.match("https://example.com/doc.pdf")).toBe(true); + expect(config.rules[0]?.match("https://example.com/doc.pdf?dl=1")).toBe(true); + expect(config.rules[0]?.match("https://example.com/doc")).toBe(false); + }); + + it("compiles an anyHost matcher", () => { + const config = parseClassifyUrlStepConfig({ + rules: [{ signal: "code_host", anyHost: ["github.com"] }] + }); + expect(config.rules[0]?.match("https://github.com/user/repo")).toBe(true); + expect(config.rules[0]?.match("https://www.github.com/user/repo")).toBe(true); + expect(config.rules[0]?.match("https://gist.github.com/user/repo")).toBe(true); + expect(config.rules[0]?.match("https://gitlab.com/user/repo")).toBe(false); + }); + + it("compiles an extensionIn matcher", () => { + const config = parseClassifyUrlStepConfig({ + rules: [{ signal: "binary_doc", extensionIn: ["pdf", "docx"] }] + }); + expect(config.rules[0]?.match("https://example.com/doc.pdf")).toBe(true); + expect(config.rules[0]?.match("https://example.com/doc.PDF?foo=1")).toBe(true); + expect(config.rules[0]?.match("https://example.com/file.docx")).toBe(true); + expect(config.rules[0]?.match("https://example.com/doc")).toBe(false); + expect(config.rules[0]?.match("https://example.com/doc.txt")).toBe(false); + }); + + it("rejects rules with multiple matchers", () => { + expect(() => + parseClassifyUrlStepConfig({ + rules: [{ signal: "x", pattern: ".*", anyHost: ["a.com"] }] + }) + ).toThrow("Exactly one of pattern, anyHost, or extensionIn is required per rule"); + }); + + it("rejects rules with no matchers", () => { + expect(() => + parseClassifyUrlStepConfig({ + rules: [{ signal: "x" }] + }) + ).toThrow("Exactly one of pattern, anyHost, or extensionIn is required per rule"); + }); + + it("rejects invalid regex pattern", () => { + expect(() => + parseClassifyUrlStepConfig({ + rules: [{ signal: "x", pattern: "(" }] + }) + ).toThrow(ConfigurationError); + }); + + it("rejects invalid signal names", () => { + expect(() => + parseClassifyUrlStepConfig({ + rules: [{ signal: "BinaryDoc", pattern: "\\.pdf" }] + }) + ).toThrow(ConfigurationError); + }); + + it("normalizes host names (www strip, leading dot strip)", () => { + const config = parseClassifyUrlStepConfig({ + rules: [{ signal: "code", anyHost: [".GitHub.Com", "WWW.example.com"] }] + }); + expect(config.rules[0]?.match("https://github.com/repo")).toBe(true); + expect(config.rules[0]?.match("https://example.com/page")).toBe(true); + }); + + it("normalizes extensions (leading dot strip)", () => { + const config = parseClassifyUrlStepConfig({ + rules: [{ signal: "bin", extensionIn: [".PDF", "DOCX"] }] + }); + expect(config.rules[0]?.match("https://example.com/file.pdf")).toBe(true); + expect(config.rules[0]?.match("https://example.com/file.docx")).toBe(true); + }); + + it("silently returns false from match on invalid URL", () => { + const config = parseClassifyUrlStepConfig({ + rules: [{ signal: "c", anyHost: ["a.com"] }] + }); + expect(config.rules[0]?.match("not a url")).toBe(false); + const extConfig = parseClassifyUrlStepConfig({ + rules: [{ signal: "c", extensionIn: ["pdf"] }] + }); + expect(extConfig.rules[0]?.match("not a url")).toBe(false); + }); +}); diff --git a/tests/builtins/pipeline-steps/classify-url/classify-url-step-descriptor.test.ts b/tests/builtins/pipeline-steps/classify-url/classify-url-step-descriptor.test.ts new file mode 100644 index 0000000..66e88ae --- /dev/null +++ b/tests/builtins/pipeline-steps/classify-url/classify-url-step-descriptor.test.ts @@ -0,0 +1,32 @@ +/** Verifies the classify-url step descriptor shape and bundle registration. */ +import { describe, expect, it } from "vitest"; + +import { classifyUrlStepDescriptor } from "../../../../src/builtins/pipeline-steps/classify-url/classify-url-step-descriptor.js"; +import { DEFAULT_ENGINE_DESCRIPTOR_BUNDLE } from "../../../../src/bundles/default-engine-descriptors.js"; + +import type { PipelineStepDescriptor } from "../../../../src/contracts/pipeline/step.js"; + +describe("classifyUrlStepDescriptor", () => { + it("has type classify-url", () => { + expect(classifyUrlStepDescriptor.type).toBe("classify-url"); + }); + + it("creates a ClassifyUrlStep", async () => { + const step = await classifyUrlStepDescriptor.create({ + name: "classify", + type: "classify-url", + timeoutSeconds: 30, + config: classifyUrlStepDescriptor.parseConfig({ rules: [] }), + deps: { tools: {} as never, logger: {} as never }, + services: {} as never + }); + + expect(step.constructor.name).toBe("ClassifyUrlStep"); + }); + + it("is registered in the default engine descriptor bundle", () => { + const descriptor = DEFAULT_ENGINE_DESCRIPTOR_BUNDLE.pipelineSteps["classify-url"]; + expect(descriptor).toBeDefined(); + expect(descriptor?.type).toBe("classify-url"); + }); +}); diff --git a/tests/builtins/pipeline-steps/classify-url/classify-url-step.test.ts b/tests/builtins/pipeline-steps/classify-url/classify-url-step.test.ts new file mode 100644 index 0000000..b48463d --- /dev/null +++ b/tests/builtins/pipeline-steps/classify-url/classify-url-step.test.ts @@ -0,0 +1,67 @@ +/** Verifies the classify-url pipeline step runtime behavior. */ +import { describe, expect, it } from "vitest"; + +import { ClassifyUrlStep } from "../../../../src/builtins/pipeline-steps/classify-url/classify-url-step.js"; +import { createTestLogger } from "../../../helpers/logger.js"; +import { makeStepContext } from "../utils.js"; + +import type { UrlClassifyRule } from "../../../../src/builtins/pipeline-steps/classify-url/classify-url-step-config.js"; + +describe("ClassifyUrlStep", () => { + it("emits no signals when no rules match", async () => { + const step = makeStep([{ signal: "binary_doc", match: () => false }]); + + const result = await step.run(makeStepContext()); + + expect(result.status).toBe("ok"); + expect(result.effects).toBeUndefined(); + expect(result.diagnostics?.attributes?.matched).toBe("none"); + }); + + it("emits a signal on match", async () => { + const step = makeStep([{ signal: "binary_doc", match: () => true }]); + + const result = await step.run(makeStepContext()); + + expect(result.status).toBe("ok"); + expect(result.effects?.signals).toEqual({ binary_doc: true }); + expect(result.diagnostics?.attributes?.matched).toBe("binary_doc"); + }); + + it("emits multiple signals when multiple rules match", async () => { + const step = makeStep([ + { signal: "binary_doc", match: () => true }, + { signal: "code_host", match: () => true } + ]); + + const result = await step.run(makeStepContext()); + + expect(result.effects?.signals).toEqual({ binary_doc: true, code_host: true }); + expect(result.diagnostics?.attributes?.matched).toContain("binary_doc"); + expect(result.diagnostics?.attributes?.matched).toContain("code_host"); + }); + + it("deduplicates same signal across multiple matching rules", async () => { + const step = makeStep([ + { signal: "binary_doc", match: () => true }, + { signal: "binary_doc", match: () => true } + ]); + + const result = await step.run(makeStepContext()); + + expect(result.effects?.signals).toEqual({ binary_doc: true }); + }); + + it("emits no effects when rules array is empty", async () => { + const step = makeStep([]); + + const result = await step.run(makeStepContext()); + + expect(result.status).toBe("ok"); + expect(result.effects).toBeUndefined(); + }); +}); + +function makeStep(rules: ReadonlyArray): ClassifyUrlStep { + return new ClassifyUrlStep({ rules }, { logger: createTestLogger() }); +} diff --git a/tests/builtins/source-providers/docling/docling-provider-config.test.ts b/tests/builtins/source-providers/docling/docling-provider-config.test.ts index 3fd05ce..7dab949 100644 --- a/tests/builtins/source-providers/docling/docling-provider-config.test.ts +++ b/tests/builtins/source-providers/docling/docling-provider-config.test.ts @@ -8,33 +8,54 @@ describe("parseDoclingConfig", () => { expect(parseDoclingConfig({ baseUrl: "http://docling.example" })).toMatchObject({ apiKey: "", output: "markdown", - doOcr: true, - tableMode: "accurate" + options: {} }); - expect( - parseDoclingConfig({ baseUrl: "http://docling.example", output: "html", doOcr: " false ", tableMode: "fast" }) - ).toMatchObject({ - output: "html", - doOcr: false, - tableMode: "fast" - }); - expect(parseDoclingConfig({ baseUrl: "http://docling.example", doOcr: "", apiKey: "", output: "" })).toMatchObject({ + expect(parseDoclingConfig({ baseUrl: "http://docling.example", apiKey: "", output: "" })).toMatchObject({ output: "markdown", - doOcr: true, apiKey: "" }); }); - it("rejects invalid boolean strings", () => { - expect(() => parseDoclingConfig({ baseUrl: "http://docling.example", doOcr: "yes" })).toThrow(); + it("preserves opaque options verbatim", () => { + const config = parseDoclingConfig({ + baseUrl: "http://docling.example", + options: { do_ocr: false, table_mode: "fast", ocr_preset: "easyocr", nested: { force_full_page_ocr: true } } + }); + expect(config.options).toEqual({ + do_ocr: false, + table_mode: "fast", + ocr_preset: "easyocr", + nested: { force_full_page_ocr: true } + }); }); - it("rejects invalid output values", () => { - expect(() => parseDoclingConfig({ baseUrl: "http://docling.example", output: "rawHtml" })).toThrow(); + it("parses a JSON-string blob from a single env var", () => { + const config = parseDoclingConfig({ + baseUrl: "http://docling.example", + options: '{"do_ocr":false,"table_mode":"fast"}' + }); + expect(config.options).toEqual({ do_ocr: false, table_mode: "fast" }); + }); + + it("coerces a blank env var to empty options", () => { + const config = parseDoclingConfig({ baseUrl: "http://docling.example", options: "" }); + expect(config.options).toEqual({}); }); - it("rejects invalid tableMode values", () => { - expect(() => parseDoclingConfig({ baseUrl: "http://docling.example", tableMode: "exact" })).toThrow(); + it("passes through inline objects (non-string) unmodified", () => { + const config = parseDoclingConfig({ + baseUrl: "http://docling.example", + options: { pipeline: "standard" } + }); + expect(config.options).toEqual({ pipeline: "standard" }); + }); + + it("rejects invalid JSON string", () => { + expect(() => parseDoclingConfig({ baseUrl: "http://docling.example", options: "{invalid" })).toThrow(); + }); + + it("rejects invalid output values", () => { + expect(() => parseDoclingConfig({ baseUrl: "http://docling.example", output: "rawHtml" })).toThrow(); }); it("rejects unknown fields", () => { diff --git a/tests/builtins/source-providers/docling/docling-provider.test.ts b/tests/builtins/source-providers/docling/docling-provider.test.ts index 126b04f..c502ad1 100644 --- a/tests/builtins/source-providers/docling/docling-provider.test.ts +++ b/tests/builtins/source-providers/docling/docling-provider.test.ts @@ -21,7 +21,7 @@ describe("DoclingProvider", () => { }); }; const provider = new DoclingProvider( - parseDoclingConfig({ baseUrl: "http://docling.example", doOcr: false, tableMode: "fast" }), + parseDoclingConfig({ baseUrl: "http://docling.example", options: { do_ocr: false, table_mode: "fast" } }), { httpFetch: fetchFn, logger: createTestLogger() } ); @@ -30,10 +30,9 @@ describe("DoclingProvider", () => { expect(requestBody).toEqual({ sources: [{ kind: "http", url: "https://example.com" }], options: { - to_formats: ["md", "json"], - image_export_mode: "placeholder", do_ocr: false, - table_mode: "fast" + table_mode: "fast", + to_formats: ["md", "json"] } }); expect(document).toEqual({ kind: "text", content: "# hello", mediaType: "text/markdown", title: "Hello" }); @@ -62,6 +61,29 @@ describe("DoclingProvider", () => { expect(document).toEqual({ kind: "text", content: "

Hello

", mediaType: "text/html", title: "Hello" }); }); + it("passes opaque options but keeps to_formats provider-managed", async () => { + let requestBody: { options?: Record } | undefined; + const fetchFn: typeof fetch = async (_input, init) => { + requestBody = JSON.parse(String(init?.body)); + return jsonResponse({ document: { md_content: "# ok" }, status: "success", processing_time: 0.5 }); + }; + const provider = new DoclingProvider( + parseDoclingConfig({ + baseUrl: "http://docling.example", + options: { image_export_mode: "embedded", force_ocr: true, to_formats: ["html"] } + }), + { httpFetch: fetchFn, logger: createTestLogger() } + ); + + await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(requestBody?.options).toEqual({ + image_export_mode: "embedded", + force_ocr: true, + to_formats: ["md", "json"] + }); + }); + it("accepts partial_success status and missing title", async () => { const provider = new DoclingProvider(parseDoclingConfig({ baseUrl: "http://docling.example" }), { httpFetch: async () => diff --git a/tests/builtins/source-providers/firecrawl/firecrawl-provider-config.test.ts b/tests/builtins/source-providers/firecrawl/firecrawl-provider-config.test.ts index 6529b1d..5297441 100644 --- a/tests/builtins/source-providers/firecrawl/firecrawl-provider-config.test.ts +++ b/tests/builtins/source-providers/firecrawl/firecrawl-provider-config.test.ts @@ -4,51 +4,54 @@ import { describe, expect, it } from "vitest"; import { parseFirecrawlConfig } from "../../../../src/builtins/source-providers/firecrawl/firecrawl-provider-config.js"; describe("parseFirecrawlConfig", () => { - it("applies defaults and coerces env-substituted scalar strings", () => { + it("applies defaults and coerces env-substituted blank strings", () => { expect(parseFirecrawlConfig({ baseUrl: "https://firecrawl.example" })).toMatchObject({ output: "markdown", - onlyMainContent: true, - stripBase64Images: true, - parsePdf: true + options: {} }); + }); + + it("coerces blank output to markdown", () => { + expect(parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "" })).toMatchObject({ + output: "markdown" + }); + }); + + it("parses options from a JSON string", () => { expect( - parseFirecrawlConfig({ - baseUrl: "https://firecrawl.example", - output: "html", - onlyMainContent: " false ", - stripBase64Images: " false ", - parsePdf: " false " - }) + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", options: '{"onlyMainContent":false}' }) ).toMatchObject({ - output: "html", - onlyMainContent: false, - stripBase64Images: false, - parsePdf: false + options: { onlyMainContent: false } }); + }); + + it("returns empty options for blank env vars", () => { + expect(parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", options: "" })).toMatchObject({ + options: {} + }); + }); + + it("passes through inline YAML object options", () => { expect( parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", - onlyMainContent: "", - stripBase64Images: "", - parsePdf: "" + options: { onlyMainContent: false, timeout: 30000 } }) ).toMatchObject({ - onlyMainContent: true, - stripBase64Images: true, - parsePdf: true + options: { onlyMainContent: false, timeout: 30000 } }); }); + it("rejects invalid JSON in options", () => { + expect(() => parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", options: "not-json" })).toThrow(); + }); + it("accepts rawHtml output", () => { expect(parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "rawHtml" })).toMatchObject({ output: "rawHtml" }); }); - it("rejects invalid boolean strings", () => { - expect(() => parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", onlyMainContent: "yes" })).toThrow(); - }); - it("rejects invalid output values", () => { expect(() => parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "markup" })).toThrow(); }); diff --git a/tests/builtins/source-providers/firecrawl/firecrawl-provider.test.ts b/tests/builtins/source-providers/firecrawl/firecrawl-provider.test.ts index af99e36..5a7316e 100644 --- a/tests/builtins/source-providers/firecrawl/firecrawl-provider.test.ts +++ b/tests/builtins/source-providers/firecrawl/firecrawl-provider.test.ts @@ -8,7 +8,7 @@ import { createTestLogger } from "../../../helpers/logger.js"; import { jsonResponse } from "../../../helpers/responses.js"; describe("FirecrawlProvider", () => { - it("sends the supported scrape request and returns markdown content", async () => { + it("sends the scrape request with only provider-managed fields and returns markdown", async () => { let requestBody: Record | undefined; const fetchFn: typeof fetch = async (input, init) => { requestBody = JSON.parse(String(init?.body)); @@ -16,28 +16,43 @@ describe("FirecrawlProvider", () => { expect(init?.method).toBe("POST"); return jsonResponse({ success: true, data: { markdown: "# hello", metadata: { title: "Hello" } } }); }; + const provider = new FirecrawlProvider(parseFirecrawlConfig({ baseUrl: "https://firecrawl.example" }), { + httpFetch: fetchFn, + logger: createTestLogger() + }); + + const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + + expect(requestBody).toEqual({ + url: "https://example.com", + formats: ["markdown"] + }); + expect(document).toEqual({ kind: "text", content: "# hello", mediaType: "text/markdown", title: "Hello" }); + }); + + it("merges options into the body and locks formats last", async () => { + let requestBody: Record | undefined; + const fetchFn: typeof fetch = async (_input, init) => { + requestBody = JSON.parse(String(init?.body)); + return jsonResponse({ success: true, data: { markdown: "merged" } }); + }; const provider = new FirecrawlProvider( parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", - apiKey: "", output: "markdown", - onlyMainContent: false, - stripBase64Images: true, - parsePdf: true + options: { onlyMainContent: false, removeBase64Images: true, formats: ["should-be-overridden"] } }), { httpFetch: fetchFn, logger: createTestLogger() } ); - const document = await provider.load("https://example.com", { signal: new AbortController().signal }); + await provider.load("https://example.com", { signal: new AbortController().signal }); - expect(requestBody).toEqual({ + expect(requestBody).toMatchObject({ url: "https://example.com", - formats: ["markdown"], onlyMainContent: false, - removeBase64Images: true, - parsers: ["pdf"] + removeBase64Images: true }); - expect(document).toEqual({ kind: "text", content: "# hello", mediaType: "text/markdown", title: "Hello" }); + expect(requestBody?.formats).toEqual(["markdown"]); }); it("returns html content when output is html", async () => { @@ -166,7 +181,7 @@ describe("FirecrawlProvider", () => { }); }); - it("returns binary body when parsePdf is false for a PDF", async () => { + it("returns binary body when content is %PDF base64 passthrough", async () => { const base64Payload = Buffer.from("%PDF-1.4 test document").toString("base64"); const fetchFn: typeof fetch = async (_input, init) => { return jsonResponse({ @@ -175,7 +190,7 @@ describe("FirecrawlProvider", () => { }); }; const provider = new FirecrawlProvider( - parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "markdown", parsePdf: false }), + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "markdown" }), { httpFetch: fetchFn, logger: createTestLogger() } ); @@ -188,7 +203,7 @@ describe("FirecrawlProvider", () => { } }); - it("strips html wrapper from binary base64 when parsePdf is false and output is html", async () => { + it("strips html wrapper before %PDF content sniff when output is html", async () => { const payload = "JVBERi0xLjQK"; const fetchFn: typeof fetch = async () => { return jsonResponse({ @@ -197,7 +212,7 @@ describe("FirecrawlProvider", () => { }); }; const provider = new FirecrawlProvider( - parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "html", parsePdf: false }), + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "html" }), { httpFetch: fetchFn, logger: createTestLogger() } ); @@ -209,9 +224,9 @@ describe("FirecrawlProvider", () => { } }); - it("returns text body when parsePdf is false for a non-PDF (HTML) source", async () => { + it("returns text body for non-PDF content (HTML markdown)", async () => { const provider = new FirecrawlProvider( - parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "markdown", parsePdf: false }), + parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "markdown" }), { httpFetch: async () => jsonResponse({ @@ -234,61 +249,43 @@ describe("FirecrawlProvider", () => { }); }); - it("returns text body when parsePdf is false for an image source", async () => { - const base64Payload = Buffer.from("RIFF fake").toString("base64"); + it("returns text body for non-PDF base64 content", async () => { + const base64Payload = Buffer.from("RIFF fake image").toString("base64"); const fetchFn: typeof fetch = async () => { return jsonResponse({ success: true, data: { - markdown: `![](https://example.com/img.jpg)`, + markdown: base64Payload, metadata: { contentType: "image/png" } } }); }; - const provider = new FirecrawlProvider( - parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", parsePdf: false }), - { httpFetch: fetchFn, logger: createTestLogger() } - ); + const provider = new FirecrawlProvider(parseFirecrawlConfig({ baseUrl: "https://firecrawl.example" }), { + httpFetch: fetchFn, + logger: createTestLogger() + }); const document = await provider.load("https://example.com", { signal: new AbortController().signal }); expect(document.kind).toBe("text"); + if (document.kind === "text") expect(document.content).toBe(base64Payload); }); - it("rejects empty base64 when parsePdf is false for PDF", async () => { - const provider = new FirecrawlProvider( - parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", parsePdf: false }), - { - httpFetch: async () => - jsonResponse({ - success: true, - data: { markdown: "", metadata: { contentType: "application/pdf" } } - }), - logger: createTestLogger() - } - ); + it("rejects empty content as upstream empty", async () => { + const provider = new FirecrawlProvider(parseFirecrawlConfig({ baseUrl: "https://firecrawl.example" }), { + httpFetch: async () => + jsonResponse({ + success: true, + data: { markdown: "", metadata: { contentType: "application/pdf" } } + }), + logger: createTestLogger() + }); await expect(provider.load("https://example.com", { signal: new AbortController().signal })).rejects.toMatchObject({ upstreamCode: "empty" }); }); - it("omits parsers when parsePdf is false", async () => { - let requestBody: Record | undefined; - const fetchFn: typeof fetch = async (_input, init) => { - requestBody = JSON.parse(String(init?.body)); - return jsonResponse({ success: true, data: { markdown: "ok" } }); - }; - const provider = new FirecrawlProvider( - parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", parsePdf: false }), - { httpFetch: fetchFn, logger: createTestLogger() } - ); - - await provider.load("https://example.com", { signal: new AbortController().signal }); - - expect(requestBody?.parsers).toEqual([]); - }); - it("accepts top-level response fields for any output format", async () => { const mdProvider = new FirecrawlProvider( parseFirecrawlConfig({ baseUrl: "https://firecrawl.example", output: "markdown" }), diff --git a/tests/config/yaml/yaml-app-config.test.ts b/tests/config/yaml/yaml-app-config.test.ts index a2caa06..d295aa7 100644 --- a/tests/config/yaml/yaml-app-config.test.ts +++ b/tests/config/yaml/yaml-app-config.test.ts @@ -29,6 +29,44 @@ describe("yamlToAppConfig", () => { expect(appConfig.adapters.http).toEqual(raw.httpAdapters); expect(appConfig.metadata?.schemaVersion).toBe(1); }); + + it("preserves conditional step gates", () => { + const raw = rawYamlConfigSchema.parse({ + schemaVersion: 1, + pipelines: { + default: { + steps: [ + { + type: "transform", + name: "clean", + runIf: { all: ["binary_doc", { not: "code_host" }] }, + skipIf: "skip_clean" + } + ] + } + } + }); + + const appConfig = yamlToAppConfig(raw); + + expect(appConfig.engineConfig.pipelines.default?.steps[0]).toMatchObject({ + runIf: { all: ["binary_doc", { not: "code_host" }] }, + skipIf: "skip_clean" + }); + }); + + it("rejects invalid conditional gate signal names", () => { + expect(() => + rawYamlConfigSchema.parse({ + schemaVersion: 1, + pipelines: { + default: { + steps: [{ type: "transform", name: "clean", runIf: "BinaryDoc" }] + } + } + }) + ).toThrow("Invalid signal name"); + }); }); describe("loadYamlAppConfig", () => { diff --git a/tests/core/pipeline/conditions.test.ts b/tests/core/pipeline/conditions.test.ts new file mode 100644 index 0000000..fabc834 --- /dev/null +++ b/tests/core/pipeline/conditions.test.ts @@ -0,0 +1,119 @@ +/** Verifies Condition evaluation and gate resolution. */ +import { describe, expect, it } from "vitest"; + +import { evaluateCondition, isTruthySignal, resolveStepGate } from "../../../src/core/pipeline/conditions.js"; + +describe("isTruthySignal", () => { + it("returns false for undefined", () => { + expect(isTruthySignal(undefined)).toBe(false); + }); + + it("returns false for false", () => { + expect(isTruthySignal(false)).toBe(false); + }); + + it("returns false for 0", () => { + expect(isTruthySignal(0)).toBe(false); + }); + + it("returns false for empty string", () => { + expect(isTruthySignal("")).toBe(false); + }); + + it("returns true for boolean true", () => { + expect(isTruthySignal(true)).toBe(true); + }); + + it("returns true for a non-empty string", () => { + expect(isTruthySignal("matched")).toBe(true); + }); + + it("returns true for a non-zero number", () => { + expect(isTruthySignal(1)).toBe(true); + }); +}); + +describe("evaluateCondition", () => { + const lookup = (name: string) => ({ binary_doc: true, code_host: true, pdf: "yes", flag_off: false })[name]; + + it("returns true when a signal exists and is truthy", () => { + expect(evaluateCondition("binary_doc", lookup)).toBe(true); + }); + + it("returns false when a signal is missing", () => { + expect(evaluateCondition("missing_signal", lookup)).toBe(false); + }); + + it("returns false when a signal exists but is falsy", () => { + expect(evaluateCondition("flag_off", lookup)).toBe(false); + }); + + it("accepts a non-boolean truthy signal", () => { + expect(evaluateCondition("pdf", lookup)).toBe(true); + }); + + it("evaluates all: empty array as true", () => { + expect(evaluateCondition({ all: [] }, lookup)).toBe(true); + }); + + it("evaluates all: all truthy as true", () => { + expect(evaluateCondition({ all: ["binary_doc", "code_host"] }, lookup)).toBe(true); + }); + + it("evaluates all: one falsy as false", () => { + expect(evaluateCondition({ all: ["binary_doc", "missing_signal"] }, lookup)).toBe(false); + }); + + it("evaluates any: empty array as false", () => { + expect(evaluateCondition({ any: [] }, lookup)).toBe(false); + }); + + it("evaluates any: one truthy as true", () => { + expect(evaluateCondition({ any: ["binary_doc", "missing_signal"] }, lookup)).toBe(true); + }); + + it("evaluates any: all falsy as false", () => { + expect(evaluateCondition({ any: ["missing_signal", "flag_off"] }, lookup)).toBe(false); + }); + + it("evaluates not: inverts true", () => { + expect(evaluateCondition({ not: "binary_doc" }, lookup)).toBe(false); + }); + + it("evaluates not: inverts false", () => { + expect(evaluateCondition({ not: "missing_signal" }, lookup)).toBe(true); + }); + + it("evaluates deeply nested conditions", () => { + const cond = { any: [{ all: ["binary_doc", "code_host"] }, { not: "missing_signal" }] }; + expect(evaluateCondition(cond, lookup)).toBe(true); + }); +}); + +describe("resolveStepGate", () => { + const lookup = (name: string) => ({ binary_doc: true, code_host: true })[name]; + + it("returns undefined when no gate is set", () => { + expect(resolveStepGate({}, lookup)).toBeUndefined(); + }); + + it("returns undefined when runIf matches", () => { + expect(resolveStepGate({ runIf: "binary_doc" }, lookup)).toBeUndefined(); + }); + + it("returns run_if_unmet when runIf does not match", () => { + expect(resolveStepGate({ runIf: "missing" }, lookup)).toBe("run_if_unmet"); + }); + + it("returns undefined when skipIf does not match", () => { + expect(resolveStepGate({ skipIf: "missing" }, lookup)).toBeUndefined(); + }); + + it("returns skip_if_met when skipIf matches", () => { + expect(resolveStepGate({ skipIf: "code_host" }, lookup)).toBe("skip_if_met"); + }); + + it("prefers runIf over skipIf when both are present", () => { + expect(resolveStepGate({ runIf: "missing", skipIf: "code_host" }, lookup)).toBe("run_if_unmet"); + }); +}); diff --git a/tests/core/pipeline/orchestrator.test.ts b/tests/core/pipeline/orchestrator.test.ts index b6cc0c7..5c783a9 100644 --- a/tests/core/pipeline/orchestrator.test.ts +++ b/tests/core/pipeline/orchestrator.test.ts @@ -383,6 +383,155 @@ describe("PipelineOrchestrator", () => { expect(result.report.finalLength).toBe(4); expect(result.report.returned).toBe("fetch"); }); + + it("gates a step with runIf unmet", async () => { + let ran = false; + const gatedStep: PipelineStep & { readonly name: string; readonly type: string } = { + name: "gated", + type: "tracker", + async run(_ctx: PipelineContext): Promise { + ran = true; + return { status: "ok" }; + } + }; + const pipeline = makePipeline({ + steps: [ + withMeta( + new FakeStep("source", { + status: "ok", + effects: { signals: { has_pdf: false } } + }), + 5 + ), + withMeta(gatedStep, 5, undefined, { runIf: "has_pdf" }) + ] + }); + + const result = await makeOrchestrator().run(pipeline, { url: "https://example.com/" }); + + expect(ran).toBe(false); + expect(result.report.steps[1]?.status).toBe("skipped"); + expect(result.report.steps[1]?.reason).toBe("run_if_unmet"); + }); + + it("gates a step with skipIf met", async () => { + let ran = false; + const gatedStep: PipelineStep & { readonly name: string; readonly type: string } = { + name: "gated", + type: "tracker", + async run(_ctx: PipelineContext): Promise { + ran = true; + return { status: "ok" }; + } + }; + const pipeline = makePipeline({ + steps: [ + withMeta( + new FakeStep("source", { + status: "ok", + effects: { signals: { code_host: true } } + }), + 5 + ), + withMeta(gatedStep, 5, undefined, { skipIf: "code_host" }) + ] + }); + + const result = await makeOrchestrator().run(pipeline, { url: "https://example.com/" }); + + expect(ran).toBe(false); + expect(result.report.steps[1]?.status).toBe("skipped"); + expect(result.report.steps[1]?.reason).toBe("skip_if_met"); + }); + + it("runs normally when gate signals are absent or falsy", async () => { + let ran = false; + const gatedStep: PipelineStep & { readonly name: string; readonly type: string } = { + name: "gated", + type: "tracker", + async run(_ctx: PipelineContext): Promise { + ran = true; + return { status: "ok", effects: { body: textBody({ content: "hello" }) } }; + } + }; + const pipeline = makePipeline({ + steps: [ + withMeta( + new FakeStep("source", { + status: "ok", + effects: { signals: { flag: false } } + }), + 5 + ), + withMeta(gatedStep, 5, undefined, { runIf: "flag", skipIf: "missing" }) + ] + }); + + const result = await makeOrchestrator().run(pipeline, { url: "https://example.com/" }); + + expect(ran).toBe(false); + expect(result.report.steps[1]?.status).toBe("skipped"); + expect(result.report.steps[1]?.reason).toBe("run_if_unmet"); + }); + + it("applies effects from a gated step's prior step before evaluating gate", async () => { + let ran = false; + const pdfStep: PipelineStep & { readonly name: string; readonly type: string } = { + name: "pdf_only", + type: "tracker", + async run(_ctx: PipelineContext): Promise { + ran = true; + return { status: "ok" }; + } + }; + const pipeline = makePipeline({ + steps: [ + withMeta( + new FakeStep("classify", { + status: "ok", + effects: { signals: { is_pdf: true } } + }), + 5 + ), + withMeta(pdfStep, 5, undefined, { runIf: "is_pdf" }) + ] + }); + + const result = await makeOrchestrator().run(pipeline, { url: "https://example.com/" }); + + expect(ran).toBe(true); + expect(result.report.steps[1]?.status).toBe("ok"); + }); + + it("records a gated skip in outcomes visible to later steps", async () => { + const outcomes: string[] = []; + const gatedStep: PipelineStep & { readonly name: string; readonly type: string } = { + name: "gated", + type: "fake", + async run(): Promise { + return { status: "ok" }; + } + }; + const inspectorStep: PipelineStep & { readonly name: string; readonly type: string } = { + name: "inspector", + type: "fake", + async run(ctx: PipelineContext): Promise { + for (const o of ctx.outcomes) outcomes.push(`${o.name}=${o.status}:${o.reason ?? "-"}`); + return { status: "ok" }; + } + }; + const pipeline = makePipeline({ + steps: [ + withMeta(new FakeStep("classify", { status: "ok", effects: { signals: { skip_me: true } } }), 5), + withMeta(gatedStep, 5, undefined, { skipIf: "skip_me" }), + withMeta(inspectorStep, 5) + ] + }); + + await makeOrchestrator().run(pipeline, { url: "https://example.com/" }); + + expect(outcomes).toContain("gated=skipped:skip_if_met"); + }); }); function makeOrchestrator(): PipelineOrchestrator { @@ -392,7 +541,8 @@ function makeOrchestrator(): PipelineOrchestrator { function withMeta( step: T, timeoutSeconds: number, - concurrencyGroup?: string + concurrencyGroup?: string, + overrides?: Partial> ): CompiledPipelineStep { - return { name: step.name, type: step.type, timeoutSeconds, concurrencyGroup, step }; + return { name: step.name, type: step.type, timeoutSeconds, concurrencyGroup, ...overrides, step }; } diff --git a/tests/engine/step-disable.test.ts b/tests/engine/step-disable.test.ts new file mode 100644 index 0000000..a7f753b --- /dev/null +++ b/tests/engine/step-disable.test.ts @@ -0,0 +1,312 @@ +/** Verifies disabled-step compile-time exclusion (D1). */ +import { describe, expect, it } from "vitest"; + +import { createEngine } from "../../src/engine/create-engine.js"; +import { createTestHostTools } from "../helpers/host-tools.js"; +import { createTestLogger, type CapturedLog } from "../helpers/logger.js"; +import { markdownRendererDescriptor } from "./renderer-descriptor.js"; + +import { sourceProviderRegistryKey } from "../../src/contracts/extensions/source-provider.js"; +import type { SourceProviderDescriptor } from "../../src/contracts/extensions/source-provider.js"; +import type { PipelineStepDescriptor } from "../../src/contracts/pipeline/step.js"; + +describe("step disable / compile-time exclusion", () => { + it("disabled step with invalid provider config does not fail startup", async () => { + await expect( + createEngine({ + config: { + sourceProviders: { + "bad-provider": { type: "validated-source", config: { requiredField: "" } } + }, + contentTransformers: {}, + llmProviders: {}, + outputRenderers: { markdown: { type: "markdown", config: {} } }, + pipelines: { + default: { + enabled: true, + outputRenderer: "markdown", + limiters: {}, + steps: [ + { + type: "uses-provider", + name: "fetch", + enabled: false, + timeoutSeconds: 5, + config: { provider: "bad-provider" } + }, + { type: "static-body", name: "write", timeoutSeconds: 5, config: { content: "ok" } } + ] + } + } + }, + descriptors: { + sourceProviders: { + "validated-source": { + type: "validated-source", + parseConfig: raw => raw, + create: ({ config }) => { + if (!config || !(config as { requiredField: string }).requiredField) + throw new Error("validated-source requiredField is missing"); + return { load: async () => ({ kind: "text" as const, content: "", mediaType: "text/markdown" }) }; + } + } satisfies SourceProviderDescriptor + }, + contentTransformers: {}, + llmProviders: {}, + outputRenderers: { markdown: markdownRendererDescriptor }, + pipelineSteps: { + "uses-provider": { + type: "uses-provider", + parseConfig: raw => raw as { provider: string }, + create: async ({ config, services }) => { + const cfg = config as { provider: string }; + services.require(sourceProviderRegistryKey).require(cfg.provider); + return { run: async () => ({ status: "ok" as const }) }; + } + } satisfies PipelineStepDescriptor, + "static-body": staticBodyStepDescriptor + } + }, + tools: createTestHostTools(), + logger: createTestLogger() + }) + ).resolves.toBeDefined(); + }); + + it("disabled step is absent from compiled pipeline steps", async () => { + const loaded = await createEngine({ + config: { + sourceProviders: {}, + contentTransformers: {}, + llmProviders: {}, + outputRenderers: { markdown: { type: "markdown", config: {} } }, + pipelines: { + default: { + enabled: true, + outputRenderer: "markdown", + limiters: {}, + steps: [ + { type: "static-body", name: "should_appear", timeoutSeconds: 5, config: { content: "a" } }, + { + type: "static-body", + name: "should_not_appear", + enabled: false, + timeoutSeconds: 5, + config: { content: "b" } + }, + { type: "static-body", name: "also_appears", timeoutSeconds: 5, config: { content: "c" } } + ] + } + } + }, + descriptors: { + sourceProviders: {}, + contentTransformers: {}, + llmProviders: {}, + outputRenderers: { markdown: markdownRendererDescriptor }, + pipelineSteps: { "static-body": staticBodyStepDescriptor } + }, + tools: createTestHostTools(), + logger: createTestLogger() + }); + + expect( + loaded + .listPipelines() + .find(p => p.name === "default") + ?.steps.map(s => s.name) + ).toEqual(["should_appear", "also_appears"]); + }); + + it("disabled step with unknown type does not throw", async () => { + await expect( + createEngine({ + config: { + sourceProviders: {}, + contentTransformers: {}, + llmProviders: {}, + outputRenderers: { markdown: { type: "markdown", config: {} } }, + pipelines: { + default: { + enabled: true, + outputRenderer: "markdown", + limiters: {}, + steps: [ + { type: "nonexistent-step-type", name: "ghost", enabled: false, timeoutSeconds: 5, config: {} }, + { type: "static-body", name: "real", timeoutSeconds: 5, config: { content: "ok" } } + ] + } + } + }, + descriptors: { + sourceProviders: {}, + contentTransformers: {}, + llmProviders: {}, + outputRenderers: { markdown: markdownRendererDescriptor }, + pipelineSteps: { "static-body": staticBodyStepDescriptor } + }, + tools: createTestHostTools(), + logger: createTestLogger() + }) + ).resolves.toBeDefined(); + }); + + it("disabled step does not participate in duplicate-name detection", async () => { + await expect( + createEngine({ + config: { + sourceProviders: {}, + contentTransformers: {}, + llmProviders: {}, + outputRenderers: { markdown: { type: "markdown", config: {} } }, + pipelines: { + default: { + enabled: true, + outputRenderer: "markdown", + limiters: {}, + steps: [ + { type: "static-body", name: "dup", enabled: false, timeoutSeconds: 5, config: { content: "first" } }, + { type: "static-body", name: "dup", timeoutSeconds: 5, config: { content: "second" } } + ] + } + } + }, + descriptors: { + sourceProviders: {}, + contentTransformers: {}, + llmProviders: {}, + outputRenderers: { markdown: markdownRendererDescriptor }, + pipelineSteps: { "static-body": staticBodyStepDescriptor } + }, + tools: createTestHostTools(), + logger: createTestLogger() + }) + ).resolves.toBeDefined(); + }); + + it("logs steps disabled; excluded from compilation with disabled names", async () => { + const logs: CapturedLog[] = []; + await createEngine({ + config: { + sourceProviders: {}, + contentTransformers: {}, + llmProviders: {}, + outputRenderers: { markdown: { type: "markdown", config: {} } }, + pipelines: { + default: { + enabled: true, + outputRenderer: "markdown", + limiters: {}, + steps: [ + { type: "static-body", name: "off1", enabled: false, timeoutSeconds: 5, config: { content: "a" } }, + { type: "static-body", name: "on", timeoutSeconds: 5, config: { content: "b" } }, + { type: "static-body", name: "off2", enabled: false, timeoutSeconds: 5, config: { content: "c" } } + ] + } + } + }, + descriptors: { + sourceProviders: {}, + contentTransformers: {}, + llmProviders: {}, + outputRenderers: { markdown: markdownRendererDescriptor }, + pipelineSteps: { "static-body": staticBodyStepDescriptor } + }, + tools: createTestHostTools(), + logger: createTestLogger(logs) + }); + + const disabledLog = logs.find( + log => log.level === "info" && log.message === "steps disabled; excluded from compilation" + ); + expect(disabledLog?.value).toEqual({ pipeline: "default", disabled: ["off1", "off2"] }); + }); + + it("logs pipeline disabled; excluded from compilation for a disabled pipeline", async () => { + const logs: CapturedLog[] = []; + await createEngine({ + config: { + sourceProviders: {}, + contentTransformers: {}, + llmProviders: {}, + outputRenderers: { markdown: { type: "markdown", config: {} } }, + pipelines: { + active: { + enabled: true, + outputRenderer: "markdown", + limiters: {}, + steps: [{ type: "static-body", name: "write", timeoutSeconds: 5, config: { content: "ok" } }] + }, + parked: { + enabled: false, + outputRenderer: "markdown", + limiters: {}, + steps: [] + } + } + }, + descriptors: { + sourceProviders: {}, + contentTransformers: {}, + llmProviders: {}, + outputRenderers: { markdown: markdownRendererDescriptor }, + pipelineSteps: { "static-body": staticBodyStepDescriptor } + }, + tools: createTestHostTools(), + logger: createTestLogger(logs) + }); + + const disabledLog = logs.find( + log => log.level === "info" && log.message === "pipeline disabled; excluded from compilation" + ); + expect(disabledLog?.value).toEqual({ pipeline: "parked" }); + }); + + it("warns when a pipeline compiles with no enabled steps", async () => { + const logs: CapturedLog[] = []; + await createEngine({ + config: { + sourceProviders: {}, + contentTransformers: {}, + llmProviders: {}, + outputRenderers: { markdown: { type: "markdown", config: {} } }, + pipelines: { + empty: { + enabled: true, + outputRenderer: "markdown", + limiters: {}, + steps: [{ type: "static-body", name: "off", enabled: false, timeoutSeconds: 5, config: { content: "a" } }] + } + } + }, + descriptors: { + sourceProviders: {}, + contentTransformers: {}, + llmProviders: {}, + outputRenderers: { markdown: markdownRendererDescriptor }, + pipelineSteps: { "static-body": staticBodyStepDescriptor } + }, + tools: createTestHostTools(), + logger: createTestLogger(logs) + }); + + const warnLog = logs.find(log => log.level === "warn" && log.message === "pipeline compiled with no enabled steps"); + expect(warnLog?.value).toEqual({ pipeline: "empty" }); + }); +}); + +const staticBodyStepDescriptor = { + type: "static-body", + parseConfig: (raw: unknown) => raw, + create: ({ config }: { config: unknown }) => { + const cfg = config as { content: string }; + return { + run: async () => ({ + status: "ok" as const, + effects: { + body: { kind: "text" as const, content: cfg.content, mediaType: "text/markdown" as const } + } + }) + }; + } +} satisfies PipelineStepDescriptor;