feat!: add content transformers (readability, mdream) and cleanup pipelines

.env.example

@@ -23,50 +23,69 @@ LOG_PRETTY=auto
 
 # --- Pipeline ------------------------------------------------------------
 # Active pipeline selected by HTTP adapters. Default: truncate (load-source + truncate)
-DEFAULT_PIPELINE=truncate
+DEFAULT_PIPELINE=clean-combined
 
-# Maximum concurrent source-loading groups. Default: 1
-SOURCE_CONCURRENCY=4
+# Maximum concurrent source-loading groups. Default: 1; raise only if providers can handle it.
+SOURCE_CONCURRENCY=5
 
-# Maximum concurrent LLM workflow groups. Default: 1
+# Maximum concurrent processing (transform) groups. Default: 5
+PROCESS_CONCURRENCY=5
+
+# Maximum concurrent LLM workflow groups. Default: 1; raise only if the model endpoint can handle it.
 LLM_CONCURRENCY=2
 
-# Output renderer used by the default pipeline. Default: debug-xml
+# Output renderer override used by shipped pipelines. Default: debug-xml
 DEFAULT_OUTPUT_RENDERER=debug-xml
 
 # Include skipped steps in the debug-xml footer. Default: false
 DEBUG_XML_INCLUDE_SKIPPED=false
 
-# Desired maximum output size used by summarize and truncate steps. Default: 25000
+# Desired maximum output size used by summarize and truncate steps. Default: 25000; larger values return more content.
 OUTPUT_TARGET_CHARS=35000
 
 # --- Source provider -----------------------------------------------------
-# Active source provider name used by pipelines (truncate and clean-llm). Default: default-http (native HTTP fetch, no external service needed)
-SOURCE_PROVIDER=default-http
+# Optional source-provider override. Leave empty to use the selected pipeline's fallback.
+SOURCE_PROVIDER=
 
-# Firecrawl base URL. Required only when a referenced provider uses it (e.g. default-firecrawl).
+# Firecrawl base URL. Required only when a referenced provider uses it (e.g. firecrawl-markdown, firecrawl-html).
 FIRECRAWL_BASE_URL=https://firecrawl.example
 
 # Optional Firecrawl bearer token. Default: empty (no token)
 FIRECRAWL_API_KEY=
 
-# Docling Serve base URL. Required only when a referenced provider uses it (e.g. default-docling).
+# Firecrawl PDF parsing. Set to false to return raw PDF bytes (binary body) instead of parsed text.
+FIRECRAWL_PARSE_PDF=
+
+# Docling Serve base URL. Required only when a referenced provider uses it (e.g. docling-ocr).
 DOCLING_BASE_URL=https://docling.example
 
 # Optional Docling API key (X-Api-Key header). Default: empty (no token)
 DOCLING_API_KEY=
 
+# --- Content transformers ------------------------------------------------
+# Post-conversion link and whitespace cleanup for mdream-convert. Ignored when minimal=true (mdream-aggressive). Default: true
+MDREAM_CLEAN=true
+
+# Minimum content length for Readability's isProbablyReaderable gate. Default: 140
+READABILITY_MIN_CONTENT_CHARS=140
+
+# Minimum readerable score for Readability's isProbablyReaderable gate. Default: 20
+READABILITY_MIN_SCORE=20
+
+# Maximum DOM elements Readability parses; 0 = unlimited. Default: 0
+READABILITY_MAX_ELEMENTS=0
+
 # --- LLM provider --------------------------------------------------------
-# OpenAI-compatible Chat Completions /v1 base URL. Required only when a referenced provider uses it (e.g. default-llm).
+# OpenAI-compatible Chat Completions /v1 base URL. Required only when a referenced provider uses it (e.g. llm-default).
 LLM_BASE_URL=https://openai-compatible.example/v1
 
 # Optional LLM bearer token. Default: empty (no token)
 LLM_API_KEY=
 
-# Model identifier sent to the chat-completions endpoint. Required only when a referenced provider uses it (e.g. default-llm).
+# Model identifier sent to the chat-completions endpoint. Required only when a referenced provider uses it (e.g. llm-default).
 LLM_MODEL=model-name
 
-# Model context window in tokens. Default: empty (disables the context-fit gate)
+# Model context window in tokens. Default: empty (disables the context-fit gate); set to enable it.
 LLM_CONTEXT_TOKENS=131072
 
 # Conservative chars-per-token estimate used by the context-fit gate. Default: 3.5
@@ -79,10 +98,10 @@ LLM_SAFETY_MARGIN_TOKENS=128
 # Timeout in seconds for the source-loading step. Default: 20
 LOAD_SOURCE_TIMEOUT_SECONDS=20
 
-# Timeout in seconds for the clean LLM pass. Default: 60
+# Timeout in seconds for the clean LLM pass. Default: 60; raise for slower models.
 CLEAN_TIMEOUT_SECONDS=90
 
-# Minimum body length before the clean LLM pass runs. Default: 1000
+# Minimum body length before the clean LLM pass runs. Default: 1000; lower values run cleanup more often.
 CLEAN_MIN_INPUT_CHARS=500
 
 # Timeout in seconds for the summarize LLM pass. Default: 60

AGENTS.md

@@ -37,3 +37,13 @@ Read the row that matches your task. Each focused doc is self-contained for its
 - When changing behavior, keep the docs that describe it accurate in the same change.
 - No gratuitous comments. Inline comments that narrate a change are prohibited ([coding-conventions](docs/agents/coding-conventions.md)). Before adding a comment, ask: will it still be useful after the transient context is gone?
 - Don't over-document trivial changes. A new knob, helper, or internal refactor does not warrant updates to `ARCHITECTURE.md` or `extension-authoring.md`. Update only docs that directly describe the changed surface.
+
+## Terminal Behavior on Windows
+
+When using the terminal on Windows:
+
+- Prefer single-line PowerShell commands.
+- Avoid interactive commands, pagers, prompts, and commands that wait for input.
+- Prefer `pwsh` or PowerShell with `-NoLogo -NoProfile`.
+- For git commands, use non-interactive flags such as `--no-pager` and `--no-edit` when appropriate.
+- If a command appears stuck after output is printed, do not blindly rerun it. First check whether the command already completed and inspect the visible terminal output.

README.md

@@ -11,9 +11,10 @@ LLM Context Loader is a small HTTP service that turns URLs into markdown for LLM
 
 ## Current Shape
 
-- **Pipelines:** `truncate` (default, no LLM) and `clean-llm` (Firecrawl + LLM), selected via `DEFAULT_PIPELINE`.
+- **Pipelines:** `truncate` (default, no LLM), `clean-deterministic` (Firecrawl HTML + Readability + mdream, no LLM), `clean-llm` (Firecrawl + LLM), and `clean-combined` (Firecrawl HTML + Readability + mdream + LLM summarize), selected via `DEFAULT_PIPELINE`.
 - **Source providers:** native HTTP fetch, Firecrawl, Docling.
-- **LLM provider:** OpenAI-compatible `/chat/completions`.
+- **Content transformers:** `readability` (article HTML extraction), `mdream` (HTML to markdown).
+- **LLM providers:** OpenAI-compatible `/chat/completions`.
 - **Output renderers:** `debug-xml` and `passthrough`, selected per pipeline in YAML.
 
 ## Quick Start

compose.deploy.yaml

@@ -15,18 +15,26 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l
 
   # Pipeline concurrency and renderer selection.
   SOURCE_CONCURRENCY: "${SOURCE_CONCURRENCY:-1}"
+  PROCESS_CONCURRENCY: "${PROCESS_CONCURRENCY:-5}"
   LLM_CONCURRENCY: "${LLM_CONCURRENCY:-1}"
   DEFAULT_OUTPUT_RENDERER: "${DEFAULT_OUTPUT_RENDERER:-debug-xml}"
   DEBUG_XML_INCLUDE_SKIPPED: "${DEBUG_XML_INCLUDE_SKIPPED:-false}"
   OUTPUT_TARGET_CHARS: "${OUTPUT_TARGET_CHARS:-25000}"
 
   # Source provider.
-  SOURCE_PROVIDER: "${SOURCE_PROVIDER:-default-http}"
+  SOURCE_PROVIDER: "${SOURCE_PROVIDER:-}"
   FIRECRAWL_BASE_URL: "${FIRECRAWL_BASE_URL:-}"
   FIRECRAWL_API_KEY: "${FIRECRAWL_API_KEY:-}"
+  FIRECRAWL_PARSE_PDF: "${FIRECRAWL_PARSE_PDF:-}"
   DOCLING_BASE_URL: "${DOCLING_BASE_URL:-}"
   DOCLING_API_KEY: "${DOCLING_API_KEY:-}"
 
+  # Content transformers.
+  MDREAM_CLEAN: "${MDREAM_CLEAN:-true}"
+  READABILITY_MIN_CONTENT_CHARS: "${READABILITY_MIN_CONTENT_CHARS:-140}"
+  READABILITY_MIN_SCORE: "${READABILITY_MIN_SCORE:-20}"
+  READABILITY_MAX_ELEMENTS: "${READABILITY_MAX_ELEMENTS:-0}"
+
   # LLM provider.
   LLM_BASE_URL: "${LLM_BASE_URL:-}"
   LLM_API_KEY: "${LLM_API_KEY:-}"

compose.yaml

@@ -15,18 +15,26 @@ x-llmc-environment: &llmc-environment # Bootstrap environment read before YAML l
 
   # Pipeline concurrency and renderer selection.
   SOURCE_CONCURRENCY: "${SOURCE_CONCURRENCY:-1}"
+  PROCESS_CONCURRENCY: "${PROCESS_CONCURRENCY:-5}"
   LLM_CONCURRENCY: "${LLM_CONCURRENCY:-1}"
   DEFAULT_OUTPUT_RENDERER: "${DEFAULT_OUTPUT_RENDERER:-debug-xml}"
   DEBUG_XML_INCLUDE_SKIPPED: "${DEBUG_XML_INCLUDE_SKIPPED:-false}"
   OUTPUT_TARGET_CHARS: "${OUTPUT_TARGET_CHARS:-25000}"
 
   # Source provider.
-  SOURCE_PROVIDER: "${SOURCE_PROVIDER:-default-http}"
+  SOURCE_PROVIDER: "${SOURCE_PROVIDER:-}"
   FIRECRAWL_BASE_URL: "${FIRECRAWL_BASE_URL:-}"
   FIRECRAWL_API_KEY: "${FIRECRAWL_API_KEY:-}"
+  FIRECRAWL_PARSE_PDF: "${FIRECRAWL_PARSE_PDF:-}"
   DOCLING_BASE_URL: "${DOCLING_BASE_URL:-}"
   DOCLING_API_KEY: "${DOCLING_API_KEY:-}"
 
+  # Content transformers.
+  MDREAM_CLEAN: "${MDREAM_CLEAN:-true}"
+  READABILITY_MIN_CONTENT_CHARS: "${READABILITY_MIN_CONTENT_CHARS:-140}"
+  READABILITY_MIN_SCORE: "${READABILITY_MIN_SCORE:-20}"
+  READABILITY_MAX_ELEMENTS: "${READABILITY_MAX_ELEMENTS:-0}"
+
   # LLM provider.
   LLM_BASE_URL: "${LLM_BASE_URL:-}"
   LLM_API_KEY: "${LLM_API_KEY:-}"

config/llm-context-loader.yaml

@@ -26,24 +26,31 @@ outputRenderers:
     type: passthrough
     config: {}
 
-# Source providers turn an input URL into source markdown.
+# Source providers turn an input URL into a pipeline body.
 sourceProviders:
-  default-http:
+  http-default:
     type: http
     config:
       userAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
       maxBytes: 5000000
       titleFromHtml: true
-  default-firecrawl:
+  firecrawl-html:
+    type: firecrawl
+    config:
+      baseUrl: ${FIRECRAWL_BASE_URL:-}
+      apiKey: ${FIRECRAWL_API_KEY:-}
+      output: rawHtml
+      parsePdf: ${FIRECRAWL_PARSE_PDF:-true}
+  firecrawl-markdown:
     type: firecrawl
     config:
       baseUrl: ${FIRECRAWL_BASE_URL:-}
       apiKey: ${FIRECRAWL_API_KEY:-}
       output: markdown
       onlyMainContent: true
       stripBase64Images: true
-      parsePdf: true
-  default-docling:
+      parsePdf: ${FIRECRAWL_PARSE_PDF:-true}
+  docling-ocr:
     type: docling
     config:
       baseUrl: ${DOCLING_BASE_URL:-}
@@ -52,9 +59,26 @@ sourceProviders:
       doOcr: true
       tableMode: accurate
 
+# Content transformers rewrite the body between representations (e.g. HTML to markdown).
+contentTransformers:
+  readability-default:
+    type: readability
+    config:
+      minContentLength: ${READABILITY_MIN_CONTENT_CHARS:-140}
+      minScore: ${READABILITY_MIN_SCORE:-20}
+      maxElements: ${READABILITY_MAX_ELEMENTS:-0}
+  mdream-convert:
+    type: mdream
+    config:
+      clean: ${MDREAM_CLEAN:-true}
+  mdream-aggressive:
+    type: mdream
+    config:
+      minimal: true
+
 # LLM providers run prompt-rendered chat-completions passes.
 llmProviders:
-  default-llm:
+  llm-default:
     type: openai-chat
     config:
       baseUrl: ${LLM_BASE_URL:-}
@@ -77,7 +101,38 @@ pipelines:
         concurrencyGroup: source
         timeoutSeconds: ${LOAD_SOURCE_TIMEOUT_SECONDS:-20}
         config:
-          provider: ${SOURCE_PROVIDER:-default-http}
+          provider: ${SOURCE_PROVIDER:-http-default}
+      - type: truncate
+        name: truncate
+        config:
+          targetChars: ${OUTPUT_TARGET_CHARS:-25000}
+
+  clean-deterministic:
+    outputRenderer: ${DEFAULT_OUTPUT_RENDERER:-debug-xml}
+    limiters:
+      source: ${SOURCE_CONCURRENCY:-1}
+      process: ${PROCESS_CONCURRENCY:-5}
+    steps:
+      - type: load-source
+        name: fetch
+        concurrencyGroup: source
+        timeoutSeconds: ${LOAD_SOURCE_TIMEOUT_SECONDS:-20}
+        config:
+          provider: ${SOURCE_PROVIDER:-firecrawl-html}
+      - type: transform
+        name: clean
+        concurrencyGroup: process
+        config:
+          transformer: readability-default
+          target: text/html
+          emitDiagnostics: true
+      - type: transform
+        name: convert
+        concurrencyGroup: process
+        config:
+          transformer: mdream-convert
+          target: text/markdown
+          emitDiagnostics: true
       - type: truncate
         name: truncate
         config:
@@ -94,7 +149,7 @@ pipelines:
         concurrencyGroup: source
         timeoutSeconds: ${LOAD_SOURCE_TIMEOUT_SECONDS:-20}
         config:
-          provider: ${SOURCE_PROVIDER:-default-firecrawl}
+          provider: ${SOURCE_PROVIDER:-firecrawl-markdown}
       - type: capture-urls
         name: capture_source_urls
         concurrencyGroup: source
@@ -105,7 +160,7 @@ pipelines:
         concurrencyGroup: llm
         timeoutSeconds: ${CLEAN_TIMEOUT_SECONDS:-60}
         config:
-          provider: default-llm
+          provider: llm-default
           minInputChars: ${CLEAN_MIN_INPUT_CHARS:-1000}
           outputReserveRatio: 1.0
           templates:
@@ -123,7 +178,64 @@ pipelines:
         concurrencyGroup: llm
         timeoutSeconds: ${SUMMARIZE_TIMEOUT_SECONDS:-60}
         config:
-          provider: default-llm
+          provider: llm-default
+          minInputChars: ${OUTPUT_TARGET_CHARS:-25000}
+          outputReserveChars: ${OUTPUT_TARGET_CHARS:-25000}
+          templates:
+            system: ../templates/summarize.system.md
+            user: ../templates/summarize.user.md
+            vars:
+              targetChars: ${OUTPUT_TARGET_CHARS:-25000}
+      - type: verify-urls
+        name: verify_after_summarize
+        concurrencyGroup: llm
+        config:
+          artifact: trusted-urls
+          onHallucination: report
+          maxReportedUrls: 50
+      - type: truncate
+        name: truncate
+        config:
+          targetChars: ${OUTPUT_TARGET_CHARS:-25000}
+
+  clean-combined:
+    outputRenderer: ${DEFAULT_OUTPUT_RENDERER:-debug-xml}
+    limiters:
+      source: ${SOURCE_CONCURRENCY:-1}
+      process: ${PROCESS_CONCURRENCY:-5}
+      llm: ${LLM_CONCURRENCY:-1}
+    steps:
+      - type: load-source
+        name: fetch
+        concurrencyGroup: source
+        timeoutSeconds: ${LOAD_SOURCE_TIMEOUT_SECONDS:-20}
+        config:
+          provider: ${SOURCE_PROVIDER:-firecrawl-html}
+      - type: transform
+        name: clean
+        concurrencyGroup: process
+        config:
+          transformer: readability-default
+          target: text/html
+          emitDiagnostics: true
+      - type: transform
+        name: convert
+        concurrencyGroup: process
+        config:
+          transformer: mdream-convert
+          target: text/markdown
+          emitDiagnostics: true
+      - type: capture-urls
+        name: capture_source_urls
+        concurrencyGroup: process
+        config:
+          artifact: trusted-urls
+      - type: llm-pass
+        name: summarize
+        concurrencyGroup: llm
+        timeoutSeconds: ${SUMMARIZE_TIMEOUT_SECONDS:-60}
+        config:
+          provider: llm-default
           minInputChars: ${OUTPUT_TARGET_CHARS:-25000}
           outputReserveChars: ${OUTPUT_TARGET_CHARS:-25000}
           templates: