diff --git a/.github/workflows/check-mkdocs-build.yml b/.github/workflows/check-mkdocs-build.yml index c64d51358..a916b42a5 100644 --- a/.github/workflows/check-mkdocs-build.yml +++ b/.github/workflows/check-mkdocs-build.yml @@ -34,3 +34,28 @@ jobs: - name: Build MkDocs site run: mkdocs build -d site + + - name: Check the AI corpus feed + # Validates the real deploy artifact — the feed hooks/ai_feed.py just wrote via + # the build above — so this gate can't diverge from what actually ships (same + # page set, resolved snippets, and macros env as production). + run: | + python - <<'EOF' + import json, re + path = 'site/ai/llms-full.jsonl' + raw = open(path, encoding='utf-8').read() + rows = [json.loads(l) for l in raw.splitlines() if l.strip()] + pages = {r['page_id'] for r in rows} + assert len(rows) > 1500, f"chunk count collapsed: {len(rows)}" + assert len(pages) > 180, f"page count collapsed: {len(pages)}" + required = {"source", "page_id", "url", "anchor", "page_version_hash", + "page_title", "title", "text"} + missing = required - set(rows[0]) + assert not missing, f"schema regression, missing: {missing}" + assert '--8<--' not in raw, "unresolved --8<-- snippet directives in feed" + # leaked mkdocs template filters (e.g. {{ x | fix_url }}); targets the pipe so + # legit literals like {{ $labels.instance }} don't false-positive. + leaked = re.findall(r'\{\{[^}\n]*\|[^}\n]*\}\}', raw) + assert not leaked, f"unresolved template filters leaked into feed: {leaked[:3]}" + print(f"feed OK: {len(rows)} chunks / {len(pages)} pages, schema intact, no raw snippets or template leaks") + EOF diff --git a/docs/js/docs-assistant.js b/docs/js/docs-assistant.js new file mode 100644 index 000000000..89e9f7e09 --- /dev/null +++ b/docs/js/docs-assistant.js @@ -0,0 +1,227 @@ +/* Polkadot Docs Assistant widget. + Self-hosted chat over the docs corpus — replaces the former kapa.ai embed. + Talks to the assistant API (see the service repo): POST /chat, POST /feedback. + + Configured from the script tag in main.html: + + If data-api-url is empty the widget renders nothing (dark launch). +*/ + +(function () { + 'use strict'; + + if (typeof window === 'undefined') return; + + const script = document.currentScript; + const API_BASE = (script && script.dataset.apiUrl ? script.dataset.apiUrl : '').replace(/\/$/, ''); + if (!API_BASE) return; // no endpoint configured — ship dark + + const HISTORY_TURNS = 6; // question+answer pairs sent back for follow-up rewriting + const history = []; + const sessionId = 'da-' + Date.now().toString(36) + '-' + Math.random().toString(36).slice(2, 8); + + // ---------- tiny markdown renderer (no dependencies) ---------- + function escapeHtml(s) { + // Must escape quotes too: the answer is untrusted LLM output rendered via + // innerHTML, and mdInline interpolates captured text into href="..." — an + // unescaped " would break out of the attribute and inject a handler (XSS). + return s.replace(/&/g, '&').replace(//g, '>') + .replace(/"/g, '"').replace(/'/g, '''); + } + function mdInline(s) { + return s + .replace(/`([^`]+)`/g, '$1') + .replace(/\*\*([^*]+)\*\*/g, '$1') + .replace(/\*(\S([^*\n]*?\S)?)\*/g, '$1') // require non-space at the *edges* so "2 * 3" isn't emphasis + // s is already HTML-escaped, so a real URL has no quote/space; if any escaped + // quote survived in the captured URL, refuse to build a link (render literal). + .replace(/\[([^\]]+)\]\((https?:(?:[^()\s]|\([^()\s]*\))+)\)/g, function (m, text, url) { + return /"|'/.test(url) ? m : '' + text + ''; + }); + } + function md(src) { + let s = escapeHtml(src || ''); + s = s.replace(/\[[a-z0-9\-]+#[a-z0-9\-]+\]/gi, ''); // citation tokens become source chips + const lines = s.split(/\n/); + let out = ''; + let i = 0; + const special = /^\s*([-*]\s+|\d+\.\s+|#{1,6}\s+|```)/; + while (i < lines.length) { + if (/^\s*```/.test(lines[i])) { + const code = []; + i++; + while (i < lines.length && !/^\s*```/.test(lines[i])) { code.push(lines[i]); i++; } + i++; + out += '
' + code.join('\n') + '
'; + } else if (/^\s*[-*]\s+/.test(lines[i])) { + const ul = []; + while (i < lines.length && /^\s*[-*]\s+/.test(lines[i])) { ul.push('
  • ' + mdInline(lines[i].replace(/^\s*[-*]\s+/, '')) + '
    • '); i++; } + out += ''; + } else if (/^\s*\d+\.\s+/.test(lines[i])) { + const ol = []; + while (i < lines.length && /^\s*\d+\.\s+/.test(lines[i])) { ol.push('
  • ' + mdInline(lines[i].replace(/^\s*\d+\.\s+/, '')) + '
    • '); i++; } + out += '
      ' + ol.join('') + '
    '; + } else if (/^\s*#{1,6}\s+/.test(lines[i])) { + out += '

    ' + mdInline(lines[i].replace(/^\s*#{1,6}\s+/, '')) + '

    '; + i++; + } else if (lines[i].trim() === '') { + i++; + } else { + const p = []; + while (i < lines.length && lines[i].trim() !== '' && !special.test(lines[i])) { p.push(lines[i]); i++; } + out += '

    ' + mdInline(p.join(' ').trim()) + '

    '; + } + } + return out; + } + + // ---------- DOM ---------- + function el(tag, className, text) { + const node = document.createElement(tag); + if (className) node.className = className; + if (text) node.textContent = text; + return node; + } + + // s.url comes from the API (untrusted); only allow it as an href if it's http(s), + // matching the scheme guard mdInline applies to links in the answer body. + function httpUrl(u) { + try { return /^https?:$/i.test(new URL(u, location.href).protocol) ? u : null; } + catch (e) { return null; } + } + + const btn = el('button', 'da-launcher'); + btn.type = 'button'; + btn.setAttribute('aria-label', 'Open the docs assistant'); + btn.innerHTML = + '' + + 'Ask the docs'; + + const panel = el('div', 'da-panel'); + panel.setAttribute('role', 'dialog'); + panel.setAttribute('aria-label', 'Polkadot Docs Assistant'); + panel.innerHTML = + '
    ' + + '
    Polkadot Docs AssistantAnswers from the docs, with sources
    ' + + ' ' + + '
    ' + + '
    ' + + '
    ' + + ' ' + + ' ' + + '
    ' + + '
    AI-generated from the documentation — verify against the linked sources and use your best judgement. ' + + 'Please don’t share personal or private information. ' + + 'By submitting a query you agree to these conditions. ' + + 'Need more help? Polkadot Support.
    '; + + const msgs = panel.querySelector('.da-msgs'); + const input = panel.querySelector('input'); + const sendBtn = panel.querySelector('.da-send'); + + function togglePanel(open) { + const isOpen = typeof open === 'boolean' ? open : !panel.classList.contains('da-open'); + panel.classList.toggle('da-open', isOpen); + btn.setAttribute('aria-expanded', String(isOpen)); + if (isOpen) input.focus(); + } + btn.addEventListener('click', function () { togglePanel(); }); + panel.querySelector('.da-close').addEventListener('click', function () { togglePanel(false); }); + document.addEventListener('keydown', function (e) { + if (e.key === 'Escape' && panel.classList.contains('da-open')) togglePanel(false); + }); + + // ---------- messages ---------- + function addUser(text) { + msgs.appendChild(el('div', 'da-msg da-user', text)); + msgs.scrollTop = msgs.scrollHeight; + } + + function addBot(data, question) { + const wrap = el('div', 'da-msg da-bot'); + wrap.innerHTML = md(data.answer); + + if (data.sources && data.sources.length) { + const chips = el('div', 'da-cites'); + const seen = {}; + data.sources.forEach(function (s) { + const pid = (s.ref || '').split('#')[0]; + if (seen[pid]) return; + seen[pid] = true; + const href = s.url && httpUrl(s.url); + let chip; + if (href) { + chip = el('a', 'da-chip', s.page_title || s.title || pid); + chip.href = href; + chip.target = '_blank'; + chip.rel = 'noopener'; + } else { + chip = el('span', 'da-chip', s.page_title || s.title || pid); + } + chip.title = s.ref; + chips.appendChild(chip); + }); + wrap.appendChild(chips); + } + + const fb = el('div', 'da-fb'); + ['up', 'down'].forEach(function (rating) { + const b = el('button', 'da-fb-btn', rating === 'up' ? '👍' : '👎'); + b.type = 'button'; + b.setAttribute('aria-label', rating === 'up' ? 'Good answer' : 'Bad answer'); + b.addEventListener('click', function () { + Array.prototype.forEach.call(fb.children, function (c) { c.classList.remove('da-on'); }); + b.classList.add('da-on'); + fetch(API_BASE + '/feedback', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ question: question, answer: data.answer, rating: rating, session_id: sessionId }) + }).catch(function () {}); + }); + fb.appendChild(b); + }); + wrap.appendChild(fb); + + msgs.appendChild(wrap); + msgs.scrollTop = msgs.scrollHeight; + } + + function send() { + const q = input.value.trim(); + if (!q || sendBtn.disabled) return; + input.value = ''; + sendBtn.disabled = true; + addUser(q); + const thinking = el('div', 'da-msg da-bot da-thinking', '…'); + msgs.appendChild(thinking); + msgs.scrollTop = msgs.scrollHeight; + + fetch(API_BASE + '/chat', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ question: q, history: history.slice(-HISTORY_TURNS * 2), session_id: sessionId }) + }) + .then(function (res) { + if (!res.ok) throw new Error('HTTP ' + res.status); + return res.json(); + }) + .then(function (data) { + thinking.remove(); + addBot(data, q); + history.push({ role: 'user', content: q }, { role: 'assistant', content: data.answer }); + }) + .catch(function () { + thinking.classList.remove('da-thinking'); + thinking.textContent = 'The assistant is temporarily unavailable. Please try again in a moment, or browse the docs directly.'; + }) + .then(function () { + sendBtn.disabled = false; + input.focus(); + }); + } + sendBtn.addEventListener('click', send); + input.addEventListener('keydown', function (e) { if (e.key === 'Enter') send(); }); + + document.body.appendChild(btn); + document.body.appendChild(panel); +})(); diff --git a/generator/.gitignore b/generator/.gitignore new file mode 100644 index 000000000..6b5855994 --- /dev/null +++ b/generator/.gitignore @@ -0,0 +1,4 @@ +# generated artifacts — never commit +*.new.jsonl +llms-full.jsonl +corpus_emb.npy diff --git a/generator/generate_feed.py b/generator/generate_feed.py new file mode 100644 index 000000000..2f12e3c39 --- /dev/null +++ b/generator/generate_feed.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +"""In-house generator for /ai/llms-full.jsonl — a clean-room replacement for the +papermoon ai_docs plugin's chunk feed, with per-chunk `source` and `url` added. + +Reproduces papermoon 0.1.0a18's algorithm: heading-split sections (## .. ###), +slugified/deduped anchors, sha256(cleaned_body) version hash, the heuristic-v1 +token estimator, git last-commit timestamps, and the same page_id derivation. +Snippet resolution uses the real pymdownx.snippets preprocessor (line-range, +block-form, and remote `--8<--` syntaxes), matching the site build. + +`process_page()`/`chunk_row()` are the single source of truth for page → feed +records; hooks/ai_feed.py (the mkdocs integration) imports them so the deployed +feed and the CI-validated feed cannot diverge. + + python generate_feed.py --out llms-full.new.jsonl +""" +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +from datetime import datetime, timezone + +import yaml +from jinja2 import Environment + +MAX_DEPTH = 3 +TOKEN_ESTIMATOR = "heuristic-v1" +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# Kept in sync with mkdocs.yml `exclude_docs` so the standalone/CI feed matches the +# in-build feed (the hook additionally uses mkdocs' own file list — see ai_feed.py). +DEFAULT_EXCLUDE_BASENAMES = {"README.md", "AGENTS.md", "CONTRIBUTING.md", "LICENSE.md"} + +HEADING_RE = re.compile(r"^(#{2,6})\s+(.+?)\s*#*\s*$") +FENCE_RE = re.compile(r"^(\s*)(`{3,}|~{3,})") +HTML_COMMENT_RE = re.compile(r"", re.DOTALL) +# pymdownx attr_list blocks: {#id}, {.class}, {: ...}, {key=val}, {target=_blank}, ... +_ATTR = r'(?:[#.][\w:-]+|[\w-]+=(?:"[^"\n]*"|\'[^\'\n]*\'|[^\s}]+))' +ATTR_BLOCK_RE = re.compile(r'\{:?\s*' + _ATTR + r'(?:\s+' + _ATTR + r')*\s*\}') + +# ----------------------------------------------------------------- preprocessing + +def split_front_matter(text): + m = re.match(r"^---\n(.*?)\n---\n?", text, re.DOTALL) + if m: + try: + fm = yaml.safe_load(m.group(1)) or {} + except Exception: + fm = {} + return fm if isinstance(fm, dict) else {}, text[m.end():] + return {}, text + + +_SNIPPET_PRE = {} + + +def _snippet_preprocessor(base, url_download): + """The real pymdownx.snippets preprocessor, configured to match mkdocs.yml + (base_path, dedent_subsections). Cached; run() resets its own cycle-detection + state each call, so reuse across pages is safe.""" + key = (base, url_download) + if key not in _SNIPPET_PRE: + import markdown + md = markdown.Markdown(extensions=["pymdownx.snippets"], extension_configs={ + "pymdownx.snippets": {"base_path": [base], "url_download": url_download, + "dedent_subsections": True, "check_paths": False}}) + _SNIPPET_PRE[key] = md.preprocessors["snippet"] + return _SNIPPET_PRE[key] + + +# Fetch remote (url_download) snippets by default to match the site build; set +# GEN_SNIPPET_URL_DOWNLOAD=0 (e.g. in CI) to stay fully offline — local snippets still +# resolve and remote ones are dropped rather than fetched. +_URL_DOWNLOAD = os.environ.get("GEN_SNIPPET_URL_DOWNLOAD", "1") != "0" + + +def resolve_snippets(text, base): + # Prefer remote-enabled resolution (matches the build); if a remote snippet is + # dead it raises, so retry local-only — that resolves every local snippet and + # drops the dead remote cleanly, rather than leaving raw --8<-- directives. + last = None + for url_download in ((True, False) if _URL_DOWNLOAD else (False,)): + try: + return "\n".join(_snippet_preprocessor(base, url_download).run(text.split("\n"))) + except ImportError: + # markdown/pymdownx not installed is a setup error, not a dead snippet — + # fail loud instead of silently leaving every --8<-- directive raw. + raise + except Exception as e: + last = e + print(f"[generate_feed] snippet resolution failed ({last}); leaving raw", file=sys.stderr) + return text + + +def resolve_vars(text, variables, env): + try: + return env.from_string(text).render(**variables) + except Exception: + return text # forgiving; parity report will flag any drift + + +def clean_body(raw, variables, env, snippet_base): + """FM split -> snippets -> {{vars}} -> strip HTML comments -> strip {..} attr blocks.""" + fm, body = split_front_matter(raw) + body = resolve_snippets(body, snippet_base) + body = resolve_vars(body, variables, env) + body = HTML_COMMENT_RE.sub("", body) + body = ATTR_BLOCK_RE.sub("", body) + return fm, body + +# ----------------------------------------------------------------- chunking + +def slugify_anchor(text, seen): + v = text.strip().lower() + v = re.sub(r"`+", "", v) + v = re.sub(r"[^\w\s\-]", "", v, flags=re.UNICODE) + v = re.sub(r"\s+", "-", v) + v = re.sub(r"-{2,}", "-", v).strip("-") + if not v: + v = "section" + if v in seen: + seen[v] += 1 + v = f"{v}-{seen[v]}" + else: + seen[v] = 1 + return v + + +def estimate_tokens(text): + return len(re.findall(r"\w+|[^\s\w]", text, flags=re.UNICODE)) + + +def extract_sections(body, max_depth=MAX_DEPTH): + lines = body.splitlines(keepends=True) + starts = [0] + for ln in lines[:-1]: + starts.append(starts[-1] + len(ln)) + heads, fence_char = [], None + for i, ln in enumerate(lines): + fm = FENCE_RE.match(ln) + if fm: + ch = fm.group(2)[0] + if fence_char is None: + fence_char = ch + elif ch == fence_char: + fence_char = None + continue + if fence_char is not None: + continue + hm = HEADING_RE.match(ln) + if hm and 2 <= len(hm.group(1)) <= max_depth: + heads.append((i, len(hm.group(1)), hm.group(2).strip())) + sections, seen = [], {} + for idx, (li, depth, title) in enumerate(heads): + start = starts[li] + end = starts[heads[idx + 1][0]] if idx + 1 < len(heads) else len(body) + sections.append({ + "index": idx, "depth": depth, "title": title, + "anchor": slugify_anchor(title, seen), + "start_char": start, "end_char": end, + "text": body[start:end].strip(), + }) + return sections + +# ----------------------------------------------------------------- page metadata + +def compute_route(docs_dir, path): + rel = os.path.relpath(path, docs_dir).replace(os.sep, "/") + route = re.sub(r"\.(md|mdx)$", "", rel) + if route.endswith("/index"): + route = route[: -len("/index")] + return route + + +_GIT_DATES = None + + +def _git_dates(): + """One `git log` walk over the repo, newest-commit ISO per path — vs one + subprocess per file (~30ms x 250 files). Built once, cached.""" + global _GIT_DATES + if _GIT_DATES is None: + _GIT_DATES = {} + try: + out = subprocess.run( + ["git", "log", "--name-only", "--diff-filter=ACMR", "--pretty=format:%x00%cI"], + capture_output=True, text=True, cwd=REPO_ROOT).stdout + ts = None + for line in out.splitlines(): + if line.startswith("\x00"): + ts = line[1:].strip() + elif line and ts and line not in _GIT_DATES: + _GIT_DATES[line] = ts + except Exception: + pass + return _GIT_DATES + + +def git_last_updated(path): + rel = os.path.relpath(os.path.abspath(path), REPO_ROOT).replace(os.sep, "/") + ts = _git_dates().get(rel) + if ts: + try: + return datetime.fromisoformat(ts.replace("Z", "+00:00")).astimezone(timezone.utc).isoformat() + except Exception: + pass + return datetime.fromtimestamp(os.path.getmtime(path), timezone.utc).isoformat() + + +def iter_markdown(docs_dir): + files = [] + for root, dirs, names in os.walk(docs_dir): + dirs[:] = [d for d in dirs if not d.startswith(".")] + for n in names: + if n.startswith(".") or not n.endswith((".md", ".mdx")): + continue + full = os.path.join(root, n) + if os.path.abspath(full) == os.path.abspath(os.path.join(docs_dir, "index.md")): + continue # root index.md skipped, matching papermoon + files.append(os.path.abspath(full)) + return sorted(files) + +# ----------------------------------------------------------------- shared page -> records + +def chunk_row(source, page_id, page_url, page_title, version_hash, last_updated, sec): + """The single feed-record shape. Used by build() and hooks/ai_feed.py so the + deployed and CI-validated feeds share one schema.""" + return { + "source": source, + "page_id": page_id, + "url": f'{page_url}#{sec["anchor"]}', + "page_title": page_title, + "index": sec["index"], + "depth": sec["depth"], + "title": sec["title"], + "anchor": sec["anchor"], + "start_char": sec["start_char"], + "end_char": sec["end_char"], + "estimated_token_count": estimate_tokens(sec["text"]), + "token_estimator": TOKEN_ESTIMATOR, + "page_version_hash": version_hash, + "last_updated": last_updated, + "text": sec["text"], + } + + +def process_page(path, docs_dir, variables, env, snippet_base, docs_base_url, source): + """Resolve one markdown file to its cleaned body + metadata + feed rows.""" + with open(path, encoding="utf-8") as f: + raw = f.read() + fm, body = clean_body(raw, variables, env, snippet_base) + route = compute_route(docs_dir, path) + page_id = route.replace("/", "-").lower() + title = fm.get("title") or page_id + page_url = f"{docs_base_url.rstrip('/')}/{route}/" + version_hash = "sha256:" + hashlib.sha256(body.encode("utf-8")).hexdigest() + last_updated = git_last_updated(path) + chunks = [chunk_row(source, page_id, page_url, title, version_hash, last_updated, sec) + for sec in extract_sections(body)] + return {"fm": fm, "body": body, "route": route, "page_id": page_id, "title": title, + "url": page_url, "version_hash": version_hash, "last_updated": last_updated, + "chunks": chunks} + +# ----------------------------------------------------------------- main + +def build(args): + with open(args.vars, encoding="utf-8") as f: + variables = yaml.safe_load(f) or {} + env = Environment() + with open(args.config, encoding="utf-8") as f: + cfg = json.load(f) + excl = cfg.get("content", {}).get("exclusions", {}) + skip_base = set(excl.get("skip_basenames", [])) | DEFAULT_EXCLUDE_BASENAMES + skip_paths = excl.get("skip_paths", []) + docs_base_url = cfg.get("project", {}).get("docs_base_url", "https://docs.polkadot.com/") + + rows = [] + for path in iter_markdown(args.docs): + rel = os.path.relpath(path, args.docs) + if os.path.basename(path) in skip_base or any(sp in rel.split(os.sep) for sp in skip_paths): + continue + rows.extend(process_page(path, args.docs, variables, env, args.snippets, + docs_base_url, args.source)["chunks"]) + with open(args.out, "w", encoding="utf-8") as f: + for r in rows: + f.write(json.dumps(r, ensure_ascii=False) + "\n") + pages = len({r["page_id"] for r in rows}) + print(f"wrote {args.out}: {len(rows)} chunks across {pages} pages") + return rows + + +if __name__ == "__main__": + here = os.path.dirname(os.path.abspath(__file__)) + root = os.path.dirname(here) + ap = argparse.ArgumentParser() + ap.add_argument("--docs", default=os.path.join(root, "docs")) + ap.add_argument("--vars", default=os.path.join(root, "docs", "variables.yml")) + ap.add_argument("--snippets", default=os.path.join(root, "docs", ".snippets")) + ap.add_argument("--config", default=os.path.join(root, "llms_config.json")) + ap.add_argument("--source", default="polkadot-docs") + ap.add_argument("--out", default=os.path.join(here, "llms-full.new.jsonl")) + args = ap.parse_args() + build(args) diff --git a/hooks/ai_feed.py b/hooks/ai_feed.py new file mode 100644 index 000000000..9fbf652a9 --- /dev/null +++ b/hooks/ai_feed.py @@ -0,0 +1,242 @@ +"""Build-time AI artifacts — in-house replacement for the papermoon `ai_docs` plugin. + +Produces, during `mkdocs build`: + 1. site/ai/llms-full.jsonl — the chunked corpus feed (with `source` + `url`). + 2. site/.md — one resolved-markdown artifact per page. + 3. Page-action dropdowns — injected into `.page-header-row`. + 4. site/llms.txt — llms.txt index. + 5. The ai-resources.md page — fills in its body (via on_page_markdown). + +All chunking/metadata logic lives in generator/generate_feed.py (single source of +truth via process_page/chunk_row); this hook only handles mkdocs integration. +It honors mkdocs `exclude_docs` by working off the build's own file list. +""" +import html +import json +import logging +import os +import re +import sys +from urllib.parse import quote, urlparse + +_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(_ROOT, "generator")) + +import yaml # noqa: E402 +from jinja2 import Environment # noqa: E402 + +from generate_feed import estimate_tokens, process_page # noqa: E402 + +log = logging.getLogger("mkdocs") +SOURCE_TAG = "polkadot-docs" +_CFG = None +_INCLUDED = None # {src_uri: abs_path} for pages mkdocs actually builds (post exclude_docs) + + +def _load_cfg(config): + global _CFG + if _CFG is not None: + return _CFG + with open(os.path.join(_ROOT, "llms_config.json"), encoding="utf-8") as f: + llms = json.load(f) + excl = llms.get("content", {}).get("exclusions", {}) + _CFG = { + "docs_base_url": llms.get("project", {}).get("docs_base_url", config["site_url"]).rstrip("/"), + "skip_basenames": set(excl.get("skip_basenames", [])), + "skip_paths": excl.get("skip_paths", []), + "variables": yaml.safe_load(open(os.path.join(config["docs_dir"], "variables.yml"), encoding="utf-8")) or {}, + "env": Environment(), + "snippets": os.path.join(config["docs_dir"], ".snippets"), + "site_path": urlparse(config["site_url"] or "/").path.rstrip("/"), + } + return _CFG + + +def _feed_excluded(src_uri, cfg, meta=None): + """True = keep this page OUT of the aggregate feed/llms.txt. (Artifacts and the + dropdown are still emitted; only the AI corpus is opt-out.)""" + if src_uri == "index.md" or os.path.basename(src_uri) in cfg["skip_basenames"]: + return True + if any(part in cfg["skip_paths"] for part in src_uri.split("/")): + return True + if meta and meta.get("hide_ai_actions"): # full opt-out: no dropdown AND no feed text + return True + return False + + +def on_files(files, config, **kwargs): + # Authoritative page set AFTER mkdocs applies exclude_docs — so README.md, + # AGENTS.md, etc. never get an artifact, feed entry, or 404'd llms.txt link. + global _INCLUDED + _INCLUDED = {f.src_uri: f.abs_src_path for f in files.documentation_pages()} + return files + + +# --------------------------------------------------------------- ai-resources page + +AI_RESOURCES_MD = """ +Machine-readable versions of this documentation, for LLMs and AI tools. + +## Full corpus feed + +- [`/ai/llms-full.jsonl`](/ai/llms-full.jsonl): every documentation page, pre-chunked + by section. One JSON object per line with `source`, `page_id`, `url`, heading + `anchor`, `page_version_hash`, and the section `text`. Regenerated on every docs + deploy — it is a snapshot of the current docs, not a log. +- [`/llms.txt`](/llms.txt): index file following the [llms.txt convention](https://llmstxt.org/). + +## Per-page Markdown + +Every page is also published as resolved Markdown next to its HTML — append `.md` +to a page's path (for example `/apps/build.md`), or use the "Markdown for LLMs" +menu at the top of any page to copy, view, or download it. +""" + + +def on_page_markdown(markdown, page, config, files, **kwargs): + if page.file.src_uri == "ai-resources.md": + return AI_RESOURCES_MD + return markdown + + +# --------------------------------------------------------------- dropdown injection + +_MD_ICON = ( + '' +) +_CHEVRON = ( + '' +) + + +def _dropdown_html(md_href, page_url_abs, filename): + prompt = quote(f"Read {page_url_abs} so I can ask questions about it.") + esc = html.escape(md_href, quote=True) + return f""" +
    + + +
    """ + + +# A
    carrying `page-header-row` as a whole class token (not a substring like +# page-header-row-wrapper, and only inside class="…", never a data-* value). Tolerant +# of a second class / data-variant / attribute order. (Template-level injection in +# page-badges.html would be even cleaner; this keeps the theme untouched.) +_ROW_RE = re.compile( + r']*\bclass\s*=\s*"[^"]*(?]*>', re.I) +_DIV_TAG_RE = re.compile(r'', re.I) + + +def _inject_dropdown(output, widget): + parts, pos = [], 0 + for m in _ROW_RE.finditer(output): + # Insert before the row's DEPTH-MATCHED closing
    (robust to nested + # divs) so the widget renders after the badges, on the right, as before. + depth, close = 1, None + for t in _DIV_TAG_RE.finditer(output, m.end()): + depth += 1 if t.group(0)[1] != "/" else -1 + if depth == 0: + close = t.start() + break + if close is None: + continue + parts.append(output[pos:close]) + parts.append(widget) + pos = close + if not parts: + return output + parts.append(output[pos:]) + return "".join(parts) + + +def on_post_page(output, page, config, **kwargs): + cfg = _load_cfg(config) + if _feed_excluded(page.file.src_uri, cfg, page.meta): # hidden pages get no dropdown + return output + route = page.url.rstrip("/") + if not route: + return output + md_href = f'{cfg["site_path"]}/{route}.md' + page_abs = f'{cfg["docs_base_url"]}/{route}/' + widget = _dropdown_html(md_href, page_abs, os.path.basename(md_href)) + return _inject_dropdown(output, widget) + + +# --------------------------------------------------------------- artifacts on build + +def _write_md(site_dir, route, page): + header = { + "title": page["title"], + "description": page["fm"].get("description"), + "categories": page["fm"].get("categories"), + "url": page["url"], + "word_count": len(re.findall(r"\b\w+\b", page["body"], re.UNICODE)), + "token_estimate": estimate_tokens(page["body"]), + "version_hash": page["version_hash"], + "last_updated": page["last_updated"], + } + out_md = os.path.join(site_dir, *route.split("/")) + ".md" + os.makedirs(os.path.dirname(out_md), exist_ok=True) + with open(out_md, "w", encoding="utf-8") as f: + f.write("---\n") + for k, v in header.items(): + if v not in (None, "", []): + f.write(f"{k}: {json.dumps(v, ensure_ascii=False) if isinstance(v, list) else v}\n") + f.write("---\n\n") + f.write(page["body"].strip() + "\n") + + +def on_post_build(config, **kwargs): + cfg = _load_cfg(config) + docs_dir = os.path.abspath(config["docs_dir"]) + site_dir = config["site_dir"] + included = _INCLUDED or {} + rows, pages = [], [] + + for src_uri, abs_path in sorted(included.items()): + if src_uri == "index.md": + continue # homepage: no .md artifact / feed entry (matches the generator) + page = process_page(abs_path, docs_dir, cfg["variables"], cfg["env"], + cfg["snippets"], cfg["docs_base_url"], SOURCE_TAG) + _write_md(site_dir, page["route"], page) + if not _feed_excluded(src_uri, cfg, page["fm"]): + pages.append((page["title"], page["url"])) + rows.extend(page["chunks"]) + + ai_dir = os.path.join(site_dir, "ai") + os.makedirs(ai_dir, exist_ok=True) + with open(os.path.join(ai_dir, "llms-full.jsonl"), "w", encoding="utf-8") as f: + for r in rows: + f.write(json.dumps(r, ensure_ascii=False) + "\n") + + with open(os.path.join(site_dir, "llms.txt"), "w", encoding="utf-8") as f: + f.write("# Polkadot Developer Docs\n\n") + f.write("> Developer documentation for Polkadot. Full pre-chunked corpus: " + f'{cfg["docs_base_url"]}/ai/llms-full.jsonl — per-page resolved Markdown: ' + "append .md to any page URL.\n\n## Pages\n\n") + for title, url in pages: + f.write(f"- [{title}]({url})\n") + + log.info("[ai_feed] wrote %d chunks across %d pages -> ai/llms-full.jsonl, " + "per-page .md, llms.txt", len(rows), len(pages)) diff --git a/material-overrides/assets/stylesheets/docs-assistant.css b/material-overrides/assets/stylesheets/docs-assistant.css new file mode 100644 index 000000000..83cd96491 --- /dev/null +++ b/material-overrides/assets/stylesheets/docs-assistant.css @@ -0,0 +1,217 @@ +/* Polkadot Docs Assistant widget. + Uses the site's theme variables (see polkadot.css) so the widget follows + light/dark mode automatically. Occupies the slot the kapa.ai button used. */ + +.da-launcher { + position: fixed; + right: 1.2rem; + bottom: 10vh; + z-index: 50; + display: flex; + flex-direction: column; + align-items: center; + gap: 0.3rem; + padding: 0.7rem 0.9rem; + background: var(--md-default-bg-color); + color: var(--color-text-primary); + border: 1px solid var(--color-border-strong); + border-radius: 0.8rem; + font: inherit; + font-size: 0.7rem; + cursor: pointer; +} + +.da-launcher:hover { + background: var(--color-surface); +} + +.da-panel { + position: fixed; + right: 1.2rem; + bottom: calc(10vh + 5.2rem); + z-index: 51; + width: 21rem; + max-width: calc(100vw - 2.4rem); + height: 29rem; + max-height: calc(100vh - 12rem); + display: none; + flex-direction: column; + overflow: hidden; + background: var(--md-default-bg-color); + border: 1px solid var(--color-border-strong); + border-radius: 0.8rem; + box-shadow: 0 0.6rem 2rem rgba(0, 0, 0, 0.25); + font-size: 0.7rem; +} + +.da-panel.da-open { + display: flex; +} + +.da-head { + display: flex; + align-items: center; + justify-content: space-between; + padding: 0.7rem 0.8rem; + border-bottom: 1px solid var(--color-border); + color: var(--color-text-primary); +} + +.da-head small { + display: block; + color: var(--color-text-secondary); + font-size: 0.55rem; +} + +.da-close { + background: none; + border: none; + color: var(--color-text-secondary); + font-size: 0.8rem; + cursor: pointer; +} + +.da-msgs { + flex: 1; + overflow-y: auto; + padding: 0.7rem; + display: flex; + flex-direction: column; + gap: 0.6rem; +} + +.da-msg { + max-width: 90%; + padding: 0.5rem 0.65rem; + border-radius: 0.6rem; + line-height: 1.5; + overflow-wrap: break-word; +} + +.da-user { + align-self: flex-end; + background: var(--accent-default); + color: var(--white); + border-bottom-right-radius: 0.15rem; +} + +.da-bot { + align-self: flex-start; + background: var(--color-surface-secondary); + color: var(--color-text-primary); + border-bottom-left-radius: 0.15rem; +} + +.da-bot p { margin: 0 0 0.45rem; } +.da-bot p:last-child { margin-bottom: 0; } +.da-bot ul, .da-bot ol { margin: 0.3rem 0; padding-left: 1.1rem; } +.da-bot h4 { margin: 0.5rem 0 0.2rem; font-size: 0.72rem; } + +.da-bot code { + background: var(--md-default-bg-color); + border: 1px solid var(--color-border); + border-radius: 0.2rem; + padding: 0 0.2rem; + font-size: 0.9em; +} + +.da-bot pre { + background: var(--color-surface); + border: 1px solid var(--color-border); + border-radius: 0.4rem; + padding: 0.5rem; + overflow-x: auto; +} + +.da-bot pre code { background: none; border: none; padding: 0; } + +.da-cites { + display: flex; + flex-wrap: wrap; + gap: 0.25rem; + margin-top: 0.45rem; +} + +.da-chip { + font-size: 0.55rem; + padding: 0.1rem 0.45rem; + border: 1px solid var(--color-border-strong); + border-radius: 1rem; + background: var(--md-default-bg-color); + color: var(--color-text-secondary); + text-decoration: none; +} + +a.da-chip:hover { + border-color: var(--accent-default); + color: var(--accent-default); +} + +.da-fb { + display: flex; + gap: 0.25rem; + margin-top: 0.45rem; +} + +.da-fb-btn { + background: none; + border: 1px solid var(--color-border); + border-radius: 0.3rem; + padding: 0.05rem 0.35rem; + cursor: pointer; + font-size: 0.65rem; +} + +.da-fb-btn.da-on { + border-color: var(--accent-default); +} + +.da-thinking { + color: var(--color-text-secondary); +} + +.da-input { + display: flex; + gap: 0.4rem; + padding: 0.55rem; + border-top: 1px solid var(--color-border); +} + +.da-input input { + flex: 1; + padding: 0.45rem 0.6rem; + background: var(--color-surface); + color: var(--color-text-primary); + border: 1px solid var(--color-border); + border-radius: 0.5rem; + font: inherit; + font-size: 0.7rem; +} + +.da-send { + padding: 0 0.7rem; + background: var(--accent-default); + color: var(--white); + border: none; + border-radius: 0.5rem; + font-weight: 600; + cursor: pointer; +} + +.da-send:disabled { + opacity: 0.5; + cursor: default; +} + +.da-note { + padding: 0 0.7rem 0.55rem; + font-size: 0.5rem; + line-height: 1.4; + color: var(--color-text-secondary); + text-align: center; +} + +.da-note a { + color: inherit; + text-decoration: underline; +} diff --git a/material-overrides/main.html b/material-overrides/main.html index 46d0d11c0..c04bf4600 100644 --- a/material-overrides/main.html +++ b/material-overrides/main.html @@ -29,6 +29,17 @@ it — Jinja still processes the block tag.) #} {% block announce %}{% endblock %} +{# Docs Assistant — self-hosted chat over the docs corpus. The API endpoint comes + from extra.docs_assistant_api_url (env DOCS_ASSISTANT_API_URL); when unset, the + widget renders nothing. #} +{% block libs %} + {{ super() }} + + +{% endblock %} {% block site_nav %} {#- Navigation (left menu) -#} diff --git a/material-overrides/partials/page-badges.html b/material-overrides/partials/page-badges.html index 906c5e021..e008e0ba0 100644 --- a/material-overrides/partials/page-badges.html +++ b/material-overrides/partials/page-badges.html @@ -3,13 +3,14 @@ render_badges(badges, variant=None) Renders a single .page-header-row div containing tutorial difficulty and - CI test-workflow badges sourced from page frontmatter. AI dropdowns are also - injected into this div by the ai_docs plugin. + CI test-workflow badges sourced from page frontmatter. The "Markdown for LLMs" + dropdown is injected into this div at build time by hooks/ai_feed.py, matched + on the .page-header-row class. badges — a page_badges dict from frontmatter (may be None/empty) variant — optional data-variant string; when set, scopes the row with - data-variant so the ai_docs plugin can inject the per-page AI - widget into the correct slot on toggle pages. + data-variant so toggle-pages.js/css can target the right row on + toggle pages. -#} {# Signal-cellular icons for tutorial difficulty level #} diff --git a/mkdocs.yml b/mkdocs.yml index 147e69360..ae4fcc57b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -117,6 +117,7 @@ markdown_extensions: # Hooks hooks: + - hooks/ai_feed.py # In-house AI artifacts: llms-full.jsonl, per-page .md, page-action dropdowns (replaces the papermoon ai_docs plugin) - hooks/auto_index_table.py - hooks/footer_nav.py - hooks/glossary_abbreviations.py @@ -125,11 +126,6 @@ hooks: # Plugins plugins: - search - - ai_docs: - llms_config: llms_config.json - enabled: !ENV [ENABLED_LLMS_PLUGINS, True] - ai_page_actions_anchor: page-header-row - ai_page_actions_style: dropdown - awesome-nav - git-revision-date-localized: type: date @@ -180,6 +176,8 @@ plugins: # Extra configuration extra: + # Docs Assistant chat API endpoint. Empty = widget disabled (ships dark). + docs_assistant_api_url: !ENV [DOCS_ASSISTANT_API_URL, ''] glossary_tooltips: exclude_terms: - Polkadot