feat: Google OKF-conformant wiki frontmatter (#102)

* feat(compiler): capitalize entity type, rename brief→description

Write entity frontmatter with type capitalized (e.g. "Organization"),
use the key `description` instead of `brief` for the one-liner field.
Readers normalize type to lowercase and prefer `description`, falling
back to legacy `brief` for pre-migration pages.

LLM prompts and JSON extraction updated to use the `description` key.

* feat(compiler): add type: Concept and rename brief→description on concept pages

* feat(compiler): add type+description to short-doc summary frontmatter

* feat(tree_renderer): add type+description to PageIndex summary frontmatter

* feat(compiler): backfill type+description on long-doc summaries during recompile

* feat(lint): flag knowledge pages missing OKF type/description

* docs(schema): document type/description frontmatter in AGENTS_MD

* fix(compiler): strip legacy brief: on entity update (parity with concept pages)

* fix(review): harden frontmatter writes (title-case type, quote full_text, backfill order + dirty-check, lint isinstance)

* refactor(compiler): use shared frontmatter module; fix --- boundary + concept malformed fallback

- Import openkb.frontmatter; replace local _yaml_kv_line / _yaml_list_line /
  _parse_yaml_list_value / _set_fm_line bodies with thin aliases to the shared
  module exports (call sites unchanged).
- Replace all ad-hoc text.find("---", 3) splits with frontmatter.split():
  _write_summary, _write_concept, _write_entity, _read_concept_briefs,
  _read_entity_briefs, compile_long_doc backfill, _gen_update,
  _gen_entity_update, summary-rewrite strip, _prepend_source_to_frontmatter,
  _remove_source_from_frontmatter, scan_affected_pages.
- Add module-level _resolve_description(fm) helper; use it in both reader
  functions to consolidate description/brief fallback logic.
- _read_concept_briefs and _read_entity_briefs now use frontmatter.parse()
  instead of manual yaml.safe_load.
- Replace re.sub(r"^brief:.*\n?", ...) with frontmatter.drop_line() in both
  _write_concept and _write_entity update paths.
- Add malformed-frontmatter fallback to _write_concept update path: when
  frontmatter.split returns None, rebuild valid block (type: Concept,
  sources, description) rather than writing a bare body.
- Remove now-unused direct yaml import.
- Add 3 regression tests: concept round-trip with "---" in brief, entity
  round-trip with "---" in brief, concept update on malformed frontmatter.

* refactor(tree_renderer, lint): use shared frontmatter module

* perf(lint): read wiki pages once and share across frontmatter checks

Adds _load_wiki_pages() to walk wiki/*.md (excluding reports/, sources/,
and _EXCLUDED_FILES) and return a {path: text} dict.  Both
find_invalid_frontmatter and find_missing_okf_fields accept an optional
pages= kwarg; run_structural_lint builds it once and passes it to both,
reducing 2N file reads to N.  Standalone callers that pass only wiki
continue to work unchanged.  Also switches find_invalid_frontmatter to
use frontmatter.split() for line-anchored delimiter detection.

Author: KylinMountain

openkb/agent/compiler.py

@@ -0,0 +1,110 @@
+"""Shared helpers for the YAML frontmatter blocks of OpenKB wiki pages.
+
+Single source of truth for building, splitting, parsing, and mutating the
+``---`` frontmatter used by summaries / concepts / entities. The closing
+delimiter is always matched at the start of a line (``\\n---``), so a ``---``
+that appears inside a quoted value never truncates the block — the failure
+mode that ad-hoc ``text.find("---", 3)`` parsing was prone to.
+"""
+from __future__ import annotations
+
+import json
+import re
+
+import yaml
+
+
+def kv_line(key: str, value: str) -> str:
+    """Render ``key: "value"`` with the value JSON-quoted (always single-line).
+
+    JSON strings are a strict subset of YAML: always single-line, always
+    correctly escaped (newlines, quotes, colons, control chars), and never
+    auto-promoted to a block scalar.
+    """
+    return f"{key}: {json.dumps(value, ensure_ascii=False)}"
+
+
+def list_line(key: str, items) -> str:
+    """Render ``key: ["a", "b"]`` as JSON-style YAML (always single-line)."""
+    return f"{key}: {json.dumps(list(items), ensure_ascii=False)}"
+
+
+def block(lines: list[str]) -> str:
+    """Assemble a complete frontmatter block (with delimiters + trailing blank)."""
+    return "---\n" + "\n".join(lines) + "\n---\n\n"
+
+
+def parse_list_value(line: str) -> list[str] | None:
+    """Parse the right-hand side of ``key: [...]`` into a list of strings.
+
+    Returns ``None`` when the value cannot be interpreted as a list — callers
+    treat that as "leave the frontmatter alone".
+    """
+    colon = line.find(":")
+    if colon == -1:
+        return None
+    try:
+        parsed = yaml.safe_load(line[colon + 1:])
+    except yaml.YAMLError:
+        return None
+    if not isinstance(parsed, list):
+        return None
+    return [str(x) for x in parsed]
+
+
+def split(text: str) -> tuple[str, str] | None:
+    """Split ``text`` into ``(frontmatter_block, body)``.
+
+    ``frontmatter_block`` includes both ``---`` delimiters and the newline that
+    ends the closing delimiter line; ``body`` is everything after, so
+    ``frontmatter_block + body == text`` exactly (lossless).
+
+    Returns ``None`` when ``text`` has no well-formed frontmatter: no leading
+    ``---`` or no line-anchored closing ``---``. Because the closing delimiter
+    must start a line (``\\n---``), a ``---`` inside a quoted value is ignored.
+    """
+    if not text.startswith("---"):
+        return None
+    nl = text.find("\n---", 3)
+    if nl == -1:
+        return None
+    after = text.find("\n", nl + 1)  # newline ending the closing '---' line
+    if after == -1:
+        return text, ""
+    return text[:after + 1], text[after + 1:]
+
+
+def parse(text: str) -> dict:
+    """Return the frontmatter as a dict (``{}`` when absent or malformed)."""
+    parts = split(text)
+    if parts is None:
+        return {}
+    fm_block = parts[0]
+    inner = fm_block[3:]                 # drop opening '---'
+    close = inner.rfind("\n---")        # drop closing '---' line
+    if close != -1:
+        inner = inner[:close]
+    try:
+        data = yaml.safe_load(inner)
+    except yaml.YAMLError:
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+def set_line(fm_block: str, key: str, value: str) -> str:
+    """Set or insert a single scalar ``key:`` line in a frontmatter block.
+
+    Replaces an existing line for ``key``; otherwise inserts it right after the
+    opening ``---``. A lambda replacement is used so values containing regex
+    backrefs (``\\1``, ``\\g<…>``) are inserted literally.
+    """
+    line = kv_line(key, value)
+    if re.search(rf"^{re.escape(key)}:", fm_block, flags=re.MULTILINE):
+        return re.sub(rf"^{re.escape(key)}:.*", lambda _m: line, fm_block,
+                      count=1, flags=re.MULTILINE)
+    return fm_block.replace("---\n", f"---\n{line}\n", 1)
+
+
+def drop_line(fm_block: str, key: str) -> str:
+    """Remove any ``key:`` line from a frontmatter block (no-op if absent)."""
+    return re.sub(rf"^{re.escape(key)}:.*\n?", "", fm_block, flags=re.MULTILINE)

openkb/frontmatter.py

@@ -174,7 +174,7 @@ def index_long_document(
     # Write wiki/summaries/ (no images, just summaries)
     summaries_dir = kb_dir / "wiki" / "summaries"
     summaries_dir.mkdir(parents=True, exist_ok=True)
-    summary_md = render_summary_md(tree, source_name, doc_id)
+    summary_md = render_summary_md(tree, source_name, doc_id, description=description)
     (summaries_dir / f"{source_name}.md").write_text(summary_md, encoding="utf-8")
 
     return IndexResult(doc_id=doc_id, description=description, tree=tree)

openkb/indexer.py

@@ -15,6 +15,7 @@
 
 import yaml
 
+from openkb import frontmatter
 from openkb.schema import PAGE_CONTENT_DIRS
 
 # Matches [[wikilink]] or [[subdir/link]]
@@ -449,42 +450,111 @@ def check_index_sync(wiki: Path) -> list[str]:
     return sorted(issues)
 
 
-def find_invalid_frontmatter(wiki: Path) -> list[str]:
+def _load_wiki_pages(wiki: Path) -> dict[Path, str]:
+    """Read all wiki .md files once and return a ``{path: text}`` mapping.
+
+    Applies the same scope as :func:`find_invalid_frontmatter`: walks the
+    whole wiki, skips names in :data:`_EXCLUDED_FILES`, and skips any path
+    whose first relative-to-wiki directory part is ``"reports"`` or
+    ``"sources"``.  The result is the superset that both frontmatter checks
+    iterate; :func:`find_missing_okf_fields` additionally filters down to
+    :data:`PAGE_CONTENT_DIRS <openkb.schema.PAGE_CONTENT_DIRS>` paths.
+    """
+    pages: dict[Path, str] = {}
+    for path in wiki.rglob("*.md"):
+        if path.name in _EXCLUDED_FILES:
+            continue
+        rel_parts = path.relative_to(wiki).parts
+        if rel_parts and rel_parts[0] in ("reports", "sources"):
+            continue
+        pages[path] = _read_md(path)
+    return pages
+
+
+def find_invalid_frontmatter(
+    wiki: Path,
+    pages: dict[Path, str] | None = None,
+) -> list[str]:
     """Return wiki pages whose YAML frontmatter fails ``yaml.safe_load``.
 
     Catches the silent-write class of bug where an LLM-authored field
     (e.g. ``brief:``) ships unquoted and turns a colon-bearing value
     into invalid YAML that OpenKB itself reads with string slicing but
     external YAML-aware tools (VS Code, Obsidian, doc generators) reject.
+
+    Args:
+        wiki: Path to the wiki root directory.
+        pages: Optional pre-loaded ``{path: text}`` mapping from
+            :func:`_load_wiki_pages`.  When ``None`` (the default), the
+            mapping is built internally so existing callers that pass only
+            ``wiki`` keep working unchanged.
     """
     issues: list[str] = []
     if not wiki.exists():
         return issues
-    for path in sorted(wiki.rglob("*.md")):
-        if path.name in _EXCLUDED_FILES:
-            continue
-        # Skip reports/ and sources/ — auto-generated / user-uploaded
-        # content, not wiki pages we manage. Matches the convention in
-        # find_broken_links / find_orphans.
-        rel_parts = path.relative_to(wiki).parts
-        if rel_parts and rel_parts[0] in ("reports", "sources"):
-            continue
-        text = _read_md(path)
-        if not text.startswith("---"):
-            continue
-        end = text.find("\n---", 3)
-        if end == -1:
+    if pages is None:
+        pages = _load_wiki_pages(wiki)
+    for path, text in sorted(pages.items()):
+        parts = frontmatter.split(text)
+        if parts is None:
             continue
-        fm = text[3:end].strip("\n")
+        fm_block = parts[0]
+        # Extract the raw YAML between the two ``---`` delimiters so that
+        # yaml.safe_load can surface the exact error message.  The closing
+        # delimiter is line-anchored (frontmatter.split guarantees this),
+        # so we strip the opening ``---\n`` and everything from the final
+        # ``\n---`` onward.
+        inner = fm_block[4:]          # drop "---\n"
+        close = inner.rfind("\n---")
+        if close != -1:
+            inner = inner[:close]
         try:
-            yaml.safe_load(fm)
+            yaml.safe_load(inner)
         except yaml.YAMLError as exc:
             rel = path.relative_to(wiki)
             msg = str(exc).splitlines()[0]
             issues.append(f"{rel}: {msg}")
     return issues
 
 
+def find_missing_okf_fields(
+    wiki: Path,
+    pages: dict[Path, str] | None = None,
+) -> list[str]:
+    """Return knowledge pages missing a non-empty ``type`` or ``description``.
+
+    OKF v0.1 requires every non-reserved knowledge page to carry a non-empty
+    ``type``; ``description`` is OpenKB's required one-liner. Only summaries/,
+    concepts/, entities/ are checked — index.md, log.md and sources/ are exempt.
+
+    Args:
+        wiki: Path to the wiki root directory.
+        pages: Optional pre-loaded ``{path: text}`` mapping from
+            :func:`_load_wiki_pages`.  When ``None`` (the default), the
+            mapping is built internally so existing callers that pass only
+            ``wiki`` keep working unchanged.
+    """
+    issues: list[str] = []
+    if not wiki.exists():
+        return issues
+    if pages is None:
+        pages = _load_wiki_pages(wiki)
+    for path, text in sorted(pages.items()):
+        # find_missing_okf_fields covers only PAGE_CONTENT_DIRS subsets.
+        rel_parts = path.relative_to(wiki).parts
+        if not rel_parts or rel_parts[0] not in PAGE_CONTENT_DIRS:
+            continue
+        fm = frontmatter.parse(text)
+        rel = path.relative_to(wiki)
+        type_val = fm.get("type")
+        if not (isinstance(type_val, str) and type_val.strip()):
+            issues.append(f"{rel}: missing non-empty 'type'")
+        desc_val = fm.get("description")
+        if not (isinstance(desc_val, str) and desc_val.strip()):
+            issues.append(f"{rel}: missing non-empty 'description'")
+    return issues
+
+
 def run_structural_lint(kb_dir: Path) -> str:
     """Run all structural lint checks and return a formatted Markdown report.
 
@@ -497,11 +567,16 @@ def run_structural_lint(kb_dir: Path) -> str:
     wiki = kb_dir / "wiki"
     raw = kb_dir / "raw"
 
+    # Load all wiki pages once and share across both frontmatter checks
+    # (avoids 2N disk reads when the wiki is large).
+    wiki_pages = _load_wiki_pages(wiki) if wiki.exists() else {}
+
     broken = find_broken_links(wiki)
     orphans = find_orphans(wiki)
     missing = find_missing_entries(raw, wiki, kb_dir=kb_dir)
     sync_issues = check_index_sync(wiki)
-    bad_frontmatter = find_invalid_frontmatter(wiki)
+    bad_frontmatter = find_invalid_frontmatter(wiki, pages=wiki_pages)
+    missing_okf = find_missing_okf_fields(wiki, pages=wiki_pages)
 
     lines = ["## Structural Lint Report\n"]
 
@@ -548,5 +623,14 @@ def run_structural_lint(kb_dir: Path) -> str:
             lines.append(f"- {issue}")
     else:
         lines.append("All frontmatter parses as valid YAML.")
+    lines.append("")
+
+    # OKF conformance
+    lines.append(f"### OKF Conformance ({len(missing_okf)})")
+    if missing_okf:
+        for issue in missing_okf:
+            lines.append(f"- {issue}")
+    else:
+        lines.append("All knowledge pages carry required OKF fields.")
 
     return "\n".join(lines)

openkb/lint.py

@@ -48,7 +48,13 @@
 - Use [[wikilink]] to link other wiki pages (e.g., [[concepts/attention]])
 - Standard Markdown heading hierarchy
 - Keep each page focused on a single topic
-- Do not include YAML frontmatter (---) in generated content; it is managed by code
+
+## Frontmatter (managed by code — do NOT emit it in generated content)
+- Every summary/concept/entity page carries a non-empty `type:` — `Summary`,
+  `Concept`, or a capitalized entity subtype (e.g. `Organization`). This is the
+  one field OKF requires; consumers use it for routing/filtering/presentation.
+- `description:` — a single-sentence one-liner (the field formerly named `brief`).
+- Do not include YAML frontmatter (---) in generated content; it is managed by code.
 """
 
 # Backward compat alias

openkb/schema.py

@@ -1,16 +1,17 @@
 """Markdown renderers for PageIndex tree structures."""
 from __future__ import annotations
 
+from openkb import frontmatter
 
-def _yaml_frontmatter(source_name: str, doc_id: str) -> str:
-    """Return a YAML frontmatter block for a PageIndex wiki page."""
-    return (
-        "---\n"
-        "doc_type: pageindex\n"
-        f"full_text: sources/{source_name}.json\n"
-        "---\n"
-    )
 
+def _yaml_frontmatter(source_name: str, doc_id: str, description: str = "") -> str:
+    """Return a YAML frontmatter block for a PageIndex wiki page."""
+    lines = [frontmatter.kv_line("type", "Summary")]
+    if description:
+        lines.append(frontmatter.kv_line("description", description))
+    lines.append("doc_type: pageindex")
+    lines.append(frontmatter.kv_line("full_text", f"sources/{source_name}.json"))
+    return "---\n" + "\n".join(lines) + "\n---\n"
 
 
 def _render_nodes_summary(nodes: list[dict], depth: int) -> str:
@@ -24,7 +25,7 @@ def _render_nodes_summary(nodes: list[dict], depth: int) -> str:
         summary = node.get("summary", "")
         children = node.get("nodes", [])
 
-        lines.append(f"{heading_prefix} {title} (pages {start}\u2013{end})\n")
+        lines.append(f"{heading_prefix} {title} (pages {start}–{end})\n")
         if summary:
             lines.append(f"Summary: {summary}\n")
         if children:
@@ -33,13 +34,15 @@ def _render_nodes_summary(nodes: list[dict], depth: int) -> str:
     return "\n".join(lines)
 
 
-
-def render_summary_md(tree: dict, source_name: str, doc_id: str) -> str:
+def render_summary_md(tree: dict, source_name: str, doc_id: str,
+                      description: str = "") -> str:
     """Render the summary Markdown page for a PageIndex tree.
 
     Renders each node as a heading with page range and its summary text.
+    Includes a YAML frontmatter block with ``type: "Summary"`` and an
+    optional ``description`` field.
     """
-    frontmatter = _yaml_frontmatter(source_name, doc_id)
+    frontmatter = _yaml_frontmatter(source_name, doc_id, description)
     structure = tree.get("structure", [])
     body = _render_nodes_summary(structure, depth=1)
     return frontmatter + "\n" + body