diff --git a/skills/factlog/SKILL.md b/skills/factlog/SKILL.md index 3c86dc7d..daa2081f 100644 --- a/skills/factlog/SKILL.md +++ b/skills/factlog/SKILL.md @@ -803,6 +803,11 @@ and carry no extraction confidence by construction. The verdict stays binary in every case. For an out-of-band trace (any fact, full or partial triple, all statuses), use `factlog provenance [relation] [object]`. +The renderer shows at most 20 answer rows by default and explicitly reports any +omitted rows as `… N more rows (full output: --all)` while keeping `rows: N` as +the real total. When the full audit trail is needed, rerun the same command with +`--all`; do not ask the model to select or summarize omitted rows. + A verified-negative relation query may additionally carry an informational `note: ... (possible predicate mismatch): ...` line (#189). It appears **only** when the queried subject is an accepted entity that has **no** fact under the @@ -828,6 +833,10 @@ unverified excerpts cite only source text, never `facts/accepted.dl`. Do NOT present wiki excerpts as confirmed facts. Optionally record the unanswered question for later review (a non-engine-input sink, never `facts/query.dl`): +The wiki renderer applies the same explicit row cap to cited excerpts and +engine-grounding rows. Its warning is printed before those rows, and `--all` +returns every available excerpt and grounding fact for audit. + ```bash "${CLAUDE_PLUGIN_ROOT}/tools/factlog_python.sh" "${CLAUDE_PLUGIN_ROOT}/tools/ask_router.py" note "" --target "$FACTLOG_ROOT" ``` diff --git a/tests/test_ask_router.sh b/tests/test_ask_router.sh index 79435d13..367c28ad 100755 --- a/tests/test_ask_router.sh +++ b/tests/test_ask_router.sh @@ -643,6 +643,65 @@ if hrouter render 'relation("존재안함", "게재연도", O)?' | grep -qF "pos if [ -f "$HKB/facts/query.dl" ]; then bad "#189: coverage-hint path wrote facts/query.dl"; else ok "#189: coverage-hint path never writes facts/query.dl"; fi if [ "$(cat "$HKB/facts/accepted.dl")" = "$HACCEPTED_BEFORE" ]; then ok "#189: coverage-hint path leaves accepted.dl unchanged"; else bad "#189: coverage-hint path mutated accepted.dl"; fi +# --- #279: renderer row caps are explicit and escapable --------------------- +# Test below / at / above the same small cap directly. This keeps the boundary +# deterministic without making the fixture depend on the production default. +if "$PYTHON" -c " +import sys +sys.path.insert(0, '$PLUGIN_ROOT/tools') +from ask_router import render_engine_answer, render_wiki_answer + +rows = [['S1', 'rel', 'O1'], ['S2', 'rel', 'O2'], ['S3', 'rel', 'O3']] +for count in (1, 2): + out = render_engine_answer('relation(S, rel, O)?', rows[:count], limit=2) + assert f'rows: {count}' in out, out + assert 'more rows' not in out, out +out = render_engine_answer('relation(S, rel, O)?', rows, limit=2) +assert 'rows: 3' in out, out +assert 'S1, rel, O1' in out and 'S2, rel, O2' in out and 'S3, rel, O3' not in out, out +assert '… 1 more rows (full output: --all)' in out, out +all_out = render_engine_answer('relation(S, rel, O)?', rows, limit=None) +assert all(row[0] in all_out for row in rows), all_out +assert 'more rows' not in all_out, all_out + +# One varying column is an indexed, lossless projection: the fixed positions and +# every varying value are printed, so the displayed triples can be reconstructed. +same_tail = [['S1', 'rel', 'O'], ['S2', 'rel', 'O'], ['S3', 'rel', 'O']] +signals = {('S1', 'rel', 'O'): {'sources': 1, 'source_paths': ['sources/a.md'], 'confidence': '0.90', 'stale': False}} +compact = render_engine_answer('relation(S, rel, O)?', same_tail, signals, limit=None) +assert 'rows differ only at column 0; fixed: [1] rel, [2] O' in compact, compact +assert all(f' - S{i}' in compact for i in range(1, 4)), compact +assert '← sources/a.md' in compact, compact + +results = [ + {'file': 'sources/a.md', 'line': 1, 'dir': 'sources', 'excerpt': 'alpha'}, + {'file': 'sources/b.md', 'line': 2, 'dir': 'sources', 'excerpt': 'beta'}, + {'file': 'sources/c.md', 'line': 3, 'dir': 'sources', 'excerpt': 'gamma'}, +] +grounding = [ + {'subject': 'S1', 'relation': 'rel', 'object': 'O1'}, + {'subject': 'S2', 'relation': 'rel', 'object': 'O2'}, + {'subject': 'S3', 'relation': 'rel', 'object': 'O3'}, +] +wiki = render_wiki_answer('question', 'reason', results, grounding, limit=2, total_results=3) +assert 'UNVERIFIED — wiki exploration' in wiki, wiki +assert 'WARNING: unverified candidates' in wiki, wiki +assert wiki.index('WARNING: unverified candidates') < wiki.index('VERIFIED — engine (grounding'), wiki +assert 'grounding facts: 3' in wiki and 'S3, rel, O3' not in wiki, wiki +assert '[sources/a.md:1] (sources)' in wiki and '[sources/b.md:2] (sources)' in wiki and '[sources/c.md:3]' not in wiki, wiki +assert wiki.count('… 1 more rows (full output: --all)') == 2, wiki +" 2>/dev/null; then ok "#279: engine/wiki caps retain totals, citations, warning, and explicit truncation"; else bad "#279: renderer cap contract failed"; fi + +# JSON search keeps its existing `results` array and adds an explicit total and +# truncation flag. --all is the lossless escape hatch for the same corpus. +LKB="$(mktemp -d)/wiki" +"$PYTHON" -m factlog init --target "$LKB" >/dev/null +for n in $(seq 1 11); do printf 'limitprobe item %s\n' "$n" > "$LKB/sources/$n.md"; done +limited_search="$("$PYTHON" "$ROUTER" search limitprobe --target "$LKB")" +all_search="$("$PYTHON" "$ROUTER" search limitprobe --all --target "$LKB")" +if printf '%s' "$limited_search" | "$PYTHON" -c "import json,sys; d=json.load(sys.stdin); assert len(d['results']) == 10 and d['total'] == 11 and d['truncated'] is True"; then ok "#279: JSON search exposes capped total and truncation"; else bad "#279: JSON search cap metadata missing/wrong"; fi +if printf '%s' "$all_search" | "$PYTHON" -c "import json,sys; d=json.load(sys.stdin); assert len(d['results']) == 11 and d['total'] == 11 and d['truncated'] is False"; then ok "#279: JSON search --all returns every excerpt"; else bad "#279: JSON search --all is not lossless"; fi + echo "" echo "========================================" echo "test_ask_router: $pass passed, $fail failed" diff --git a/tools/ask_router.py b/tools/ask_router.py index cf7f1769..32324bf9 100644 --- a/tools/ask_router.py +++ b/tools/ask_router.py @@ -29,7 +29,9 @@ Usage: python3 ask_router.py validate "" [--target ] python3 ask_router.py evaluate "" [--target ] - python3 ask_router.py render "" [--target ] + python3 ask_router.py render "" [--all] [--target ] + python3 ask_router.py search "" [--all] [--target ] + python3 ask_router.py wiki "" [--all] [--target ] Each subcommand prints JSON (validate/evaluate) or the rendered answer (render) to stdout. --target overrides FACTLOG_ROOT (authoritative). @@ -89,6 +91,11 @@ ) from factlog import literal_types # noqa: E402 +# Keep the default answer short enough to scan while retaining an explicit, +# deterministic escape hatch for audit work. This cap is deliberately applied +# by renderers, not by an LLM deciding which facts matter. +DEFAULT_RENDER_ROW_LIMIT = 20 + def _policy_program_optional() -> str: """Return the fully assembled policy text — the generated `logic-policy.dl` @@ -469,6 +476,8 @@ def render_engine_answer( rows: list[list[str]], signals: dict[tuple[str, str, str], dict[str, object]] | None = None, annotate_objects: bool = False, + limit: int | None = DEFAULT_RENDER_ROW_LIMIT, + project: bool = True, ) -> str: """Render the VERIFIED — engine answer block (positive or negative). @@ -499,8 +508,17 @@ def render_engine_answer( """ lines = ["VERIFIED — engine", f"query: {draft}", f"rows: {len(rows)}"] if rows: - for row in rows: - line = f" - {', '.join(row)}" + visible_rows = rows if limit is None else rows[:_render_limit(limit)] + projection = _single_column_projection(visible_rows) if project else None + if projection: + varying_index, fixed_columns = projection + fixed = ", ".join(f"[{index}] {value}" for index, value in fixed_columns) + lines.append(f" - rows differ only at column {varying_index}; fixed: {fixed}") + for row in visible_rows: + line = ( + f" - {row[projection[0]]}" + if projection else f" - {', '.join(row)}" + ) # Display-only: annotate a compound-term object (amount/date/number) # with its human-friendly form. Gated to relation rows via # annotate_objects so a coincidental 3-element shape on a path/policy @@ -529,6 +547,9 @@ def render_engine_answer( if sig: for path in sig.get("source_paths", []): lines.append(f" ← {path}") + truncation = _truncation_line(len(rows), len(visible_rows)) + if truncation: + lines.append(truncation) else: lines.append("no such fact (verified negative)") return "\n".join(lines) @@ -643,7 +664,7 @@ def _semantic_rerank(question: str, results: list[dict[str, object]]) -> list[di return results # graceful degrade to lexical ranking -def search(question: str, root: Path, *, limit: int = 10) -> list[dict[str, object]]: +def search(question: str, root: Path, *, limit: int | None = 10) -> list[dict[str, object]]: """Relevance-ranked search over the wiki corpus (sources/ + runs/sources/). Collects keyword-matched excerpts, ranks them by relevance (keyword coverage, @@ -699,10 +720,45 @@ def search(question: str, root: Path, *, limit: int = 10) -> list[dict[str, obje # Rank by relevance (desc); ties keep corpus/line order (stable sort over the # already-ordered collection). Then take the cap, then optional neural rerank. scored.sort(key=lambda item: item[0], reverse=True) - ranked = [result for _score, result in scored][:limit] + ranked = [result for _score, result in scored] + if limit is not None: + ranked = ranked[:limit] return _semantic_rerank(question, ranked) +def _render_limit(value: int | None) -> int | None: + """Translate the public ``--all`` mode to an internal row cap.""" + return None if value is None else max(0, value) + + +def _truncation_line(total: int, shown: int) -> str | None: + """Return an explicit audit escape-hatch notice when rows were omitted.""" + omitted = total - shown + if omitted <= 0: + return None + return f"… {omitted} more rows (full output: --all)" + + +def _single_column_projection(rows: list[list[str]]) -> tuple[int, list[tuple[int, str]]] | None: + """Describe a lossless projection when exactly one column varies. + + The returned column index and fixed indexed values retain enough structure to + reconstruct every displayed row. Provenance stays attached to each varying + value in :func:`render_engine_answer`; this is display compaction, never an + LLM-authored summary. + """ + if len(rows) < 2 or not rows or not rows[0]: + return None + width = len(rows[0]) + if any(len(row) != width for row in rows): + return None + varying = [index for index in range(width) if any(row[index] != rows[0][index] for row in rows[1:])] + if len(varying) != 1: + return None + varying_index = varying[0] + return varying_index, [(index, rows[0][index]) for index in range(width) if index != varying_index] + + def _entity_mentioned(entity: str, question_low: str) -> bool: """Whether an accepted entity name appears in the question (bilingual, matching the keyword matcher's contract): CJK substring (length >= 2); @@ -738,6 +794,8 @@ def render_wiki_answer( reason: str, results: list[dict[str, object]], grounding: list[dict[str, str]] | None = None, + limit: int | None = DEFAULT_RENDER_ROW_LIMIT, + total_results: int | None = None, ) -> str: """Render the UNVERIFIED — wiki exploration answer block. @@ -751,21 +809,33 @@ def render_wiki_answer( "UNVERIFIED — wiki exploration", f"question: {question}", f"reason: {reason}", + "WARNING: unverified candidates — do not treat as confirmed facts.", ] + total_grounding = len(grounding or []) + visible_grounding = (grounding or []) if limit is None else (grounding or [])[:_render_limit(limit)] if grounding: lines.append("") lines.append("VERIFIED — engine (grounding: accepted facts about mentioned entities):") - lines.extend(f" - {row['subject']}, {row['relation']}, {row['object']}" for row in grounding) + lines.append(f"grounding facts: {total_grounding}") + lines.extend(f" - {row['subject']}, {row['relation']}, {row['object']}" for row in visible_grounding) + truncation = _truncation_line(total_grounding, len(visible_grounding)) + if truncation: + lines.append(truncation) lines.append("") lines.append(f"sources searched: {', '.join(label for _rel, label in _wiki_corpus())}") - if results: - for r in results: + result_total = len(results) if total_results is None else total_results + lines.append(f"source excerpts: {result_total}") + visible_results = results if limit is None else results[:_render_limit(limit)] + if visible_results: + for r in visible_results: lines.append(f"[{r['file']}:{r['line']}] ({r['dir']})") for excerpt_line in str(r["excerpt"]).splitlines(): lines.append(f" {excerpt_line}") else: lines.append("(no matching source excerpts found)") - lines.append("WARNING: unverified candidates — do not treat as confirmed facts.") + truncation = _truncation_line(result_total, len(visible_results)) + if truncation: + lines.append(truncation) return "\n".join(lines) @@ -867,6 +937,8 @@ def cmd_render(args: argparse.Namespace) -> int: result["rows"], signals, annotate_objects=is_relation, + limit=None if args.all else DEFAULT_RENDER_ROW_LIMIT, + project=not args.all, )) # The engine answer is real, but if the author wrote policy rules and # never compiled them, the engine had no policy to apply — say so, so a @@ -890,17 +962,40 @@ def cmd_render(args: argparse.Namespace) -> int: def cmd_search(args: argparse.Namespace) -> int: root = Path(os.environ["FACTLOG_ROOT"]) - print(json.dumps({"results": search(args.text, root)}, ensure_ascii=False)) + if args.all: + results = search(args.text, root, limit=None) + total = len(results) + else: + # Keep the existing top-10 retrieval/reranking behaviour for callers of + # the stable ``results`` array. The additive fields make the cap visible. + results = search(args.text, root) + total = len(search(args.text, root, limit=None)) + print(json.dumps( + {"results": results, "total": total, "truncated": len(results) < total}, + ensure_ascii=False, + )) return 0 def cmd_wiki(args: argparse.Namespace) -> int: root = Path(os.environ["FACTLOG_ROOT"]) - results = search(args.text, root) + if args.all: + results = search(args.text, root, limit=None) + total_results = len(results) + else: + results = search(args.text, root) + total_results = len(search(args.text, root, limit=None)) # Grounding: accepted facts about mentioned entities (empty if not compiled yet). accepted = load_accepted_facts() if ACCEPTED_DL.is_file() else [] grounding = grounding_facts(args.text, accepted) - print(render_wiki_answer(args.text, args.reason, results, grounding)) + print(render_wiki_answer( + args.text, + args.reason, + results, + grounding, + limit=None if args.all else DEFAULT_RENDER_ROW_LIMIT, + total_results=total_results, + )) # A wiki answer is already UNVERIFIED, but an uncompiled-but-authored policy # is a separate, actionable defect the author should fix — surface it (#193). if _policy_uncompiled(): @@ -925,18 +1020,21 @@ def build_parser() -> argparse.ArgumentParser: ): p = sub.add_parser(name, help=helptext) p.add_argument("draft", help="the candidate Datalog query line") + p.add_argument("--all", action="store_true", help="show every answer row (no renderer cap)") p.add_argument("--target", default=None, help="KB root (overrides FACTLOG_ROOT)") p.set_defaults(func=func) # Path B (wiki) subcommands take the natural-language question, not a draft. search_p = sub.add_parser("search", help="search the wiki corpus (sources/ + runs/sources/) (JSON)") search_p.add_argument("text", help="the natural-language question") + search_p.add_argument("--all", action="store_true", help="return every matching excerpt") search_p.add_argument("--target", default=None, help="KB root (overrides FACTLOG_ROOT)") search_p.set_defaults(func=cmd_search) wiki_p = sub.add_parser("wiki", help="render the UNVERIFIED — wiki exploration answer") wiki_p.add_argument("text", help="the natural-language question") wiki_p.add_argument("--reason", default="not expressible over accepted facts", help="why the engine path did not apply") + wiki_p.add_argument("--all", action="store_true", help="show every excerpt and grounding row") wiki_p.add_argument("--target", default=None, help="KB root (overrides FACTLOG_ROOT)") wiki_p.set_defaults(func=cmd_wiki)