Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions skills/factlog/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -803,6 +803,11 @@ and carry no extraction confidence by construction. The verdict stays binary in
every case. For an out-of-band trace (any fact, full or partial triple, all
statuses), use `factlog provenance <subject> [relation] [object]`.

The renderer shows at most 20 answer rows by default and explicitly reports any
omitted rows as `… N more rows (full output: --all)` while keeping `rows: N` as
the real total. When the full audit trail is needed, rerun the same command with
`--all`; do not ask the model to select or summarize omitted rows.

A verified-negative relation query may additionally carry an informational
`note: ... (possible predicate mismatch): ...` line (#189). It appears **only**
when the queried subject is an accepted entity that has **no** fact under the
Expand All @@ -828,6 +833,10 @@ unverified excerpts cite only source text, never `facts/accepted.dl`. Do NOT
present wiki excerpts as confirmed facts. Optionally record the unanswered
question for later review (a non-engine-input sink, never `facts/query.dl`):

The wiki renderer applies the same explicit row cap to cited excerpts and
engine-grounding rows. Its warning is printed before those rows, and `--all`
returns every available excerpt and grounding fact for audit.

```bash
"${CLAUDE_PLUGIN_ROOT}/tools/factlog_python.sh" "${CLAUDE_PLUGIN_ROOT}/tools/ask_router.py" note "<question>" --target "$FACTLOG_ROOT"
```
Expand Down
59 changes: 59 additions & 0 deletions tests/test_ask_router.sh
Original file line number Diff line number Diff line change
Expand Up @@ -643,6 +643,65 @@ if hrouter render 'relation("존재안함", "게재연도", O)?' | grep -qF "pos
if [ -f "$HKB/facts/query.dl" ]; then bad "#189: coverage-hint path wrote facts/query.dl"; else ok "#189: coverage-hint path never writes facts/query.dl"; fi
if [ "$(cat "$HKB/facts/accepted.dl")" = "$HACCEPTED_BEFORE" ]; then ok "#189: coverage-hint path leaves accepted.dl unchanged"; else bad "#189: coverage-hint path mutated accepted.dl"; fi

# --- #279: renderer row caps are explicit and escapable ---------------------
# Test below / at / above the same small cap directly. This keeps the boundary
# deterministic without making the fixture depend on the production default.
if "$PYTHON" -c "
import sys
sys.path.insert(0, '$PLUGIN_ROOT/tools')
from ask_router import render_engine_answer, render_wiki_answer

rows = [['S1', 'rel', 'O1'], ['S2', 'rel', 'O2'], ['S3', 'rel', 'O3']]
for count in (1, 2):
out = render_engine_answer('relation(S, rel, O)?', rows[:count], limit=2)
assert f'rows: {count}' in out, out
assert 'more rows' not in out, out
out = render_engine_answer('relation(S, rel, O)?', rows, limit=2)
assert 'rows: 3' in out, out
assert 'S1, rel, O1' in out and 'S2, rel, O2' in out and 'S3, rel, O3' not in out, out
assert '… 1 more rows (full output: --all)' in out, out
all_out = render_engine_answer('relation(S, rel, O)?', rows, limit=None)
assert all(row[0] in all_out for row in rows), all_out
assert 'more rows' not in all_out, all_out

# One varying column is an indexed, lossless projection: the fixed positions and
# every varying value are printed, so the displayed triples can be reconstructed.
same_tail = [['S1', 'rel', 'O'], ['S2', 'rel', 'O'], ['S3', 'rel', 'O']]
signals = {('S1', 'rel', 'O'): {'sources': 1, 'source_paths': ['sources/a.md'], 'confidence': '0.90', 'stale': False}}
compact = render_engine_answer('relation(S, rel, O)?', same_tail, signals, limit=None)
assert 'rows differ only at column 0; fixed: [1] rel, [2] O' in compact, compact
assert all(f' - S{i}' in compact for i in range(1, 4)), compact
assert '← sources/a.md' in compact, compact

results = [
{'file': 'sources/a.md', 'line': 1, 'dir': 'sources', 'excerpt': 'alpha'},
{'file': 'sources/b.md', 'line': 2, 'dir': 'sources', 'excerpt': 'beta'},
{'file': 'sources/c.md', 'line': 3, 'dir': 'sources', 'excerpt': 'gamma'},
]
grounding = [
{'subject': 'S1', 'relation': 'rel', 'object': 'O1'},
{'subject': 'S2', 'relation': 'rel', 'object': 'O2'},
{'subject': 'S3', 'relation': 'rel', 'object': 'O3'},
]
wiki = render_wiki_answer('question', 'reason', results, grounding, limit=2, total_results=3)
assert 'UNVERIFIED — wiki exploration' in wiki, wiki
assert 'WARNING: unverified candidates' in wiki, wiki
assert wiki.index('WARNING: unverified candidates') < wiki.index('VERIFIED — engine (grounding'), wiki
assert 'grounding facts: 3' in wiki and 'S3, rel, O3' not in wiki, wiki
assert '[sources/a.md:1] (sources)' in wiki and '[sources/b.md:2] (sources)' in wiki and '[sources/c.md:3]' not in wiki, wiki
assert wiki.count('… 1 more rows (full output: --all)') == 2, wiki
" 2>/dev/null; then ok "#279: engine/wiki caps retain totals, citations, warning, and explicit truncation"; else bad "#279: renderer cap contract failed"; fi

# JSON search keeps its existing `results` array and adds an explicit total and
# truncation flag. --all is the lossless escape hatch for the same corpus.
LKB="$(mktemp -d)/wiki"
"$PYTHON" -m factlog init --target "$LKB" >/dev/null
for n in $(seq 1 11); do printf 'limitprobe item %s\n' "$n" > "$LKB/sources/$n.md"; done
limited_search="$("$PYTHON" "$ROUTER" search limitprobe --target "$LKB")"
all_search="$("$PYTHON" "$ROUTER" search limitprobe --all --target "$LKB")"
if printf '%s' "$limited_search" | "$PYTHON" -c "import json,sys; d=json.load(sys.stdin); assert len(d['results']) == 10 and d['total'] == 11 and d['truncated'] is True"; then ok "#279: JSON search exposes capped total and truncation"; else bad "#279: JSON search cap metadata missing/wrong"; fi
if printf '%s' "$all_search" | "$PYTHON" -c "import json,sys; d=json.load(sys.stdin); assert len(d['results']) == 11 and d['total'] == 11 and d['truncated'] is False"; then ok "#279: JSON search --all returns every excerpt"; else bad "#279: JSON search --all is not lossless"; fi

echo ""
echo "========================================"
echo "test_ask_router: $pass passed, $fail failed"
Expand Down
122 changes: 110 additions & 12 deletions tools/ask_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@
Usage:
python3 ask_router.py validate "<draft>" [--target <kb>]
python3 ask_router.py evaluate "<draft>" [--target <kb>]
python3 ask_router.py render "<draft>" [--target <kb>]
python3 ask_router.py render "<draft>" [--all] [--target <kb>]
python3 ask_router.py search "<question>" [--all] [--target <kb>]
python3 ask_router.py wiki "<question>" [--all] [--target <kb>]

Each subcommand prints JSON (validate/evaluate) or the rendered answer (render)
to stdout. --target overrides FACTLOG_ROOT (authoritative).
Expand Down Expand Up @@ -89,6 +91,11 @@
)
from factlog import literal_types # noqa: E402

# Keep the default answer short enough to scan while retaining an explicit,
# deterministic escape hatch for audit work. This cap is deliberately applied
# by renderers, not by an LLM deciding which facts matter.
DEFAULT_RENDER_ROW_LIMIT = 20


def _policy_program_optional() -> str:
"""Return the fully assembled policy text — the generated `logic-policy.dl`
Expand Down Expand Up @@ -469,6 +476,8 @@ def render_engine_answer(
rows: list[list[str]],
signals: dict[tuple[str, str, str], dict[str, object]] | None = None,
annotate_objects: bool = False,
limit: int | None = DEFAULT_RENDER_ROW_LIMIT,
project: bool = True,
) -> str:
"""Render the VERIFIED — engine answer block (positive or negative).

Expand Down Expand Up @@ -499,8 +508,17 @@ def render_engine_answer(
"""
lines = ["VERIFIED — engine", f"query: {draft}", f"rows: {len(rows)}"]
if rows:
for row in rows:
line = f" - {', '.join(row)}"
visible_rows = rows if limit is None else rows[:_render_limit(limit)]
projection = _single_column_projection(visible_rows) if project else None
if projection:
varying_index, fixed_columns = projection
fixed = ", ".join(f"[{index}] {value}" for index, value in fixed_columns)
lines.append(f" - rows differ only at column {varying_index}; fixed: {fixed}")
for row in visible_rows:
line = (
f" - {row[projection[0]]}"
if projection else f" - {', '.join(row)}"
)
# Display-only: annotate a compound-term object (amount/date/number)
# with its human-friendly form. Gated to relation rows via
# annotate_objects so a coincidental 3-element shape on a path/policy
Expand Down Expand Up @@ -529,6 +547,9 @@ def render_engine_answer(
if sig:
for path in sig.get("source_paths", []):
lines.append(f" ← {path}")
truncation = _truncation_line(len(rows), len(visible_rows))
if truncation:
lines.append(truncation)
else:
lines.append("no such fact (verified negative)")
return "\n".join(lines)
Expand Down Expand Up @@ -643,7 +664,7 @@ def _semantic_rerank(question: str, results: list[dict[str, object]]) -> list[di
return results # graceful degrade to lexical ranking


def search(question: str, root: Path, *, limit: int = 10) -> list[dict[str, object]]:
def search(question: str, root: Path, *, limit: int | None = 10) -> list[dict[str, object]]:
"""Relevance-ranked search over the wiki corpus (sources/ + runs/sources/).

Collects keyword-matched excerpts, ranks them by relevance (keyword coverage,
Expand Down Expand Up @@ -699,10 +720,45 @@ def search(question: str, root: Path, *, limit: int = 10) -> list[dict[str, obje
# Rank by relevance (desc); ties keep corpus/line order (stable sort over the
# already-ordered collection). Then take the cap, then optional neural rerank.
scored.sort(key=lambda item: item[0], reverse=True)
ranked = [result for _score, result in scored][:limit]
ranked = [result for _score, result in scored]
if limit is not None:
ranked = ranked[:limit]
return _semantic_rerank(question, ranked)


def _render_limit(value: int | None) -> int | None:
"""Translate the public ``--all`` mode to an internal row cap."""
return None if value is None else max(0, value)


def _truncation_line(total: int, shown: int) -> str | None:
"""Return an explicit audit escape-hatch notice when rows were omitted."""
omitted = total - shown
if omitted <= 0:
return None
return f"… {omitted} more rows (full output: --all)"


def _single_column_projection(rows: list[list[str]]) -> tuple[int, list[tuple[int, str]]] | None:
"""Describe a lossless projection when exactly one column varies.

The returned column index and fixed indexed values retain enough structure to
reconstruct every displayed row. Provenance stays attached to each varying
value in :func:`render_engine_answer`; this is display compaction, never an
LLM-authored summary.
"""
if len(rows) < 2 or not rows or not rows[0]:
return None
width = len(rows[0])
if any(len(row) != width for row in rows):
return None
varying = [index for index in range(width) if any(row[index] != rows[0][index] for row in rows[1:])]
if len(varying) != 1:
return None
varying_index = varying[0]
return varying_index, [(index, rows[0][index]) for index in range(width) if index != varying_index]


def _entity_mentioned(entity: str, question_low: str) -> bool:
"""Whether an accepted entity name appears in the question (bilingual,
matching the keyword matcher's contract): CJK substring (length >= 2);
Expand Down Expand Up @@ -738,6 +794,8 @@ def render_wiki_answer(
reason: str,
results: list[dict[str, object]],
grounding: list[dict[str, str]] | None = None,
limit: int | None = DEFAULT_RENDER_ROW_LIMIT,
total_results: int | None = None,
) -> str:
"""Render the UNVERIFIED — wiki exploration answer block.

Expand All @@ -751,21 +809,33 @@ def render_wiki_answer(
"UNVERIFIED — wiki exploration",
f"question: {question}",
f"reason: {reason}",
"WARNING: unverified candidates — do not treat as confirmed facts.",
]
total_grounding = len(grounding or [])
visible_grounding = (grounding or []) if limit is None else (grounding or [])[:_render_limit(limit)]
if grounding:
lines.append("")
lines.append("VERIFIED — engine (grounding: accepted facts about mentioned entities):")
lines.extend(f" - {row['subject']}, {row['relation']}, {row['object']}" for row in grounding)
lines.append(f"grounding facts: {total_grounding}")
lines.extend(f" - {row['subject']}, {row['relation']}, {row['object']}" for row in visible_grounding)
truncation = _truncation_line(total_grounding, len(visible_grounding))
if truncation:
lines.append(truncation)
lines.append("")
lines.append(f"sources searched: {', '.join(label for _rel, label in _wiki_corpus())}")
if results:
for r in results:
result_total = len(results) if total_results is None else total_results
lines.append(f"source excerpts: {result_total}")
visible_results = results if limit is None else results[:_render_limit(limit)]
if visible_results:
for r in visible_results:
lines.append(f"[{r['file']}:{r['line']}] ({r['dir']})")
for excerpt_line in str(r["excerpt"]).splitlines():
lines.append(f" {excerpt_line}")
else:
lines.append("(no matching source excerpts found)")
lines.append("WARNING: unverified candidates — do not treat as confirmed facts.")
truncation = _truncation_line(result_total, len(visible_results))
if truncation:
lines.append(truncation)
return "\n".join(lines)


Expand Down Expand Up @@ -867,6 +937,8 @@ def cmd_render(args: argparse.Namespace) -> int:
result["rows"],
signals,
annotate_objects=is_relation,
limit=None if args.all else DEFAULT_RENDER_ROW_LIMIT,
project=not args.all,
))
# The engine answer is real, but if the author wrote policy rules and
# never compiled them, the engine had no policy to apply — say so, so a
Expand All @@ -890,17 +962,40 @@ def cmd_render(args: argparse.Namespace) -> int:

def cmd_search(args: argparse.Namespace) -> int:
root = Path(os.environ["FACTLOG_ROOT"])
print(json.dumps({"results": search(args.text, root)}, ensure_ascii=False))
if args.all:
results = search(args.text, root, limit=None)
total = len(results)
else:
# Keep the existing top-10 retrieval/reranking behaviour for callers of
# the stable ``results`` array. The additive fields make the cap visible.
results = search(args.text, root)
total = len(search(args.text, root, limit=None))
print(json.dumps(
{"results": results, "total": total, "truncated": len(results) < total},
ensure_ascii=False,
))
return 0


def cmd_wiki(args: argparse.Namespace) -> int:
root = Path(os.environ["FACTLOG_ROOT"])
results = search(args.text, root)
if args.all:
results = search(args.text, root, limit=None)
total_results = len(results)
else:
results = search(args.text, root)
total_results = len(search(args.text, root, limit=None))
# Grounding: accepted facts about mentioned entities (empty if not compiled yet).
accepted = load_accepted_facts() if ACCEPTED_DL.is_file() else []
grounding = grounding_facts(args.text, accepted)
print(render_wiki_answer(args.text, args.reason, results, grounding))
print(render_wiki_answer(
args.text,
args.reason,
results,
grounding,
limit=None if args.all else DEFAULT_RENDER_ROW_LIMIT,
total_results=total_results,
))
# A wiki answer is already UNVERIFIED, but an uncompiled-but-authored policy
# is a separate, actionable defect the author should fix — surface it (#193).
if _policy_uncompiled():
Expand All @@ -925,18 +1020,21 @@ def build_parser() -> argparse.ArgumentParser:
):
p = sub.add_parser(name, help=helptext)
p.add_argument("draft", help="the candidate Datalog query line")
p.add_argument("--all", action="store_true", help="show every answer row (no renderer cap)")
p.add_argument("--target", default=None, help="KB root (overrides FACTLOG_ROOT)")
p.set_defaults(func=func)

# Path B (wiki) subcommands take the natural-language question, not a draft.
search_p = sub.add_parser("search", help="search the wiki corpus (sources/ + runs/sources/) (JSON)")
search_p.add_argument("text", help="the natural-language question")
search_p.add_argument("--all", action="store_true", help="return every matching excerpt")
search_p.add_argument("--target", default=None, help="KB root (overrides FACTLOG_ROOT)")
search_p.set_defaults(func=cmd_search)

wiki_p = sub.add_parser("wiki", help="render the UNVERIFIED — wiki exploration answer")
wiki_p.add_argument("text", help="the natural-language question")
wiki_p.add_argument("--reason", default="not expressible over accepted facts", help="why the engine path did not apply")
wiki_p.add_argument("--all", action="store_true", help="show every excerpt and grounding row")
wiki_p.add_argument("--target", default=None, help="KB root (overrides FACTLOG_ROOT)")
wiki_p.set_defaults(func=cmd_wiki)

Expand Down