From 0c9631d42dfb98cb3b856d6c48884bab37241ed0 Mon Sep 17 00:00:00 2001 From: Dusan Milicevic Date: Fri, 31 Jul 2026 14:52:18 -0500 Subject: [PATCH] Repair inventing article sections instead of dropping them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A section that invented was regenerated once on a shorter budget and then dropped, which told the model to invent less without telling it what it had invented. Any drop also set invent_reject for the whole article, so an otherwise clean stitch was disqualified for a gap in coverage. On the article holdout that disqualified three items of four, one of them a zero-word draft scored against a control arm that had written 135 words. The regenerate now names the fabricated entities and figures it has to lose. If they survive that, the sentences carrying them are cut out and the section is re-checked — the sentence rather than the span, because removing only the name leaves the fabricated claim standing without its subject. A section is dropped only when nothing usable is left, and the whole-article invent flag fires on an empty stitch or a stitched text that still invents, not on the drop itself. The control arm builds its sections through the same path. A fact-lock only the product arm had to survive would separate the arms by editing policy rather than by writing. Article holdout, same carve and model: wins article 3 / base 1 (was 1 / 3), invent disqualifications 0 of 4 on both arms (was 3 of 4 on the article arm), and no arm now stitches an empty draft. The verdict stays not_supported for one remaining reason: with four holdouts a 3-1 margin cannot clear alpha=0.10. Co-authored-by: Cursor --- README.md | 4 +- src/personality_protect/eval_write_article.py | 110 ++++--- src/personality_protect/write_article.py | 299 ++++++++++++++---- src/personality_protect/writer_guards.py | 57 ++++ tests/test_eval_write_article.py | 48 +++ tests/test_write_article.py | 85 ++++- tests/test_writer_guards.py | 52 +++ 7 files changed, 530 insertions(+), 125 deletions(-) diff --git a/README.md b/README.md index bd95b71..495c52b 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,7 @@ personality-protect write --topic "…" --points "…" --json `--topic` and `--points` are the only content the draft may use; retrieved pieces supply rhythm, not facts. Every `write` above runs base weights (`adapter=none`). -On `--channel article`, each `--points` bullet becomes a section (2–8), retrieval is restricted to `linkedin_article` pieces so posts cannot become the rhythm reference, and sections that restate each other are dropped before stitching. Section prompts use an article-specific system rule set: the BRIEF is the only fact source, allowed names/figures from the BRIEF are listed explicitly, thin briefs lower the per-section word aim, and a section that still invents after retry is omitted from the stitch rather than padded in. The channel refuses to draft unless at least five `linkedin_article` pieces are in the corpus *and* five are in the voice index — a large carve that leaves retrieval empty is an error, not a silently thinner draft. +On `--channel article`, each `--points` bullet becomes a section (2–8), retrieval is restricted to `linkedin_article` pieces so posts cannot become the rhythm reference, and sections that restate each other are dropped before stitching. Section prompts use an article-specific system rule set: the BRIEF is the only fact source, allowed names/figures from the BRIEF are listed explicitly, and thin briefs lower the per-section word aim. A section that invents is repaired before it is dropped: the regenerate names the fabricated entities and figures it has to lose, and if they survive that, the sentences carrying them are cut and the section is re-checked. Only a section with nothing left is omitted, and the whole article is flagged for invention only when the stitch is empty or the stitched text still invents. The channel refuses to draft unless at least five `linkedin_article` pieces are in the corpus *and* five are in the voice index — a large carve that leaves retrieval empty is an error, not a silently thinner draft. ### Article holdout eval @@ -215,7 +215,7 @@ PP_MLX_ALLOW=1 personality-protect eval-write-article --out receipt.json The carve is deterministic (`blake2b(piece_id)` order), keeps previously carved ids pinned, and never drops the voice index below the five-article floor. Each holdout is reduced to a lossy brief — a topic plus 3–6 section bullets drawn one per segment of the piece, capped at 60 words and 10% of the source — so neither arm is handed the article back to paraphrase. -Two arms then write the same brief with the same outline, per-section budget, and trim. The product arm gets retrieved exemplars and the measured style card; the control arm gets neither. Invention is judged against the visible brief both arms saw (not the full source article). Drafts are scored on distance to the holdout's own cadence axes, and a draft that parrots its exemplars, echoes the brief, or invents entities or figures is disqualified regardless of distance. Receipts carry ids, distances, and flags — never draft or corpus text. +Two arms then write the same brief with the same outline, per-section budget, trim, and invent repair. The product arm gets retrieved exemplars and the measured style card; the control arm gets neither. Invention is judged against the visible brief both arms saw (not the full source article). Drafts are scored on distance to the holdout's own cadence axes, and a draft that parrots its exemplars, echoes the brief, or invents entities or figures is disqualified regardless of distance. Receipts carry ids, distances, and flags — never draft or corpus text. The verdict needs all three of: the article arm wins the majority, the margin clears `--alpha` (default 0.10) on a one-sided sign test, and it is not disqualified more often than the control. When both arms are disqualified on every holdout, distance never decided anything, and the receipt says so (`distance_ever_decided: false`) rather than reporting it as a cadence loss. diff --git a/src/personality_protect/eval_write_article.py b/src/personality_protect/eval_write_article.py index 1b9fb09..2c11c64 100644 --- a/src/personality_protect/eval_write_article.py +++ b/src/personality_protect/eval_write_article.py @@ -27,6 +27,7 @@ import json from collections.abc import Sequence from datetime import datetime, timezone +from functools import partial from pathlib import Path from typing import Any @@ -40,7 +41,7 @@ from personality_protect.chat_prompt import flatten_chat_messages from personality_protect.config import ProfilePaths, load_config from personality_protect.corpus_text import normalize_corpus_text -from personality_protect.draft_trim import drop_repeated_paragraphs, trim_draft, word_count +from personality_protect.draft_trim import drop_repeated_paragraphs, word_count from personality_protect.eval_write_holdout import ( TIE_EPSILON, assert_receipt_contoso_safe, @@ -50,7 +51,6 @@ write_raw_artifacts, ) from personality_protect.eval_writer_adapter import sign_test_p_value -from personality_protect.prompt_write import build_write_messages from personality_protect.style_profile import ( article_section_words, load_style_profile, @@ -65,10 +65,10 @@ DEFAULT_ARTICLE_SECTION_MAX_TOKENS, SECTION_TOKENS_PER_WORD, SECTION_TRIM_HEADROOM, - THIN_SECTION_WORD_FLOOR, - _guard_flags, _section_brief, + build_section_messages, count_indexed_article_pieces, + draft_section_with_repair, outline_from_brief, run_write_article, scale_section_words_for_brief, @@ -136,56 +136,57 @@ def run_bare_base_article( inventing entities because it had barely written any. The control therefore gets the same outline, the same per-section budget, - and the same trim. What it does not get is the voice machinery — retrieved - exemplars and the measured style profile — which is the only thing the - comparison is meant to be about. + and the same trim — and, since the product arm gained one, the same invent + repair and mechanical scrub. A fact-lock that only the product arm has to + survive would separate the arms by editing policy rather than by writing. + What the control does not get is the voice machinery — retrieved exemplars + and the measured style profile — which is the only thing the comparison is + meant to be about. """ invent_brief = str(budget.get("visible_brief") or build_brief(topic, points)) section_drafts: list[str] = [] + dropped_sections: list[str] = [] + repaired_sections: list[str] = [] + scrubbed_sections: list[str] = [] messages: list[dict[str, str]] = [] + attempts_total = 0 for index, section in enumerate(budget["sections"], start=1): section_topic, section_points = _section_brief(topic, section, points) - messages = build_write_messages( - topic=section_topic, - points=section_points, - examples=(), - channel="article", - style_directives=section_structure_directives( - section=section, - index=index, - total=budget["section_count"], - word_aim=budget["word_aim"], - section_words=budget["section_words"], - section_trim_words=budget["section_trim_words"], - allowed_entities=budget.get("allowed_entities") or (), - allowed_numbers=budget.get("allowed_numbers") or (), + outcome = draft_section_with_repair( + build_messages=partial( + build_section_messages, + topic=section_topic, + points=section_points, + examples=(), + directives=section_structure_directives( + section=section, + index=index, + total=budget["section_count"], + word_aim=budget["word_aim"], + section_words=budget["section_words"], + section_trim_words=budget["section_trim_words"], + allowed_entities=budget.get("allowed_entities") or (), + allowed_numbers=budget.get("allowed_numbers") or (), + ), ), + generate_fn=generate_fn, + base_model=base_model, + invent_brief=invent_brief, + section_words=budget["section_words"], + section_trim_words=budget["section_trim_words"], + max_tokens=budget["max_tokens"], + prompt_sink=prompt_sink, ) - draft = "" - kept = False - for attempt in range(1, 3): - attempt_trim = budget["section_trim_words"] - attempt_tokens = budget["max_tokens"] - if attempt > 1: - attempt_trim = max(THIN_SECTION_WORD_FLOOR, budget["section_words"] // 2) - attempt_tokens = max( - 256, int(round(attempt_trim * SECTION_TOKENS_PER_WORD)) - ) - raw = str( - generate_fn( - messages, - base_model=base_model, - max_tokens=attempt_tokens, - prompt_sink=prompt_sink, - ) - ).strip() - draft = trim_draft(raw, max_words=attempt_trim) - guards = _guard_flags(invent_brief, draft, ()) - if not guards["parrot_reject"] and not guards["invent_reject"]: - kept = True - break - if kept and draft.strip(): - section_drafts.append(draft) + messages = outcome["messages"] + attempts_total += int(outcome["attempts"]) + if outcome["status"] == "dropped": + dropped_sections.append(section) + continue + section_drafts.append(str(outcome["draft"])) + if outcome["status"] == "repaired": + repaired_sections.append(section) + elif outcome["status"] == "scrubbed": + scrubbed_sections.append(section) text = drop_repeated_paragraphs( "\n\n".join(part for part in section_drafts if part.strip()) @@ -198,6 +199,10 @@ def run_bare_base_article( "k": 0, "exemplar_ids": [], "section_count": len(section_drafts), + "attempts": attempts_total, + "dropped_sections": dropped_sections, + "repaired_sections": repaired_sections, + "scrubbed_sections": scrubbed_sections, "prompt": flatten_chat_messages(messages) if budget["sections"] else "", } @@ -246,6 +251,14 @@ def _item_receipt( "article_draft_words": len(str(article_result.get("text") or "").split()), "base_draft_words": len(str(base_result.get("text") or "").split()), "article_attempts": int(article_result.get("attempts") or 1), + "base_attempts": int(base_result.get("attempts") or 1), + # Counts, never titles: the outline is mined from the holdout body. + "article_repaired_sections": len(article_result.get("repaired_sections") or []), + "article_scrubbed_sections": len(article_result.get("scrubbed_sections") or []), + "article_dropped_sections": len(article_result.get("dropped_sections") or []), + "base_repaired_sections": len(base_result.get("repaired_sections") or []), + "base_scrubbed_sections": len(base_result.get("scrubbed_sections") or []), + "base_dropped_sections": len(base_result.get("dropped_sections") or []), "exemplar_ids": list(article_result.get("exemplar_ids") or []), "article_k": int(article_result.get("k") or 0), } @@ -377,8 +390,9 @@ def run_eval_write_article( tie_epsilon=tie_epsilon, visible_brief=visible, ) - # Sections dropped for invent never reach the stitch; surface that as - # invent DQ even when the remaining text is empty or brief-clean. + # The product arm sets this when the stitch is empty or the stitched + # text still invents — the two cases the scorer cannot see for itself, + # since an empty draft invents nothing. if article_result.get("invent_reject"): score["rag"]["invent_reject"] = True score["rag"]["disqualified"] = True diff --git a/src/personality_protect/write_article.py b/src/personality_protect/write_article.py index c2a07c5..aae7c94 100644 --- a/src/personality_protect/write_article.py +++ b/src/personality_protect/write_article.py @@ -9,7 +9,8 @@ import json import re -from collections.abc import Sequence +from collections.abc import Callable, Sequence +from functools import partial from typing import Any from personality_protect.chat_prompt import flatten_chat_messages @@ -41,6 +42,7 @@ check_invention, mask_exemplar_entities, parrot_reject, + scrub_invented_sentences, ) ARTICLE_SOURCES: tuple[str, ...] = ("linkedin_article",) @@ -69,6 +71,9 @@ THIN_BRIEF_WORDS = 80 THIN_SECTION_WORD_FLOOR = 80 THIN_WORDS_PER_BRIEF_WORD = 2.5 +# Floor for the repair pass: the budget is already halved, and a section that +# runs out of tokens mid-sentence is a new failure, not a repaired one. +MIN_REPAIR_TOKENS = 256 _BULLET_RE = re.compile(r"^\s*[-*•]\s+") @@ -242,6 +247,71 @@ def section_structure_directives( return lines +def section_repair_directives( + *, + invented_entities: Sequence[str] = (), + invented_numbers: Sequence[str] = (), + section_words: int, +) -> list[str]: + """Name the facts a failed section has to lose before it is written again. + + The regenerate used to differ from the first attempt only by a shorter + budget, which asks for less invention without saying what was invented; the + holdout answered with the same fabricated names inside a shorter section. + + Listing the offenders is a calculated risk. A token put in front of this + model tends to come back out of it — the reason exemplar masking redacts + names instead of labelling them — so the directive is phrased as removal + and :func:`~personality_protect.writer_guards.scrub_invented_sentences` + stands behind it for the case where the model repeats what it was told to + drop. + """ + entities = [str(item).strip() for item in invented_entities if str(item).strip()] + numbers = [str(item).strip() for item in invented_numbers if str(item).strip()] + lines = [ + "REPAIR: your previous draft of this section stated facts the BRIEF " + "never gave. Write the section again, from the BRIEF only.", + ] + if entities: + lines.append( + "Remove these names completely — do not mention them, rename them, " + "or replace them with other names: " + ", ".join(entities) + "." + ) + if numbers: + lines.append( + "Remove these figures completely and put no figures in their " + "place: " + ", ".join(numbers) + "." + ) + lines.append( + "Add no companies, people, products, places, or figures of your own. " + "Make the claim smaller instead of sourcing it." + ) + lines.append(f"Write a shorter section this time: about {section_words} words.") + return lines + + +def build_section_messages( + extra_directives: Sequence[str] = (), + *, + topic: str, + points: str, + examples: Sequence[str], + directives: Sequence[str], +) -> list[dict[str, str]]: + """Article-section prompt with room for an extra directive block. + + Both arms build their sections through here, so a repair directive reaches + the control arm in the same position it reaches the product arm. + """ + return build_write_messages( + topic=topic, + points=points, + examples=examples, + channel="article", + style_directives=[*directives, *extra_directives], + ) + + def _guard_flags(brief: str, draft: str, exemplars: Sequence[str]) -> dict[str, Any]: invention = check_invention(brief, normalize_sentence_case(draft)) return { @@ -252,6 +322,110 @@ def _guard_flags(brief: str, draft: str, exemplars: Sequence[str]) -> dict[str, } +def draft_section_with_repair( + *, + build_messages: Callable[[Sequence[str]], list[dict[str, str]]], + generate_fn: GenerateFn, + base_model: str, + invent_brief: str, + section_words: int, + section_trim_words: int, + max_tokens: int, + exemplars: Sequence[str] = (), + prompt_sink: PromptSink | None = None, +) -> dict[str, Any]: + """Draft one section, then repair it, then scrub it, and only then drop it. + + ``status`` says which of those the section survived: ``clean`` (the first + draft passed the guards), ``repaired`` (the regenerate with the offenders + named passed), ``scrubbed`` (it passed only after the inventing sentences + were cut out), or ``dropped`` (nothing usable survived). + + Dropping is last rather than second because it is the expensive outcome. An + inventing section used to be dropped outright, so a thin brief could take + every section out and leave an empty article that still counted as a draft + — the holdout produced exactly that, a zero-word arm scored against a base + arm that had written 135 words. + """ + repair_words = max(THIN_SECTION_WORD_FLOOR, int(section_words) // 2) + draft = "" + guards: dict[str, Any] = { + "parrot_reject": False, + "invent_reject": False, + "invented_entities": [], + "invented_numbers": [], + } + attempts = 0 + first_messages: list[dict[str, str]] = [] + for attempt in (1, 2): + if attempt == 1: + extra: Sequence[str] = () + trim_words = int(section_trim_words) + attempt_tokens = int(max_tokens) + else: + # Second pass: half the budget, plus the offender list when there + # is one. A parrot failure has nothing to list, so it keeps the + # shorter-budget retry on its own. + trim_words = repair_words + attempt_tokens = max( + MIN_REPAIR_TOKENS, int(round(trim_words * SECTION_TOKENS_PER_WORD)) + ) + extra = ( + section_repair_directives( + invented_entities=guards["invented_entities"], + invented_numbers=guards["invented_numbers"], + section_words=trim_words, + ) + if guards["invent_reject"] + else () + ) + messages = build_messages(extra) + if attempt == 1: + first_messages = messages + attempts += 1 + raw = str( + generate_fn( + messages, + base_model=base_model, + max_tokens=attempt_tokens, + prompt_sink=prompt_sink, + ) + ).strip() + draft = trim_draft(raw, max_words=trim_words) + guards = _guard_flags(invent_brief, draft, exemplars) + clean = not guards["parrot_reject"] and not guards["invent_reject"] + if clean and draft.strip(): + return { + "draft": draft, + "status": "clean" if attempt == 1 else "repaired", + "attempts": attempts, + "guards": guards, + "messages": first_messages, + } + + if guards["invent_reject"]: + scrubbed = scrub_invented_sentences( + draft, invent_brief, normalize=normalize_sentence_case + ) + if scrubbed.strip(): + scrub_guards = _guard_flags(invent_brief, scrubbed, exemplars) + if not scrub_guards["parrot_reject"] and not scrub_guards["invent_reject"]: + return { + "draft": scrubbed, + "status": "scrubbed", + "attempts": attempts, + "guards": scrub_guards, + "messages": first_messages, + } + return { + "draft": "", + "status": "dropped", + "attempts": attempts, + "guards": guards, + "messages": first_messages, + } + + def run_write_article( topic: str, points: str, @@ -263,7 +437,7 @@ def run_write_article( prompt_sink: PromptSink | None = None, min_articles: int = MIN_ARTICLE_CORPUS, ) -> dict[str, Any]: - """Outline → per-section RAG draft → stitch into one article.""" + """Outline → per-section RAG draft → repair/scrub → stitch into one article.""" topic = topic.strip() points = points.strip() if not topic: @@ -322,71 +496,57 @@ def run_write_article( section_drafts: list[str] = [] dropped_sections: list[str] = [] + repaired_sections: list[str] = [] + scrubbed_sections: list[str] = [] all_messages: list[list[dict[str, str]]] = [] attempts_total = 0 dropped_invented_entities: set[str] = set() dropped_invented_numbers: set[str] = set() - last_guards: dict[str, Any] = { - "parrot_reject": False, - "invent_reject": False, - "invented_entities": [], - "invented_numbers": [], - } - for section in sections: + for index, section in enumerate(sections, start=1): section_topic, section_points = _section_brief(topic, section, points) - invent_brief = full_brief - messages = build_write_messages( - topic=section_topic, - points=section_points, - examples=masked, - channel="article", - style_directives=[ - *directives, - *section_structure_directives( - section=section, - index=len(section_drafts) + len(dropped_sections) + 1, - total=len(sections), - word_aim=word_aim, - section_words=section_words, - section_trim_words=section_trim_words, - allowed_entities=allowed["entities"], - allowed_numbers=allowed["numbers"], - ), - ], + outcome = draft_section_with_repair( + build_messages=partial( + build_section_messages, + topic=section_topic, + points=section_points, + examples=masked, + directives=[ + *directives, + *section_structure_directives( + section=section, + index=index, + total=len(sections), + word_aim=word_aim, + section_words=section_words, + section_trim_words=section_trim_words, + allowed_entities=allowed["entities"], + allowed_numbers=allowed["numbers"], + ), + ], + ), + generate_fn=generator, + base_model=model_id, + invent_brief=full_brief, + section_words=section_words, + section_trim_words=section_trim_words, + max_tokens=section_max_tokens, + exemplars=exemplars, + prompt_sink=prompt_sink, ) - all_messages.append(messages) - draft = "" - kept = False - for attempt in range(1, 3): - attempts_total += 1 - # Second attempt: stricter short budget if the first invents. - attempt_trim = section_trim_words - attempt_tokens = section_max_tokens - if attempt > 1: - attempt_trim = max(THIN_SECTION_WORD_FLOOR, section_words // 2) - attempt_tokens = max( - 256, int(round(attempt_trim * SECTION_TOKENS_PER_WORD)) - ) - raw = str( - generator( - messages, - base_model=model_id, - max_tokens=attempt_tokens, - prompt_sink=prompt_sink, - ) - ).strip() - draft = trim_draft(raw, max_words=attempt_trim) - last_guards = _guard_flags(invent_brief, draft, exemplars) - if not last_guards["parrot_reject"] and not last_guards["invent_reject"]: - kept = True - break - if kept and draft.strip(): - section_drafts.append(draft) - else: + all_messages.append(outcome["messages"]) + attempts_total += int(outcome["attempts"]) + if outcome["status"] == "dropped": + guards = outcome["guards"] dropped_sections.append(section) - dropped_invented_entities.update(last_guards.get("invented_entities") or []) - dropped_invented_numbers.update(last_guards.get("invented_numbers") or []) + dropped_invented_entities.update(guards.get("invented_entities") or []) + dropped_invented_numbers.update(guards.get("invented_numbers") or []) + continue + section_drafts.append(str(outcome["draft"])) + if outcome["status"] == "repaired": + repaired_sections.append(section) + elif outcome["status"] == "scrubbed": + scrubbed_sections.append(section) # Sections are generated independently from the same brief, so two of them # can arrive as the same paragraph. Stitching them unfiltered is what turns @@ -394,17 +554,14 @@ def run_write_article( text = drop_repeated_paragraphs( "\n\n".join(part for part in section_drafts if part.strip()) ).strip() - # Final invent check against the full brief the author supplied. + # Final invent check against the full brief the author supplied. The flag + # describes the text that ships: a section dropped for inventing is a gap + # in coverage, not a fabrication in the draft. Forcing invent_reject on any + # drop disqualified whole articles whose remaining text was clean — and on + # the holdout that was three of four items. What still fails the article is + # an empty stitch, or a stitch that invents against the visible brief. final_guards = _guard_flags(full_brief, text, exemplars) - if dropped_sections: - final_guards["invent_reject"] = True - final_guards["invented_entities"] = sorted( - set(final_guards["invented_entities"]) | dropped_invented_entities - ) - final_guards["invented_numbers"] = sorted( - set(final_guards["invented_numbers"]) | dropped_invented_numbers - ) - if not section_drafts: + if not text: final_guards["invent_reject"] = True return { @@ -426,6 +583,10 @@ def run_write_article( "sections": sections, "section_count": len(section_drafts), "dropped_sections": dropped_sections, + "repaired_sections": repaired_sections, + "scrubbed_sections": scrubbed_sections, + "dropped_invented_entities": sorted(dropped_invented_entities), + "dropped_invented_numbers": sorted(dropped_invented_numbers), "draft_words": word_count(text), "allowed_entities": allowed["entities"], "allowed_numbers": allowed["numbers"], diff --git a/src/personality_protect/writer_guards.py b/src/personality_protect/writer_guards.py index 220cb01..eae50f5 100644 --- a/src/personality_protect/writer_guards.py +++ b/src/personality_protect/writer_guards.py @@ -17,6 +17,7 @@ from __future__ import annotations import re +from collections.abc import Callable from dataclasses import dataclass from typing import Iterable @@ -42,6 +43,7 @@ BRIEF_ECHO_MIN_TOKENS = 12 _WORD_RE = re.compile(r"[A-Za-z0-9]+(?:['’][A-Za-z0-9]+)?") +_SENTENCE_END_RE = re.compile(r"(?<=[.!?])\s+") _ENTITY_RE = re.compile( r"\b(?:[A-Z]{2,}|[A-Z](?:&[A-Z])+|" r"[A-Z][a-z][a-zA-Z0-9'-]*(?:\s+[A-Z][a-z][a-zA-Z0-9'-]*)*)\b" @@ -506,6 +508,61 @@ def check_invention(brief: str, draft: str) -> InventionResult: ) +def scrub_invented_sentences( + draft: str, + brief: str, + *, + normalize: Callable[[str], str] | None = None, + max_passes: int = 3, +) -> str: + """Cut the sentences carrying names or figures the brief never granted. + + Last resort after a repair regenerate has already failed. The sentence, not + the span, is the unit of removal: cutting only the fabricated name out of + "Fabrikam shipped the migration in nine weeks" leaves the fabricated claim + standing with its subject missing, which still reads as prose and is still + false. Dropping the sentence removes the claim. + + Lineation survives — sentences are dropped inside their own line — so a + scrubbed section keeps the paragraph rhythm the rest of the pipeline + measures. Returns ``""`` when nothing survives, which is the caller's + signal to drop the section. + + ``normalize`` is the draft-side case normalizer the caller's invention + guard applies before checking (sentence-initial verbs are capitalized by + syntax, not because they are names). It is injected rather than imported + because it lives above this module in the import order. + """ + + text = draft or "" + prepare = normalize or (lambda value: value) + for _ in range(max(1, int(max_passes))): + if not text.strip() or check_invention(brief, prepare(text)).passed: + break + lines: list[str] = [] + dropped = 0 + for line in text.splitlines(): + if not line.strip(): + lines.append("") + continue + sentences = [part for part in _SENTENCE_END_RE.split(line) if part.strip()] + kept = [ + part + for part in sentences + if check_invention(brief, prepare(part)).passed + ] + dropped += len(sentences) - len(kept) + lines.append(" ".join(kept).strip()) + if not dropped: + # The whole draft invents something no single sentence owns; more + # passes would only repeat this one. + break + text = "\n".join(lines) + for pattern, replacement in _TIDY_RULES: + text = pattern.sub(replacement, text) + return text.strip() + + def brief_allowed_facts(brief: str) -> dict[str, list[str]]: """Entities and figures the draft may reuse from the visible brief. diff --git a/tests/test_eval_write_article.py b/tests/test_eval_write_article.py index eb07b79..a415e11 100644 --- a/tests/test_eval_write_article.py +++ b/tests/test_eval_write_article.py @@ -117,6 +117,54 @@ def fake_generate(messages, **_kwargs: object) -> str: assert "Never use these words" not in seen[0] +def test_control_arm_repairs_and_scrubs_like_the_product_arm(tmp_path: Path): + """A fact-lock only one arm has to survive would decide the comparison.""" + paths, _ = _seed(tmp_path) + budget = article_word_budget(paths, "Contoso packaging", "- One\n- Two\n- Three") + seen: list[str] = [] + + def fake_generate(messages, **_kwargs: object) -> str: + prompt = messages[1]["content"] + seen.append(prompt) + if "REPAIR:" not in prompt: + return "The packaging work ran at Fabrikam Northwind for nine weeks." + return "Contoso packaging keeps one owner and one published list." + + result = run_bare_base_article( + "Contoso packaging", + "- One\n- Two\n- Three", + budget=budget, + generate_fn=fake_generate, + base_model="contoso-local", + ) + assert result["section_count"] == 3 + assert len(result["repaired_sections"]) == 3 + assert result["dropped_sections"] == [] + assert "Fabrikam" not in result["text"] + repairs = [prompt for prompt in seen if "REPAIR:" in prompt] + assert len(repairs) == 3 + assert "northwind" in repairs[0].lower() + # Still no voice machinery, repair or not. + assert "EXAMPLES" not in repairs[0] + + +def test_control_arm_that_cannot_be_repaired_still_disqualifies(tmp_path: Path): + paths, holdouts = _carve(tmp_path) + invented = "The rollout ran at Fabrikam Northwind across two Tailspin Toys sites." + receipt = run_eval_write_article( + paths, + [holdouts[0]], + k=1, + generate_fn=lambda _m, **_k: ARTICLE_DRAFT, + generate_fn_base=lambda _m, **_k: invented, + ) + item = receipt["items"][0] + assert item["base_dropped_sections"] == item["section_count"] + assert item["base_invent_reject"] + assert item["base_disqualified"] + assert item["winner"] != "base" + + def test_both_arms_see_the_same_outline_and_budget(tmp_path: Path): paths, holdouts = _carve(tmp_path) article_prompts: list[str] = [] diff --git a/tests/test_write_article.py b/tests/test_write_article.py index b8b9dc9..78b41ab 100644 --- a/tests/test_write_article.py +++ b/tests/test_write_article.py @@ -202,7 +202,80 @@ def test_article_retrieval_never_mixes_in_posts(tmp_path: Path): assert all(piece_id.startswith("contoso-article") for piece_id in result["exemplar_ids"]) -def test_inventing_section_is_dropped_from_the_stitch(tmp_path: Path): +def test_inventing_section_is_repaired_instead_of_dropped(tmp_path: Path): + """The regenerate is told which names to lose, and the section survives.""" + _seed_articles(tmp_path, n=MIN_ARTICLE_CORPUS) + paths, _, _ = init_profile("contoso", home=tmp_path) + seen: list[str] = [] + + def fake_generate(messages, **_kwargs: object) -> str: + prompt = messages[1]["content"] + seen.append(prompt) + if "REPAIR:" not in prompt: + return "The packaging programme ran at Fabrikam Northwind for nine weeks." + return "Contoso names one owner for packaging and writes that decision down." + + result = run_write_article( + "Contoso packaging", BRIEF_POINTS, paths, k=1, generate_fn=fake_generate + ) + assert result["section_count"] == 3 + assert result["dropped_sections"] == [] + assert len(result["repaired_sections"]) == 3 + assert result["scrubbed_sections"] == [] + assert "Fabrikam" not in result["text"] + assert not result["invent_reject"] + + repairs = [prompt for prompt in seen if "REPAIR:" in prompt] + assert len(repairs) == 3 + # The offenders are named, or the model is only being asked to guess again. + assert "fabrikam" in repairs[0].lower() + assert "9 weeks" in repairs[0] + + +def test_scrub_keeps_the_clean_sentences_when_repair_still_invents(tmp_path: Path): + """A section the repair could not fix is cut down, not thrown away.""" + _seed_articles(tmp_path, n=MIN_ARTICLE_CORPUS) + paths, _, _ = init_profile("contoso", home=tmp_path) + + def fake_generate(_messages, **_kwargs: object) -> str: + return ( + "One owner signs the packaging decision before a tier opens.\n" + "\n" + "Fabrikam Northwind published the same list first. Cut the " + "exceptions weekly and read the number out loud." + ) + + result = run_write_article( + "Contoso packaging", BRIEF_POINTS, paths, k=1, generate_fn=fake_generate + ) + assert result["section_count"] >= 1 + assert result["dropped_sections"] == [] + assert len(result["scrubbed_sections"]) == 3 + assert "Fabrikam" not in result["text"] + assert "One owner signs the packaging decision" in result["text"] + assert not result["invent_reject"] + + +def test_section_is_emptied_only_when_repair_and_scrub_both_fail(tmp_path: Path): + """Nothing left to keep is the one case that still disqualifies the article.""" + _seed_articles(tmp_path, n=MIN_ARTICLE_CORPUS) + paths, _, _ = init_profile("contoso", home=tmp_path) + + def fake_generate(_messages, **_kwargs: object) -> str: + return "Fabrikam Northwind invented a packaging vendor nobody named." + + result = run_write_article( + "Contoso packaging", BRIEF_POINTS, paths, k=1, generate_fn=fake_generate + ) + assert result["section_count"] == 0 + assert len(result["dropped_sections"]) == 3 + assert result["text"] == "" + assert result["invent_reject"] + assert result["dropped_invented_entities"] + + +def test_a_dropped_section_does_not_disqualify_a_clean_stitch(tmp_path: Path): + """One unrepairable section is a coverage gap, not a fabricated article.""" _seed_articles(tmp_path, n=MIN_ARTICLE_CORPUS) paths, _, _ = init_profile("contoso", home=tmp_path) calls = {"n": 0} @@ -210,18 +283,18 @@ def test_inventing_section_is_dropped_from_the_stitch(tmp_path: Path): def fake_generate(_messages, **_kwargs: object) -> str: calls["n"] += 1 if calls["n"] <= 2: - return ( - "Fabrikam Northwind invented a packaging vendor Contoso never named." - ) + return "Fabrikam Northwind invented a packaging vendor nobody named." return "Contoso names one owner and cuts exceptions. Keep Ledger boring." result = run_write_article( "Contoso packaging", BRIEF_POINTS, paths, k=1, generate_fn=fake_generate ) assert result["dropped_sections"] + assert result["section_count"] == 2 assert "Fabrikam" not in result["text"] - assert result["invent_reject"] - assert result["invented_entities"] + assert not result["invent_reject"] + assert result["invented_entities"] == [] + assert result["dropped_invented_entities"] def test_run_write_channel_article_delegates(tmp_path: Path): diff --git a/tests/test_writer_guards.py b/tests/test_writer_guards.py index 9eb315b..5b6fb55 100644 --- a/tests/test_writer_guards.py +++ b/tests/test_writer_guards.py @@ -2,6 +2,7 @@ from __future__ import annotations +from personality_protect.write import normalize_sentence_case from personality_protect.writer_guards import ( brief_echo_reject, check_invention, @@ -10,8 +11,11 @@ find_scaffold_markers, mask_exemplar_entities, parrot_reject, + scrub_invented_sentences, ) +SCRUB_BRIEF = "Topic: Contoso packaging\nPoints:\n- Name one owner\n- Cut exceptions" + def test_mask_exemplar_entities_absent_from_brief(): brief = "Contoso is launching Ledger for operations teams." @@ -289,3 +293,51 @@ def test_brief_echo_rejects_the_bullets_handed_back(): ) assert brief_echo_reject(echo, brief) is True + + +def test_scrub_cuts_the_inventing_sentence_and_keeps_the_rest(): + draft = ( + "One owner signs off on packaging before a tier opens.\n" + "\n" + "Fabrikam Northwind ran the same programme first. The exception list " + "gets read out loud every week." + ) + + scrubbed = scrub_invented_sentences( + draft, SCRUB_BRIEF, normalize=normalize_sentence_case + ) + + assert "Fabrikam" not in scrubbed + assert "One owner signs off on packaging" in scrubbed + assert "read out loud every week" in scrubbed + # Paragraph break survives, so the section keeps its lineation. + assert "\n\n" in scrubbed + assert check_invention(SCRUB_BRIEF, normalize_sentence_case(scrubbed)).passed + + +def test_scrub_cuts_the_sentence_carrying_an_invented_figure(): + draft = ( + "The owner is named in writing.\n" + "We cleared nine exceptions in three weeks and stopped counting." + ) + + scrubbed = scrub_invented_sentences( + draft, SCRUB_BRIEF, normalize=normalize_sentence_case + ) + + assert "nine exceptions" not in scrubbed + assert scrubbed == "The owner is named in writing." + + +def test_scrub_returns_nothing_when_every_sentence_invents(): + draft = ( + "Fabrikam Northwind shipped the tier. Tailspin Toys confirmed the " + "rollout afterwards." + ) + + assert ( + scrub_invented_sentences( + draft, SCRUB_BRIEF, normalize=normalize_sentence_case + ) + == "" + )